From e0c870744e1485fa5762d20869a96f1659ec278f Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 08:03:12 +0800 Subject: [PATCH 1/9] fix(corpus): select root-eligible immutable samples --- .github/workflows/corpus.yml | 39 +---- tests/corpus/corpus-resolve.test.ts | 160 +++++++++++++++++ tests/corpus/corpus-scan.test.ts | 16 ++ tests/workflows/ci-policy.test.ts | 62 ++++--- tools/corpus-lib.ts | 20 ++- tools/corpus-resolve.ts | 260 ++++++++++++++++++++++++++++ 6 files changed, 502 insertions(+), 55 deletions(-) create mode 100644 tests/corpus/corpus-resolve.test.ts create mode 100644 tools/corpus-resolve.ts diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index e90f5f8..ff94f3a 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -37,12 +37,11 @@ jobs: --jq '.items[].full_name' > "$CANDIDATE_FILE" gh api 'search/repositories?q=language:javascript+stars:%3E5000&sort=stars&order=desc&per_page=100' \ --jq '.items[].full_name' >> "$CANDIDATE_FILE" - - name: Select the exact deterministic repository sample + - name: Prepare the deterministic repository candidate pool env: REPO_COUNT: ${{ inputs.repo-count || '100' }} CANDIDATE_FILE: ${{ runner.temp }}/repository-candidates.txt - SELECTED_FILE: ${{ runner.temp }}/repository-names.txt - SELECTION_FILE: repository-sample.json + SELECTED_FILE: ${{ runner.temp }}/repository-candidate-pool.txt run: | set -euo pipefail [[ "$REPO_COUNT" =~ ^([1-9]|[1-9][0-9]|100)$ ]] @@ -57,8 +56,7 @@ jobs: const requested = Number(countText); const candidateFile = process.env.CANDIDATE_FILE; const selectedFile = process.env.SELECTED_FILE; - const selectionFile = process.env.SELECTION_FILE; - if (!candidateFile || !selectedFile || !selectionFile) { + if (!candidateFile || !selectedFile) { throw new Error('corpus selection file paths are required'); } @@ -76,35 +74,14 @@ jobs: `requested ${requested} repositories but only ${uniqueCandidates.length} unique candidates were returned`, ); } - const selected = uniqueCandidates.slice(0, requested); - writeFileSync(selectedFile, `${selected.join('\n')}\n`, 'utf8'); - writeFileSync( - selectionFile, - `${JSON.stringify({ schemaVersion: 1, requested, actual: selected.length }, null, 2)}\n`, - 'utf8', - ); + writeFileSync(selectedFile, `${uniqueCandidates.join('\n')}\n`, 'utf8'); NODE - - name: Resolve immutable repository commits + - name: Resolve the exact root-eligible repository sample env: - GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} REPO_COUNT: ${{ inputs.repo-count || '100' }} - SELECTED_FILE: ${{ runner.temp }}/repository-names.txt - run: | - set -euo pipefail - : > repos.txt - while IFS= read -r repository; do - [[ "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] - default_branch=$(gh api "repos/$repository" --jq '.default_branch') - [[ -n "$default_branch" ]] - commit=$(gh api -X GET "repos/$repository/commits" \ - -f sha="$default_branch" -f per_page=1 --jq '.[0].sha') - [[ "$commit" =~ ^[0-9a-f]{40}$ ]] - printf '%s@%s\n' "$repository" "$commit" >> repos.txt - done < "$SELECTED_FILE" - sort -u repos.txt -o repos.txt - awk '!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+@[0-9a-f]{40}$/ { exit 1 }' repos.txt - ACTUAL_COUNT=$(wc -l < repos.txt) - test "$ACTUAL_COUNT" -eq "$REPO_COUNT" + CANDIDATE_FILE: ${{ runner.temp }}/repository-candidate-pool.txt + run: pnpm exec tsx tools/corpus-resolve.ts "$CANDIDATE_FILE" repos.txt repository-sample.json "$REPO_COUNT" - name: Scan scripts without writing to sampled repositories env: GITHUB_TOKEN: ${{ github.token }} diff --git a/tests/corpus/corpus-resolve.test.ts b/tests/corpus/corpus-resolve.test.ts new file mode 100644 index 0000000..a13509e --- /dev/null +++ b/tests/corpus/corpus-resolve.test.ts @@ -0,0 +1,160 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { afterEach, expect, it } from 'vitest'; + +const ROOTLESS_COMMIT = '1111111111111111111111111111111111111111'; +const FIRST_ELIGIBLE_COMMIT = '2222222222222222222222222222222222222222'; +const SECOND_ELIGIBLE_COMMIT = '3333333333333333333333333333333333333333'; +const temporaryDirectories: string[] = []; + +type Resolver = (options: { + candidateFile: string; + outputFile: string; + evidenceFile: string; + requested: number; + token: string; + fetchImpl: typeof fetch; +}) => Promise; + +function temporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), 'scriptspect-corpus-resolve-test-')); + temporaryDirectories.push(directory); + return directory; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +async function resolver(): Promise { + const resolverUrl = pathToFileURL(join(process.cwd(), 'tools', 'corpus-resolve.ts')).href; + const module = (await import(resolverUrl).catch(() => ({}))) as { + resolveCorpusSample?: Resolver; + }; + expect(typeof module.resolveCorpusSample).toBe('function'); + if (module.resolveCorpusSample === undefined) throw new Error('corpus resolver was unavailable'); + return module.resolveCorpusSample; +} + +function candidateApi(): typeof fetch { + const commits = new Map([ + ['alpha/rootless', ROOTLESS_COMMIT], + ['beta/eligible', FIRST_ELIGIBLE_COMMIT], + ['gamma/eligible', SECOND_ELIGIBLE_COMMIT], + ]); + + return (async (input: string | URL | Request) => { + const url = new URL(String(input)); + const repository = [...commits.keys()].find( + (candidate) => + url.pathname === `/repos/${candidate}` || url.pathname.startsWith(`/repos/${candidate}/`), + ); + if (repository === undefined) return new Response('missing repository', { status: 404 }); + + if (url.pathname === `/repos/${repository}`) { + return Response.json({ default_branch: 'main' }); + } + if (url.pathname === `/repos/${repository}/commits`) { + return Response.json([{ sha: commits.get(repository) }]); + } + if (url.pathname === `/repos/${repository}/git/trees/${commits.get(repository)}`) { + const tree = + repository === 'alpha/rootless' + ? [ + { + path: 'packages/child/package.json', + mode: '100644', + type: 'blob', + sha: 'nested-manifest', + size: 42, + url: 'https://api.github.com/blob/nested-manifest', + }, + ] + : [ + { + path: 'package.json', + mode: '100644', + type: 'blob', + sha: `root-manifest-${repository}`, + size: 42, + url: `https://api.github.com/blob/root-manifest-${repository}`, + }, + ]; + return Response.json({ sha: `tree-${repository}`, url: url.href, tree, truncated: false }); + } + return new Response('missing route', { status: 404 }); + }) as typeof fetch; +} + +it('replaces a rootless candidate before recording the exact requested sample', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'candidates.txt'); + const outputFile = join(directory, 'repos.txt'); + const evidenceFile = join(directory, 'repository-sample.json'); + writeFileSync( + candidateFile, + 'gamma/eligible\nalpha/rootless\nbeta/eligible\nalpha/rootless\n', + 'utf8', + ); + + await (await resolver())({ + candidateFile, + outputFile, + evidenceFile, + requested: 2, + token: 'read-only-test-token', + fetchImpl: candidateApi(), + }); + + expect(readFileSync(outputFile, 'utf8')).toBe( + `beta/eligible@${FIRST_ELIGIBLE_COMMIT}\ngamma/eligible@${SECOND_ELIGIBLE_COMMIT}\n`, + ); + expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toEqual({ + schemaVersion: 1, + requested: 2, + actual: 2, + candidatesConsidered: 3, + status: 'complete', + exclusions: [ + { + repository: 'alpha/rootless', + commit: ROOTLESS_COMMIT, + reason: 'root-package-json-unavailable', + }, + ], + }); +}); + +it('hard-fails an API error and persists it instead of treating it as ineligibility', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'candidates.txt'); + const outputFile = join(directory, 'repos.txt'); + const evidenceFile = join(directory, 'repository-sample.json'); + writeFileSync(candidateFile, 'alpha/project\n', 'utf8'); + + await expect( + (await resolver())({ + candidateFile, + outputFile, + evidenceFile, + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => new Response('rate limited', { status: 403 })) as typeof fetch, + }), + ).rejects.toThrow('GitHub API 403 for https://api.github.com/repos/alpha/project'); + + expect(readFileSync(outputFile, 'utf8')).toBe(''); + expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toEqual({ + schemaVersion: 1, + requested: 1, + actual: 0, + candidatesConsidered: 1, + status: 'failed', + exclusions: [], + error: 'GitHub API 403 for https://api.github.com/repos/alpha/project', + }); +}); diff --git a/tests/corpus/corpus-scan.test.ts b/tests/corpus/corpus-scan.test.ts index da04e9f..50d0a1f 100644 --- a/tests/corpus/corpus-scan.test.ts +++ b/tests/corpus/corpus-scan.test.ts @@ -59,6 +59,22 @@ describe('bounded workspace manifest selection', () => { expect.arrayContaining(['manifest-limit:2', 'depth-limit:3', 'byte-limit:160']), ); }); + + it('keeps the root manifest inside the tree-entry budget even when GitHub lists it late', () => { + const tree: TreeEntry[] = [ + { path: 'a/readme.md', type: 'blob', mode: '100644', size: 20, sha: 'a' }, + { path: 'b/readme.md', type: 'blob', mode: '100644', size: 20, sha: 'b' }, + { path: 'package.json', type: 'blob', mode: '100644', size: 80, sha: 'root' }, + ]; + + const selected = selectCorpusFiles(tree, { + ...DEFAULT_CORPUS_LIMITS, + maxTreeEntries: 2, + }); + + expect(selected.files.map((entry) => entry.path)).toEqual(['package.json']); + expect(selected.truncations).toEqual(['tree-entry-limit:2']); + }); }); describe('corpus evidence redaction', () => { diff --git a/tests/workflows/ci-policy.test.ts b/tests/workflows/ci-policy.test.ts index 6e5d988..59a439e 100644 --- a/tests/workflows/ci-policy.test.ts +++ b/tests/workflows/ci-policy.test.ts @@ -49,9 +49,9 @@ function workflowNames(): string[] { .sort(); } -function runCorpusSelection(requested: number, candidates: string[]) { +function runCorpusCandidatePool(requested: number, candidates: string[]) { const selectionStep = allSteps(workflow('corpus.yml')).find( - (step) => step.name === 'Select the exact deterministic repository sample', + (step) => step.name === 'Prepare the deterministic repository candidate pool', ); const match = selectionStep?.run?.match( /node --input-type=module <<'NODE'\n(?[\s\S]+?)\nNODE/u, @@ -63,7 +63,6 @@ function runCorpusSelection(requested: number, candidates: string[]) { const directory = mkdtempSync(join(tmpdir(), 'scriptspect-corpus-selection-')); const candidateFile = join(directory, 'candidates.txt'); const selectedFile = join(directory, 'selected.txt'); - const selectionFile = join(directory, 'selection.json'); try { writeFileSync(candidateFile, `${candidates.join('\n')}\n`, 'utf8'); const result = spawnSync( @@ -76,17 +75,13 @@ function runCorpusSelection(requested: number, candidates: string[]) { REPO_COUNT: String(requested), CANDIDATE_FILE: candidateFile, SELECTED_FILE: selectedFile, - SELECTION_FILE: selectionFile, }, }, ); if (result.status !== 0) { throw new Error(`corpus selection failed:\n${result.stderr}`); } - return { - repositories: readFileSync(selectedFile, 'utf8').trimEnd().split('\n'), - selection: JSON.parse(readFileSync(selectionFile, 'utf8')) as unknown, - }; + return readFileSync(selectedFile, 'utf8').trimEnd().split('\n'); } finally { rmSync(directory, { recursive: true, force: true }); } @@ -383,28 +378,26 @@ describe('reproducible CI', () => { }); describe('corpus repository selection', () => { - it('selects exactly one repository and records requested and actual counts', () => { - const result = runCorpusSelection(1, ['zeta/project', 'alpha/project']); + it('retains sorted replacements beyond the requested count', () => { + const repositories = runCorpusCandidatePool(1, ['zeta/project', 'alpha/project']); - expect(result.repositories).toEqual(['alpha/project']); - expect(result.selection).toEqual({ schemaVersion: 1, requested: 1, actual: 1 }); + expect(repositories).toEqual(['alpha/project', 'zeta/project']); }); - it('selects exactly 100 repositories at the supported upper bound', () => { + it('retains every valid candidate when 100 eligible repositories are requested', () => { const candidates = Array.from( { length: 120 }, (_, index) => `owner/project-${String(119 - index).padStart(3, '0')}`, ); - const result = runCorpusSelection(100, candidates); + const repositories = runCorpusCandidatePool(100, candidates); - expect(result.repositories).toHaveLength(100); - expect(result.repositories[0]).toBe('owner/project-000'); - expect(result.repositories[99]).toBe('owner/project-099'); - expect(result.selection).toEqual({ schemaVersion: 1, requested: 100, actual: 100 }); + expect(repositories).toHaveLength(120); + expect(repositories[0]).toBe('owner/project-000'); + expect(repositories[119]).toBe('owner/project-119'); }); - it('deduplicates overlapping search results before applying the exact limit', () => { - const result = runCorpusSelection(3, [ + it('deduplicates overlapping search results without discarding replacements', () => { + const repositories = runCorpusCandidatePool(3, [ 'owner/project-c', 'owner/project-a', 'owner/project-b', @@ -413,7 +406,32 @@ describe('corpus repository selection', () => { 'owner/project-d', ]); - expect(result.repositories).toEqual(['owner/project-a', 'owner/project-b', 'owner/project-c']); - expect(result.selection).toEqual({ schemaVersion: 1, requested: 3, actual: 3 }); + expect(repositories).toEqual([ + 'owner/project-a', + 'owner/project-b', + 'owner/project-c', + 'owner/project-d', + ]); + }); + + it('resolves the exact root-eligible sample before the scanner runs', () => { + const steps = allSteps(workflow('corpus.yml')); + const resolver = steps.find( + (step) => step.name === 'Resolve the exact root-eligible repository sample', + ); + const scannerIndex = steps.findIndex( + (step) => step.name === 'Scan scripts without writing to sampled repositories', + ); + const resolverIndex = steps.indexOf(resolver as Step); + + expect(resolverIndex).toBeGreaterThanOrEqual(0); + expect(resolverIndex).toBeLessThan(scannerIndex); + expect(resolver?.run).toBe( + 'pnpm exec tsx tools/corpus-resolve.ts "$CANDIDATE_FILE" repos.txt repository-sample.json "$REPO_COUNT"', + ); + expect(resolver?.env).toMatchObject({ + GITHUB_TOKEN: `\${{ github.token }}`, + REPO_COUNT: `\${{ inputs.repo-count || '100' }}`, + }); }); }); diff --git a/tools/corpus-lib.ts b/tools/corpus-lib.ts index 586aef1..4f4636f 100644 --- a/tools/corpus-lib.ts +++ b/tools/corpus-lib.ts @@ -96,6 +96,23 @@ function addTruncation(truncations: string[], reason: string): void { if (!truncations.includes(reason)) truncations.push(reason); } +function boundedTreeEntries(tree: readonly TreeEntry[], maxTreeEntries: number): TreeEntry[] { + const rootControlFiles = tree + .filter( + (entry) => + (entry.path === 'package.json' || entry.path === 'pnpm-workspace.yaml') && + isCandidate(entry), + ) + .sort((left, right) => { + if (left.path === 'package.json') return -1; + if (right.path === 'package.json') return 1; + return left.path.localeCompare(right.path); + }); + const rootPaths = new Set(rootControlFiles.map((entry) => entry.path)); + const boundedPrefix = tree.slice(0, maxTreeEntries).filter((entry) => !rootPaths.has(entry.path)); + return [...rootControlFiles, ...boundedPrefix].slice(0, maxTreeEntries); +} + /** Select only bounded manifest inputs; every discarded limit is surfaced. */ export function selectCorpusFiles( tree: readonly TreeEntry[], @@ -105,8 +122,7 @@ export function selectCorpusFiles( if (tree.length > limits.maxTreeEntries) { addTruncation(truncations, `tree-entry-limit:${limits.maxTreeEntries}`); } - const candidates = tree - .slice(0, limits.maxTreeEntries) + const candidates = boundedTreeEntries(tree, limits.maxTreeEntries) .filter(isCandidate) .sort((left, right) => { if (left.path === 'package.json') return -1; diff --git a/tools/corpus-resolve.ts b/tools/corpus-resolve.ts new file mode 100644 index 0000000..3c094bd --- /dev/null +++ b/tools/corpus-resolve.ts @@ -0,0 +1,260 @@ +/** + * Resolve a deterministic corpus sample to immutable commits before scanning. + * + * A repository is eligible only when the exact resolved commit exposes a + * bounded, non-symlink root package.json. API and response-shape errors remain + * hard failures; only a verified missing/ineligible root manifest is replaced. + */ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + DEFAULT_CORPUS_LIMITS, + redactCorpusText, + selectCorpusFiles, + type TreeEntry, +} from './corpus-lib'; + +const GITHUB_API = 'https://api.github.com'; + +interface GitHubRepositoryResponse { + default_branch?: string; +} + +interface GitHubCommitResponse { + sha?: string; +} + +interface GitHubTreeResponse { + tree?: TreeEntry[]; + truncated?: boolean; +} + +interface CorpusSampleExclusion { + repository: string; + commit: string; + reason: 'root-package-json-unavailable'; +} + +export interface CorpusSampleEvidence { + schemaVersion: 1; + requested: number; + actual: number; + candidatesConsidered: number; + status: 'complete' | 'failed'; + exclusions: CorpusSampleExclusion[]; + error?: string; +} + +export interface CorpusResolveOptions { + candidateFile: string; + outputFile: string; + evidenceFile: string; + requested: number; + token: string; + fetchImpl?: typeof fetch; +} + +function headers(token: string): Record { + return { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'scriptspect-corpus-resolve', + }; +} + +async function fetchJson(fetchImpl: typeof fetch, url: string, token: string): Promise { + const response = await fetchImpl(url, { headers: headers(token) }); + if (!response.ok) throw new Error(`GitHub API ${response.status} for ${url}`); + return (await response.json()) as T; +} + +function validRepositoryName(value: string): boolean { + const match = /^([A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99}))$/.exec( + value, + ); + return match !== null && !value.includes('..') && !value.endsWith('.'); +} + +function readCandidates(candidateFile: string): string[] { + const candidates = readFileSync(candidateFile, 'utf8') + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter((line) => line !== '' && !line.startsWith('#')); + for (const repository of candidates) { + if (!validRepositoryName(repository)) { + throw new Error(`invalid repository name: ${repository}`); + } + } + return [...new Set(candidates)].sort(); +} + +function exactCommit(value: unknown, repository: string): string { + if (typeof value !== 'string' || !/^[a-f0-9]{40}$/.test(value)) { + throw new Error(`${repository}: GitHub commit response had no exact commit`); + } + return value; +} + +function writeEvidence( + outputFile: string, + evidenceFile: string, + locators: readonly string[], + evidence: CorpusSampleEvidence, +): void { + mkdirSync(dirname(resolve(outputFile)), { recursive: true }); + mkdirSync(dirname(resolve(evidenceFile)), { recursive: true }); + writeFileSync(outputFile, locators.length === 0 ? '' : `${locators.join('\n')}\n`, { + encoding: 'utf8', + flag: 'wx', + }); + writeFileSync(evidenceFile, `${JSON.stringify(evidence, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + }); +} + +function evidence( + requested: number, + locators: readonly string[], + candidatesConsidered: number, + exclusions: CorpusSampleExclusion[], + error?: string, +): CorpusSampleEvidence { + return { + schemaVersion: 1, + requested, + actual: locators.length, + candidatesConsidered, + status: error === undefined ? 'complete' : 'failed', + exclusions, + ...(error === undefined ? {} : { error }), + }; +} + +/** Select exactly `requested` root-eligible repositories at immutable commits. */ +export async function resolveCorpusSample( + options: CorpusResolveOptions, +): Promise { + if (options.token === '') throw new Error('GITHUB_TOKEN is required (read-only public access)'); + if ( + !Number.isSafeInteger(options.requested) || + options.requested < 1 || + options.requested > 100 + ) { + throw new Error('requested repository count must be an integer from 1 through 100'); + } + + const candidates = readCandidates(options.candidateFile); + if (candidates.length < options.requested) { + throw new Error( + `requested ${options.requested} repositories but only ${candidates.length} unique candidates were returned`, + ); + } + + const fetchImpl = options.fetchImpl ?? fetch; + const locators: string[] = []; + const exclusions: CorpusSampleExclusion[] = []; + let candidatesConsidered = 0; + + try { + for (const repository of candidates) { + candidatesConsidered += 1; + const metadata = await fetchJson( + fetchImpl, + `${GITHUB_API}/repos/${repository}`, + options.token, + ); + if (typeof metadata.default_branch !== 'string' || metadata.default_branch === '') { + throw new Error(`${repository}: GitHub repository response had no default branch`); + } + const commits = await fetchJson( + fetchImpl, + `${GITHUB_API}/repos/${repository}/commits?sha=${encodeURIComponent(metadata.default_branch)}&per_page=1`, + options.token, + ); + const commit = exactCommit(commits[0]?.sha, repository); + const rootTree = await fetchJson( + fetchImpl, + `${GITHUB_API}/repos/${repository}/git/trees/${commit}`, + options.token, + ); + if (!Array.isArray(rootTree.tree)) { + throw new Error(`${repository}@${commit}: GitHub root tree response had no tree`); + } + if (rootTree.truncated === true) { + throw new Error(`${repository}@${commit}: GitHub root tree response was truncated`); + } + + const rootManifest = selectCorpusFiles(rootTree.tree, DEFAULT_CORPUS_LIMITS).files.some( + (entry) => entry.path === 'package.json', + ); + if (!rootManifest) { + exclusions.push({ + repository, + commit, + reason: 'root-package-json-unavailable', + }); + continue; + } + + locators.push(`${repository}@${commit}`); + if (locators.length === options.requested) break; + } + + if (locators.length !== options.requested) { + throw new Error( + `requested ${options.requested} root-eligible repositories but only ${locators.length} were resolved`, + ); + } + } catch (error) { + const message = redactCorpusText(error instanceof Error ? error.message : String(error)); + const failedEvidence = evidence( + options.requested, + locators, + candidatesConsidered, + exclusions, + message, + ); + writeEvidence(options.outputFile, options.evidenceFile, locators, failedEvidence); + throw new Error(message); + } + + const completeEvidence = evidence(options.requested, locators, candidatesConsidered, exclusions); + writeEvidence(options.outputFile, options.evidenceFile, locators, completeEvidence); + return completeEvidence; +} + +async function main(): Promise { + const [candidateFile, outputFile, evidenceFile, requestedText] = process.argv.slice(2); + if ( + candidateFile === undefined || + outputFile === undefined || + evidenceFile === undefined || + requestedText === undefined + ) { + throw new Error( + 'usage: tsx tools/corpus-resolve.ts candidates.txt repos.txt repository-sample.json count', + ); + } + await resolveCorpusSample({ + candidateFile, + outputFile, + evidenceFile, + requested: Number(requestedText), + token: process.env.GITHUB_TOKEN ?? '', + }); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)) +) { + main().catch((error: unknown) => { + console.error( + `scriptspect corpus resolver: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + }); +} From 3f1e03cbff10600d6093cf76ff19f0bf710a03e8 Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 08:32:05 +0800 Subject: [PATCH 2/9] fix(corpus): bound API use and persist sample provenance --- .github/workflows/corpus.yml | 58 +-- docs/evidence/corpus-method.md | 49 ++- tests/corpus/corpus-candidates.test.ts | 193 +++++++++ tests/corpus/corpus-resolve.test.ts | 272 ++++++++++--- tests/corpus/corpus-run.test.ts | 420 ++++++++++++++++++- tests/corpus/corpus-scan.test.ts | 7 + tests/workflows/ci-policy.test.ts | 107 ++--- tools/corpus-candidates.ts | 229 +++++++++++ tools/corpus-lib.ts | 6 + tools/corpus-resolve.ts | 538 +++++++++++++++++++------ tools/corpus-scan.ts | 258 ++++++++++-- tools/github-api.ts | 145 +++++++ 12 files changed, 1905 insertions(+), 377 deletions(-) create mode 100644 tests/corpus/corpus-candidates.test.ts create mode 100644 tools/corpus-candidates.ts create mode 100644 tools/github-api.ts diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index ff94f3a..91d297f 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -27,65 +27,22 @@ jobs: corepack enable corepack prepare pnpm@11.24.0 --activate pnpm install --frozen-lockfile - - name: Collect repository candidates + - name: Collect the ranked repository candidate snapshot env: - GH_TOKEN: ${{ github.token }} - CANDIDATE_FILE: ${{ runner.temp }}/repository-candidates.txt - run: | - set -euo pipefail - gh api 'search/repositories?q=language:typescript+stars:%3E2000&sort=stars&order=desc&per_page=100' \ - --jq '.items[].full_name' > "$CANDIDATE_FILE" - gh api 'search/repositories?q=language:javascript+stars:%3E5000&sort=stars&order=desc&per_page=100' \ - --jq '.items[].full_name' >> "$CANDIDATE_FILE" - - name: Prepare the deterministic repository candidate pool - env: - REPO_COUNT: ${{ inputs.repo-count || '100' }} - CANDIDATE_FILE: ${{ runner.temp }}/repository-candidates.txt - SELECTED_FILE: ${{ runner.temp }}/repository-candidate-pool.txt - run: | - set -euo pipefail - [[ "$REPO_COUNT" =~ ^([1-9]|[1-9][0-9]|100)$ ]] - [[ "$REPO_COUNT" -le 100 ]] - node --input-type=module <<'NODE' - import { readFileSync, writeFileSync } from 'node:fs'; - - const countText = process.env.REPO_COUNT ?? ''; - if (!/^(?:[1-9]|[1-9][0-9]|100)$/u.test(countText)) { - throw new Error('REPO_COUNT must be an integer from 1 through 100'); - } - const requested = Number(countText); - const candidateFile = process.env.CANDIDATE_FILE; - const selectedFile = process.env.SELECTED_FILE; - if (!candidateFile || !selectedFile) { - throw new Error('corpus selection file paths are required'); - } - - const candidates = readFileSync(candidateFile, 'utf8') - .split(/\r?\n/u) - .filter(Boolean); - for (const repository of candidates) { - if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { - throw new Error(`invalid repository name: ${repository}`); - } - } - const uniqueCandidates = [...new Set(candidates)].sort(); - if (uniqueCandidates.length < requested) { - throw new Error( - `requested ${requested} repositories but only ${uniqueCandidates.length} unique candidates were returned`, - ); - } - writeFileSync(selectedFile, `${uniqueCandidates.join('\n')}\n`, 'utf8'); - NODE + GITHUB_TOKEN: ${{ github.token }} + run: pnpm exec tsx tools/corpus-candidates.ts repository-candidates.json - name: Resolve the exact root-eligible repository sample env: GITHUB_TOKEN: ${{ github.token }} REPO_COUNT: ${{ inputs.repo-count || '100' }} - CANDIDATE_FILE: ${{ runner.temp }}/repository-candidate-pool.txt - run: pnpm exec tsx tools/corpus-resolve.ts "$CANDIDATE_FILE" repos.txt repository-sample.json "$REPO_COUNT" + run: pnpm exec tsx tools/corpus-resolve.ts repository-candidates.json repos.txt repository-sample.json "$REPO_COUNT" - name: Scan scripts without writing to sampled repositories env: GITHUB_TOKEN: ${{ github.token }} SCRIPTSPECT_SOURCE_COMMIT: ${{ github.sha }} + CORPUS_SAMPLE_METHOD: popularity-strata-round-robin-v1 + CORPUS_CANDIDATE_SNAPSHOT: repository-candidates.json + CORPUS_SAMPLE_EVIDENCE: repository-sample.json run: pnpm exec tsx tools/corpus-scan.ts repos.txt - name: Create a deterministic 100-finding adjudication draft when available if: always() @@ -103,6 +60,7 @@ jobs: with: name: corpus-draft-${{ github.run_id }} path: | + repository-candidates.json repos.txt repository-sample.json findings.jsonl diff --git a/docs/evidence/corpus-method.md b/docs/evidence/corpus-method.md index 7d6508a..8622384 100644 --- a/docs/evidence/corpus-method.md +++ b/docs/evidence/corpus-method.md @@ -5,22 +5,40 @@ public code. A machine scan produces a **data draft**, not a precision claim. ## Immutable scan -1. The manual/monthly corpus workflow selects public JavaScript and TypeScript - repositories, resolves each default branch once, and records only - `owner/repository@40-character-commit` locators. -2. `tools/corpus-scan.ts` reads those immutable locators through GitHub's tree - and blob APIs. It never clones with credentials, executes scripts, or writes - to a sampled repository. -3. Only the root `package.json`, `pnpm-workspace.yaml`, and candidate workspace +1. The manual/monthly corpus workflow captures the complete first page of two + popularity-ranked GitHub Search strata (JavaScript and TypeScript), including + each query, response hash, rank, star count, and repository. It then uses a + deterministic rank-by-rank round robin and de-duplicates a repository at its + first appearance. The full ordered snapshot is preserved rather than sorting + away its rank or stratum. +2. The resolver batches candidates through GitHub GraphQL. One response anchors + the default-branch commit and root `package.json` blob together. Only an exact + `NOT_FOUND` error at that candidate's root-file field is an eligibility + exclusion; every other partial error or response mismatch fails closed. The + resolver records only `owner/repository@40-character-commit` locators and + hashes the complete ordered candidate snapshot into its evidence. +3. `tools/corpus-scan.ts` makes one bounded recursive-tree REST request per + selected repository. It reads each selected manifest from + `raw.githubusercontent.com` at the exact commit without an Authorization + header, then verifies byte length and the Git blob OID from the immutable tree + before analysis. Manifest downloads therefore do not consume per-blob GitHub + REST core requests. The scanner never clones, executes scripts, or writes to + a sampled repository. +4. Only the root `package.json`, `pnpm-workspace.yaml`, and candidate workspace `package.json` files are materialized in a fresh temporary directory. The normal CLI analyzer then applies the same canonical-root, workspace glob, dependency visibility, and symlink-boundary policy used for local projects. -4. Dependency/VCS/vendor/generated/build/distribution directories and symlink +5. Dependency/VCS/vendor/generated/build/distribution directories and symlink tree entries are excluded. The default ceilings are 20,000 tree entries, 500 manifests, depth 12, 1 MiB per file, and 10 MiB decoded bytes per repository. A GitHub-truncated tree or any local limit marks the repository - `truncated`; API, decoding, or analysis errors mark it `failed`. Neither - status contributes to promoted totals. + `truncated`; API, decoding, immutable-blob verification, or analysis errors + mark it `failed`. Neither status contributes to promoted totals. GitHub HTTP + failures retain their status, rate-limit limit/remaining/reset/used/resource, + Retry-After, request ID, and a rate/auth/permission classification; request + credentials are never persisted. Findings from truncated or failed + repositories are also excluded from `findings.jsonl`, so the adjudication + draft cannot silently sample incomplete repositories. The scan reports root-only and workspace-full counts separately. This prevents root-only PS040 results from being presented as monorepo truth. @@ -29,12 +47,18 @@ root-only PS040 results from being presented as monorepo truth. The workflow artifact contains: +- `repository-candidates.json`: the complete ordered popularity-strata snapshot, + including query metadata, ranks, repositories, response hashes, and status; +- `repository-sample.json`: candidate-snapshot SHA-256, deterministic method, + GraphQL request/cost evidence, rootless replacements, and selected immutable + commits/root-manifest blobs; - `repos.txt`: the exact immutable sample; - `findings.jsonl`: stable finding IDs, immutable source URLs, script SHA-256, rule metadata, spans, and redacted messages—never raw script source; - `corpus-run.json`: selected manifest paths, scanner/source commit and hashes, - rule-registry hash, limits, sample method/seed, environment, per-repository - status, separate scan modes, artifact hashes, and reproduction command; + rule-registry hash, limits, sample method/seed, hashes of the full candidate + snapshot and sample evidence, environment, per-repository status, separate + scan modes, artifact hashes, and reproduction command; - `summary.md`: an explicitly unverified summary for maintainers. The run fails if any repository fails, while still leaving `corpus-run.json` @@ -77,4 +101,3 @@ exist, scriptspect makes no head-to-head superiority claim. - Draft, partial, overdue, or failed runs leave the relevant gate `OPEN`. - No issue, pull request, comment, email, or other third-party write is made from corpus automation. Such contact requires explicit human authorization. - diff --git a/tests/corpus/corpus-candidates.test.ts b/tests/corpus/corpus-candidates.test.ts new file mode 100644 index 0000000..9cecbc2 --- /dev/null +++ b/tests/corpus/corpus-candidates.test.ts @@ -0,0 +1,193 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { afterEach, expect, it } from 'vitest'; + +const temporaryDirectories: string[] = []; + +type Collector = (options: { + outputFile: string; + token: string; + fetchImpl: typeof fetch; +}) => Promise; + +function temporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), 'scriptspect-corpus-candidates-test-')); + temporaryDirectories.push(directory); + return directory; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +async function collector(): Promise { + const moduleUrl = pathToFileURL(join(process.cwd(), 'tools', 'corpus-candidates.ts')).href; + const module = (await import(moduleUrl).catch(() => ({}))) as { + collectCorpusCandidates?: Collector; + }; + expect(typeof module.collectCorpusCandidates).toBe('function'); + if (module.collectCorpusCandidates === undefined) { + throw new Error('corpus candidate collector was unavailable'); + } + return module.collectCorpusCandidates; +} + +function searchApi(): typeof fetch { + return (async (input: string | URL | Request) => { + const url = new URL(String(input)); + const query = url.searchParams.get('q'); + const items = query?.includes('language:typescript') + ? [ + { full_name: 'alpha/shared', stargazers_count: 100 }, + { full_name: 'gamma/typescript', stargazers_count: 90 }, + ] + : [ + { full_name: 'beta/javascript', stargazers_count: 110 }, + { full_name: 'alpha/shared', stargazers_count: 100 }, + ]; + return Response.json( + { total_count: items.length, incomplete_results: false, items }, + { + headers: { + 'x-ratelimit-limit': '30', + 'x-ratelimit-remaining': '28', + 'x-ratelimit-reset': '1788220800', + 'x-ratelimit-used': '2', + 'x-ratelimit-resource': 'search', + 'x-github-request-id': 'SEARCH:TEST', + }, + }, + ); + }) as typeof fetch; +} + +it('persists both ranked strata and a deterministic round-robin candidate universe', async () => { + const directory = temporaryDirectory(); + const outputFile = join(directory, 'repository-candidates.json'); + + await (await collector())({ + outputFile, + token: 'read-only-test-token', + fetchImpl: searchApi(), + }); + + const snapshot = JSON.parse(readFileSync(outputFile, 'utf8')) as Record; + expect(snapshot).toMatchObject({ + schemaVersion: 1, + status: 'complete', + method: 'popularity-strata-round-robin-v1', + strata: [ + { + id: 'typescript', + query: 'language:typescript stars:>2000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + candidates: [ + { rank: 1, repository: 'alpha/shared', stars: 100 }, + { rank: 2, repository: 'gamma/typescript', stars: 90 }, + ], + }, + { + id: 'javascript', + query: 'language:javascript stars:>5000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + candidates: [ + { rank: 1, repository: 'beta/javascript', stars: 110 }, + { rank: 2, repository: 'alpha/shared', stars: 100 }, + ], + }, + ], + orderedCandidates: [ + { position: 1, stratum: 'typescript', rank: 1, repository: 'alpha/shared' }, + { position: 2, stratum: 'javascript', rank: 1, repository: 'beta/javascript' }, + { position: 3, stratum: 'typescript', rank: 2, repository: 'gamma/typescript' }, + ], + }); +}); + +it('persists rate-limit headers without leaking the request token', async () => { + const directory = temporaryDirectory(); + const outputFile = join(directory, 'repository-candidates.json'); + const headers = { + 'x-ratelimit-limit': '30', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': '1788224400', + 'x-ratelimit-used': '30', + 'x-ratelimit-resource': 'search', + 'retry-after': '60', + 'x-github-request-id': 'SEARCH-REQ-1', + }; + + await expect( + (await collector())({ + outputFile, + token: 'github_pat_SHOULD_NOT_LEAK_12345678901234567890', + fetchImpl: (async () => + new Response('{"message":"rate limited"}', { status: 403, headers })) as typeof fetch, + }), + ).rejects.toThrow('GitHub API 403'); + + const text = readFileSync(outputFile, 'utf8'); + expect(text).not.toContain('SHOULD_NOT_LEAK'); + expect(JSON.parse(text)).toMatchObject({ + schemaVersion: 1, + status: 'failed', + failure: { + kind: 'primary-rate-limit-exhausted', + status: 403, + rateLimit: { + limit: '30', + remaining: '0', + reset: '1788224400', + used: '30', + resource: 'search', + }, + retryAfter: '60', + requestId: 'SEARCH-REQ-1', + }, + }); +}); + +it('distinguishes a permission denial from primary rate exhaustion', async () => { + const directory = temporaryDirectory(); + const outputFile = join(directory, 'repository-candidates.json'); + + await expect( + (await collector())({ + outputFile, + token: 'read-only-test-token', + fetchImpl: (async () => + new Response('{"message":"Resource not accessible by integration"}', { + status: 403, + headers: { + 'x-ratelimit-limit': '30', + 'x-ratelimit-remaining': '29', + 'x-ratelimit-reset': '1788224400', + 'x-ratelimit-used': '1', + 'x-ratelimit-resource': 'search', + 'x-github-request-id': 'SEARCH-PERMISSION-1', + }, + })) as typeof fetch, + }), + ).rejects.toThrow('GitHub API 403'); + + expect(JSON.parse(readFileSync(outputFile, 'utf8'))).toMatchObject({ + status: 'failed', + failure: { + kind: 'permission-denied', + status: 403, + rateLimit: { remaining: '29', resource: 'search' }, + retryAfter: null, + requestId: 'SEARCH-PERMISSION-1', + }, + }); +}); diff --git a/tests/corpus/corpus-resolve.test.ts b/tests/corpus/corpus-resolve.test.ts index a13509e..4d3fdfb 100644 --- a/tests/corpus/corpus-resolve.test.ts +++ b/tests/corpus/corpus-resolve.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -7,6 +8,8 @@ import { afterEach, expect, it } from 'vitest'; const ROOTLESS_COMMIT = '1111111111111111111111111111111111111111'; const FIRST_ELIGIBLE_COMMIT = '2222222222222222222222222222222222222222'; const SECOND_ELIGIBLE_COMMIT = '3333333333333333333333333333333333333333'; +const FIRST_BLOB = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +const SECOND_BLOB = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; const temporaryDirectories: string[] = []; type Resolver = (options: { @@ -40,66 +43,125 @@ async function resolver(): Promise { return module.resolveCorpusSample; } -function candidateApi(): typeof fetch { - const commits = new Map([ - ['alpha/rootless', ROOTLESS_COMMIT], - ['beta/eligible', FIRST_ELIGIBLE_COMMIT], - ['gamma/eligible', SECOND_ELIGIBLE_COMMIT], - ]); - - return (async (input: string | URL | Request) => { - const url = new URL(String(input)); - const repository = [...commits.keys()].find( - (candidate) => - url.pathname === `/repos/${candidate}` || url.pathname.startsWith(`/repos/${candidate}/`), - ); - if (repository === undefined) return new Response('missing repository', { status: 404 }); +function candidateSnapshot(): string { + return `${JSON.stringify( + { + schemaVersion: 1, + status: 'complete', + method: 'popularity-strata-round-robin-v1', + strata: [ + { + id: 'typescript', + query: 'language:typescript stars:>2000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: 'a'.repeat(64), + candidates: [ + { rank: 1, repository: 'alpha/rootless', stars: 300 }, + { rank: 2, repository: 'gamma/eligible', stars: 200 }, + ], + }, + { + id: 'javascript', + query: 'language:javascript stars:>5000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: 'b'.repeat(64), + candidates: [ + { rank: 1, repository: 'beta/eligible', stars: 400 }, + { rank: 2, repository: 'alpha/rootless', stars: 300 }, + ], + }, + ], + orderedCandidates: [ + { position: 1, stratum: 'typescript', rank: 1, repository: 'alpha/rootless' }, + { position: 2, stratum: 'javascript', rank: 1, repository: 'beta/eligible' }, + { position: 3, stratum: 'typescript', rank: 2, repository: 'gamma/eligible' }, + ], + }, + null, + 2, + )}\n`; +} - if (url.pathname === `/repos/${repository}`) { - return Response.json({ default_branch: 'main' }); - } - if (url.pathname === `/repos/${repository}/commits`) { - return Response.json([{ sha: commits.get(repository) }]); - } - if (url.pathname === `/repos/${repository}/git/trees/${commits.get(repository)}`) { - const tree = - repository === 'alpha/rootless' - ? [ - { - path: 'packages/child/package.json', - mode: '100644', +function candidateApi(): typeof fetch { + return (async (input: string | URL | Request, init?: RequestInit) => { + expect(String(input)).toBe('https://api.github.com/graphql'); + expect(init?.method).toBe('POST'); + const body = JSON.parse(String(init?.body)) as { query: string }; + expect(body.query).toContain('file(path: "package.json")'); + expect(body.query).toContain('rateLimit'); + return Response.json({ + data: { + r0: { + nameWithOwner: 'alpha/rootless', + defaultBranchRef: { + name: 'main', + target: { __typename: 'Commit', oid: ROOTLESS_COMMIT, file: null }, + }, + }, + r1: { + nameWithOwner: 'beta/eligible', + defaultBranchRef: { + name: 'main', + target: { + __typename: 'Commit', + oid: FIRST_ELIGIBLE_COMMIT, + file: { + name: 'package.json', + mode: 33188, type: 'blob', - sha: 'nested-manifest', - size: 42, - url: 'https://api.github.com/blob/nested-manifest', + oid: FIRST_BLOB, + object: { __typename: 'Blob', oid: FIRST_BLOB, byteSize: 42, isBinary: false }, }, - ] - : [ - { - path: 'package.json', - mode: '100644', + }, + }, + }, + r2: { + nameWithOwner: 'gamma/eligible', + defaultBranchRef: { + name: 'main', + target: { + __typename: 'Commit', + oid: SECOND_ELIGIBLE_COMMIT, + file: { + name: 'package.json', + mode: 33188, type: 'blob', - sha: `root-manifest-${repository}`, - size: 42, - url: `https://api.github.com/blob/root-manifest-${repository}`, + oid: SECOND_BLOB, + object: { __typename: 'Blob', oid: SECOND_BLOB, byteSize: 43, isBinary: false }, }, - ]; - return Response.json({ sha: `tree-${repository}`, url: url.href, tree, truncated: false }); - } - return new Response('missing route', { status: 404 }); + }, + }, + }, + rateLimit: { + cost: 1, + limit: 5000, + remaining: 4999, + used: 1, + resetAt: '2026-09-01T01:00:00Z', + }, + }, + errors: [ + { + type: 'NOT_FOUND', + path: ['r0', 'defaultBranchRef', 'target', 'file'], + message: "Could not resolve file for path 'package.json'.", + }, + ], + }); }) as typeof fetch; } -it('replaces a rootless candidate before recording the exact requested sample', async () => { +it('interleaves popularity strata, replaces rootless candidates, and hashes the full snapshot', async () => { const directory = temporaryDirectory(); - const candidateFile = join(directory, 'candidates.txt'); + const candidateFile = join(directory, 'repository-candidates.json'); const outputFile = join(directory, 'repos.txt'); const evidenceFile = join(directory, 'repository-sample.json'); - writeFileSync( - candidateFile, - 'gamma/eligible\nalpha/rootless\nbeta/eligible\nalpha/rootless\n', - 'utf8', - ); + const snapshotText = candidateSnapshot(); + writeFileSync(candidateFile, snapshotText, 'utf8'); await (await resolver())({ candidateFile, @@ -114,13 +176,50 @@ it('replaces a rootless candidate before recording the exact requested sample', `beta/eligible@${FIRST_ELIGIBLE_COMMIT}\ngamma/eligible@${SECOND_ELIGIBLE_COMMIT}\n`, ); expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toEqual({ - schemaVersion: 1, + schemaVersion: 2, + method: 'popularity-strata-round-robin-v1', + candidateSnapshotSha256: createHash('sha256').update(snapshotText).digest('hex'), requested: 2, actual: 2, candidatesConsidered: 3, status: 'complete', + api: { + transport: 'github-graphql-batch-v1', + batchSize: 20, + requests: 1, + cost: 1, + rateLimit: { + limit: 5000, + remaining: 4999, + used: 1, + resetAt: '2026-09-01T01:00:00Z', + }, + }, + selected: [ + { + position: 2, + stratum: 'javascript', + rank: 1, + repository: 'beta/eligible', + commit: FIRST_ELIGIBLE_COMMIT, + rootManifestOid: FIRST_BLOB, + rootManifestBytes: 42, + }, + { + position: 3, + stratum: 'typescript', + rank: 2, + repository: 'gamma/eligible', + commit: SECOND_ELIGIBLE_COMMIT, + rootManifestOid: SECOND_BLOB, + rootManifestBytes: 43, + }, + ], exclusions: [ { + position: 1, + stratum: 'typescript', + rank: 1, repository: 'alpha/rootless', commit: ROOTLESS_COMMIT, reason: 'root-package-json-unavailable', @@ -129,12 +228,21 @@ it('replaces a rootless candidate before recording the exact requested sample', }); }); -it('hard-fails an API error and persists it instead of treating it as ineligibility', async () => { +it('hard-fails a rate exhaustion and persists non-secret response metadata', async () => { const directory = temporaryDirectory(); - const candidateFile = join(directory, 'candidates.txt'); + const candidateFile = join(directory, 'repository-candidates.json'); const outputFile = join(directory, 'repos.txt'); const evidenceFile = join(directory, 'repository-sample.json'); - writeFileSync(candidateFile, 'alpha/project\n', 'utf8'); + writeFileSync(candidateFile, candidateSnapshot(), 'utf8'); + const headers = { + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': '1788224400', + 'x-ratelimit-used': '5000', + 'x-ratelimit-resource': 'graphql', + 'retry-after': '60', + 'x-github-request-id': 'REQ-123', + }; await expect( (await resolver())({ @@ -142,19 +250,57 @@ it('hard-fails an API error and persists it instead of treating it as ineligibil outputFile, evidenceFile, requested: 1, - token: 'read-only-test-token', - fetchImpl: (async () => new Response('rate limited', { status: 403 })) as typeof fetch, + token: 'read-only-test-token-SHOULD-NOT-LEAK', + fetchImpl: (async () => + new Response('{"message":"rate limited"}', { status: 403, headers })) as typeof fetch, }), - ).rejects.toThrow('GitHub API 403 for https://api.github.com/repos/alpha/project'); + ).rejects.toThrow('GitHub API 403 for https://api.github.com/graphql'); expect(readFileSync(outputFile, 'utf8')).toBe(''); - expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toEqual({ - schemaVersion: 1, - requested: 1, - actual: 0, - candidatesConsidered: 1, + const evidenceText = readFileSync(evidenceFile, 'utf8'); + expect(evidenceText).not.toContain('SHOULD-NOT-LEAK'); + expect(JSON.parse(evidenceText)).toMatchObject({ + schemaVersion: 2, status: 'failed', - exclusions: [], - error: 'GitHub API 403 for https://api.github.com/repos/alpha/project', + failure: { + kind: 'primary-rate-limit-exhausted', + status: 403, + url: 'https://api.github.com/graphql', + rateLimit: { + limit: '5000', + remaining: '0', + reset: '1788224400', + used: '5000', + resource: 'graphql', + }, + retryAfter: '60', + requestId: 'REQ-123', + }, }); }); + +it('rejects a snapshot whose ordered universe does not reproduce its ranked strata', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const snapshot = JSON.parse(candidateSnapshot()) as { + orderedCandidates: Array>; + }; + snapshot.orderedCandidates.reverse(); + writeFileSync(candidateFile, `${JSON.stringify(snapshot)}\n`, 'utf8'); + let called = false; + + await expect( + (await resolver())({ + candidateFile, + outputFile: join(directory, 'repos.txt'), + evidenceFile: join(directory, 'repository-sample.json'), + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => { + called = true; + return new Response('unexpected'); + }) as typeof fetch, + }), + ).rejects.toThrow('candidate snapshot ordering did not match its ranked strata'); + expect(called).toBe(false); +}); diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index f8ed140..d96df9d 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -21,20 +22,49 @@ afterEach(() => { } }); -function fakeGitHub(tree: TreeEntry[], blobs: Record): typeof fetch { - return (async (input: string | URL | Request) => { +interface GitHubObservation { + rawUrls: string[]; + rawAuthorization: Array; + rawRedirect: Array; +} + +type RawResponseFactory = (path: string, bytes: Buffer) => Response; + +function fixtureGitBlobOid(bytes: Buffer): string { + return createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); +} + +function authorizationHeader(input: string | URL | Request, init?: RequestInit): string | null { + const inputHeaders = input instanceof Request ? input.headers : undefined; + return new Headers(init?.headers ?? inputHeaders).get('authorization'); +} + +function fakeGitHub( + tree: TreeEntry[], + blobs: Record, + observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }, + rawResponse: RawResponseFactory = (_path, bytes) => new Response(bytes), +): typeof fetch { + return (async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); if (url.includes('/git/trees/')) { return Response.json({ tree, truncated: false }); } - const sha = url.split('/').at(-1) ?? ''; - const bytes = blobs[sha]; + const parsed = new URL(url); + if (parsed.hostname !== 'raw.githubusercontent.com') { + throw new Error(`unexpected GitHub request: ${url}`); + } + observation.rawUrls.push(url); + observation.rawAuthorization.push(authorizationHeader(input, init)); + observation.rawRedirect.push(init?.redirect); + const [, owner, repository, commit, ...encodedPath] = parsed.pathname.split('/'); + if (owner !== 'example' || repository !== 'project' || commit !== COMMIT) { + return new Response('missing', { status: 404 }); + } + const path = encodedPath.map(decodeURIComponent).join('/'); + const bytes = blobs[path]; if (bytes === undefined) return new Response('missing', { status: 404 }); - return Response.json({ - encoding: 'base64', - content: bytes.toString('base64'), - size: bytes.length, - }); + return rawResponse(path, bytes); }) as typeof fetch; } @@ -54,13 +84,19 @@ function fixture(): { tree: TreeEntry[]; blobs: Record; rawScrip return { rawScript, tree: [ - { path: 'package.json', type: 'blob', mode: '100644', size: root.length, sha: 'root' }, + { + path: 'package.json', + type: 'blob', + mode: '100644', + size: root.length, + sha: fixtureGitBlobOid(root), + }, { path: 'packages/child/package.json', type: 'blob', mode: '100644', size: child.length, - sha: 'child', + sha: fixtureGitBlobOid(child), }, { path: 'node_modules/leak/package.json', @@ -70,7 +106,7 @@ function fixture(): { tree: TreeEntry[]; blobs: Record; rawScrip sha: 'excluded', }, ], - blobs: { root, child }, + blobs: { 'package.json': root, 'packages/child/package.json': child }, }; } @@ -80,6 +116,7 @@ describe('immutable corpus run evidence', () => { const inputFile = join(directory, 'repos.txt'); const outputDir = join(directory, 'out'); const data = fixture(); + const observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }; writeFileSync(inputFile, `example/project@${COMMIT}\n`); const manifest = await runCorpusScan({ @@ -88,7 +125,7 @@ describe('immutable corpus run evidence', () => { token: 'read-only-test-token', sourceCommit: SOURCE_COMMIT, generatedAt: '2026-09-01T00:00:00.000Z', - fetchImpl: fakeGitHub(data.tree, data.blobs), + fetchImpl: fakeGitHub(data.tree, data.blobs, observation), }); expect(manifest.repositories).toMatchObject([ @@ -116,6 +153,12 @@ describe('immutable corpus run evidence', () => { expect(evidence.every((finding) => String(finding.url).includes(COMMIT))).toBe(true); expect(findingsText).not.toContain(data.rawScript); expect(findingsText).not.toContain('CORPUS_PRIVATE_SENTINEL_7f86'); + expect(observation.rawUrls).toEqual([ + `https://raw.githubusercontent.com/example/project/${COMMIT}/package.json`, + `https://raw.githubusercontent.com/example/project/${COMMIT}/packages/child/package.json`, + ]); + expect(observation.rawAuthorization).toEqual([null, null]); + expect(observation.rawRedirect).toEqual(['error', 'error']); const persisted = JSON.parse( readFileSync(join(outputDir, 'corpus-run.json'), 'utf8'), @@ -135,6 +178,7 @@ describe('immutable corpus run evidence', () => { const inputFile = join(directory, 'repos.txt'); const outputDir = join(directory, 'out'); const data = fixture(); + const observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }; writeFileSync(inputFile, `example/project@${COMMIT}\n`); const manifest = await runCorpusScan({ @@ -143,7 +187,7 @@ describe('immutable corpus run evidence', () => { token: 'read-only-test-token', sourceCommit: SOURCE_COMMIT, generatedAt: '2026-09-01T00:00:00.000Z', - fetchImpl: fakeGitHub(data.tree, data.blobs), + fetchImpl: fakeGitHub(data.tree, data.blobs, observation), limits: { ...DEFAULT_CORPUS_LIMITS, maxManifests: 1 }, }); @@ -157,5 +201,353 @@ describe('immutable corpus run evidence', () => { scripts: 0, findings: 0, }); + expect(readFileSync(join(outputDir, 'findings.jsonl'), 'utf8')).toBe(''); + expect(observation.rawUrls).toEqual([]); + }); + + it('fails closed when raw bytes do not match the immutable tree blob OID', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + const tamperedRoot = Buffer.from(data.blobs['package.json'] as Buffer); + const rootNameOffset = tamperedRoot.indexOf('root'); + tamperedRoot.write('soot', rootNameOffset, 'utf8'); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, { + ...data.blobs, + 'package.json': tamperedRoot, + }), + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: 'package.json: raw bytes did not match the immutable tree Git blob OID', + }); + }); + + it('fails closed when raw byte length does not match the immutable tree entry', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + const tree = data.tree.map((entry) => + entry.path === 'package.json' ? { ...entry, size: (entry.size as number) + 1 } : entry, + ); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(tree, data.blobs), + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: 'package.json: raw byte length did not match the immutable tree entry', + }); + }); + + it('rejects a non-SHA-1 tree blob OID before trusting raw content', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + const tree = data.tree.map((entry) => + entry.path === 'package.json' ? { ...entry, sha: 'a'.repeat(64) } : entry, + ); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(tree, data.blobs), + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: 'package.json: immutable tree Git blob OID was not 40 lowercase hex characters', + }); + }); + + it('cancels raw streaming as soon as bytes exceed the immutable tree size', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + let pulls = 0; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub( + data.tree, + data.blobs, + { rawUrls: [], rawAuthorization: [], rawRedirect: [] }, + (path, bytes) => { + if (path !== 'package.json') return new Response(bytes); + const body = new ReadableStream( + { + pull(controller) { + pulls += 1; + if (pulls === 1) controller.enqueue(bytes); + else if (pulls === 2) controller.enqueue(Uint8Array.of(0)); + else controller.error(new Error('unbounded raw read sentinel')); + }, + }, + { highWaterMark: 0 }, + ); + return new Response(body); + }, + ), + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: 'package.json: raw byte length did not match the immutable tree entry', + }); + expect(pulls).toBe(2); + }); + + it('persists rate-limit classification and response headers for tree API failures', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async () => + new Response(JSON.stringify({ message: 'API rate limit exceeded' }), { + status: 403, + headers: { + 'content-type': 'application/json', + 'retry-after': '60', + 'x-github-request-id': 'TEST:RATE:123', + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': '1788213600', + 'x-ratelimit-resource': 'core', + 'x-ratelimit-used': '5000', + }, + })) as typeof fetch, + }), + ).rejects.toThrow('one or more repositories failed'); + + const persistedText = readFileSync(join(outputDir, 'corpus-run.json'), 'utf8'); + const persisted = JSON.parse(persistedText) as { + repositories: Array<{ failure?: unknown }>; + }; + expect(persisted.repositories[0]?.failure).toEqual({ + kind: 'primary-rate-limit-exhausted', + status: 403, + url: `https://api.github.com/repos/example/project/git/trees/${COMMIT}?recursive=1`, + rateLimit: { + limit: '5000', + remaining: '0', + reset: '1788213600', + used: '5000', + resource: 'core', + }, + retryAfter: '60', + requestId: 'TEST:RATE:123', + }); + expect(persistedText).not.toContain('read-only-test-token'); + }); + + it('preserves response headers when tree JSON or shape is invalid', async () => { + for (const body of ['not-json', JSON.stringify({ truncated: false })]) { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async () => + new Response(body, { + status: 200, + headers: { + 'x-github-request-id': 'TEST:INVALID:123', + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': '4999', + 'x-ratelimit-reset': '1788213600', + 'x-ratelimit-resource': 'core', + 'x-ratelimit-used': '1', + }, + })) as typeof fetch, + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ failure?: unknown }>; + }; + expect(persisted.repositories[0]?.failure).toEqual({ + kind: 'response-invalid', + status: 200, + url: `https://api.github.com/repos/example/project/git/trees/${COMMIT}?recursive=1`, + rateLimit: { + limit: '5000', + remaining: '4999', + reset: '1788213600', + used: '1', + resource: 'core', + }, + retryAfter: null, + requestId: 'TEST:INVALID:123', + }); + } + }); + + it('hashes the complete candidate snapshot and sample evidence into the run manifest', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const candidateSnapshot = Buffer.from('{"complete":true}\r\n', 'utf8'); + const candidateSnapshotSha256 = createHash('sha256').update(candidateSnapshot).digest('hex'); + const sampleEvidence = Buffer.from( + `${JSON.stringify({ + status: 'complete', + method: 'popularity-strata-round-robin-v1', + candidateSnapshotSha256, + selected: [{ repository: 'example/project', commit: COMMIT }], + })}\n`, + 'utf8', + ); + const data = fixture(); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, candidateSnapshot); + writeFileSync(sampleEvidenceFile, sampleEvidence); + + const manifest = await runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, data.blobs), + sampleMethod: 'popularity-strata-round-robin-v1', + candidateSnapshotFile, + sampleEvidenceFile, + }); + + const sampleEvidenceSha256 = createHash('sha256').update(sampleEvidence).digest('hex'); + expect(manifest.sampling).toEqual({ + method: 'popularity-strata-round-robin-v1', + seed: 'none', + candidateSnapshotSha256, + sampleEvidenceSha256, + }); + expect(manifest.artifactSha256).toMatchObject({ + 'repository-candidates.json': candidateSnapshotSha256, + 'repository-sample.json': sampleEvidenceSha256, + }); + }); + + it('rejects mismatched sample provenance before making a network request', async () => { + const candidateSnapshot = Buffer.from('{"complete":true}\n', 'utf8'); + const candidateSnapshotSha256 = createHash('sha256').update(candidateSnapshot).digest('hex'); + const validEvidence = { + status: 'complete', + method: 'popularity-strata-round-robin-v1', + candidateSnapshotSha256, + selected: [{ repository: 'example/project', commit: COMMIT }], + }; + const cases = [ + { name: 'status', evidence: { ...validEvidence, status: 'failed' } }, + { name: 'method', evidence: { ...validEvidence, method: 'wrong-method' } }, + { + name: 'candidate snapshot digest', + evidence: { ...validEvidence, candidateSnapshotSha256: '0'.repeat(64) }, + }, + { + name: 'selected locator sequence', + evidence: { + ...validEvidence, + selected: [{ repository: 'another/project', commit: COMMIT }], + }, + }, + ]; + + for (const testCase of cases) { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + let networkCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(testCase.evidence)}\n`, 'utf8'); + const upstream = fakeGitHub(data.tree, data.blobs); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async (input: string | URL | Request, init?: RequestInit) => { + networkCalled = true; + return upstream(input, init); + }) as typeof fetch, + sampleMethod: 'popularity-strata-round-robin-v1', + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/corpus sample evidence/); + expect(networkCalled, testCase.name).toBe(false); + } }); }); diff --git a/tests/corpus/corpus-scan.test.ts b/tests/corpus/corpus-scan.test.ts index 50d0a1f..2192152 100644 --- a/tests/corpus/corpus-scan.test.ts +++ b/tests/corpus/corpus-scan.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { DEFAULT_CORPUS_LIMITS, + gitBlobOid, parseRepoLocator, redactCorpusText, selectCorpusFiles, @@ -9,6 +10,12 @@ import { const SHA = '0123456789abcdef0123456789abcdef01234567'; +describe('Git blob integrity', () => { + it('derives the canonical SHA-1 object ID from the exact blob bytes', () => { + expect(gitBlobOid(Buffer.from('hello\n'))).toBe('ce013625030ba8dba906f756967f9e9ca394464a'); + }); +}); + describe('immutable corpus locators', () => { it('accepts only owner/repo plus an exact 40-character commit', () => { expect(parseRepoLocator(`open-source/project@${SHA}`)).toEqual({ diff --git a/tests/workflows/ci-policy.test.ts b/tests/workflows/ci-policy.test.ts index 59a439e..6d841c9 100644 --- a/tests/workflows/ci-policy.test.ts +++ b/tests/workflows/ci-policy.test.ts @@ -1,6 +1,5 @@ -import { execFileSync, spawnSync } from 'node:child_process'; -import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { execFileSync } from 'node:child_process'; +import { readdirSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; @@ -49,44 +48,6 @@ function workflowNames(): string[] { .sort(); } -function runCorpusCandidatePool(requested: number, candidates: string[]) { - const selectionStep = allSteps(workflow('corpus.yml')).find( - (step) => step.name === 'Prepare the deterministic repository candidate pool', - ); - const match = selectionStep?.run?.match( - /node --input-type=module <<'NODE'\n(?[\s\S]+?)\nNODE/u, - ); - if (!match?.groups?.program) { - throw new Error('corpus selection step must expose an executable inline Node program'); - } - - const directory = mkdtempSync(join(tmpdir(), 'scriptspect-corpus-selection-')); - const candidateFile = join(directory, 'candidates.txt'); - const selectedFile = join(directory, 'selected.txt'); - try { - writeFileSync(candidateFile, `${candidates.join('\n')}\n`, 'utf8'); - const result = spawnSync( - process.execPath, - ['--input-type=module', '--eval', match.groups.program], - { - encoding: 'utf8', - env: { - ...process.env, - REPO_COUNT: String(requested), - CANDIDATE_FILE: candidateFile, - SELECTED_FILE: selectedFile, - }, - }, - ); - if (result.status !== 0) { - throw new Error(`corpus selection failed:\n${result.stderr}`); - } - return readFileSync(selectedFile, 'utf8').trimEnd().split('\n'); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -} - describe('pull-request trust boundary', () => { it('keeps contributor-controlled workflows read-only and free of secrets or pushes', () => { const pullRequestWorkflows = workflowNames().filter((name) => @@ -331,14 +292,15 @@ describe('reproducible CI', () => { ); }); - it('bounds every hosted job and caps GitHub Search pages at 100', () => { + it('bounds every hosted job and delegates the 100-repository cap to the typed resolver', () => { for (const name of workflowNames()) { for (const [jobName, job] of Object.entries(workflow(name).jobs ?? {})) { expect(job['timeout-minutes'], `${name}:${jobName}`).toBeGreaterThan(0); } } const corpus = workflowSource('corpus.yml'); - expect(corpus).toContain('REPO_COUNT" -le 100'); + expect(corpus).toContain("default: '100'"); + expect(corpus).toContain('tools/corpus-resolve.ts'); expect(corpus).not.toContain('{0,2}'); }); @@ -378,40 +340,22 @@ describe('reproducible CI', () => { }); describe('corpus repository selection', () => { - it('retains sorted replacements beyond the requested count', () => { - const repositories = runCorpusCandidatePool(1, ['zeta/project', 'alpha/project']); - - expect(repositories).toEqual(['alpha/project', 'zeta/project']); - }); - - it('retains every valid candidate when 100 eligible repositories are requested', () => { - const candidates = Array.from( - { length: 120 }, - (_, index) => `owner/project-${String(119 - index).padStart(3, '0')}`, + it('captures the complete ranked candidate universe before resolving the sample', () => { + const steps = allSteps(workflow('corpus.yml')); + const collector = steps.find( + (step) => step.name === 'Collect the ranked repository candidate snapshot', + ); + const resolver = steps.find( + (step) => step.name === 'Resolve the exact root-eligible repository sample', ); - const repositories = runCorpusCandidatePool(100, candidates); - - expect(repositories).toHaveLength(120); - expect(repositories[0]).toBe('owner/project-000'); - expect(repositories[119]).toBe('owner/project-119'); - }); - it('deduplicates overlapping search results without discarding replacements', () => { - const repositories = runCorpusCandidatePool(3, [ - 'owner/project-c', - 'owner/project-a', - 'owner/project-b', - 'owner/project-a', - 'owner/project-c', - 'owner/project-d', - ]); - - expect(repositories).toEqual([ - 'owner/project-a', - 'owner/project-b', - 'owner/project-c', - 'owner/project-d', - ]); + expect(collector?.run).toBe( + 'pnpm exec tsx tools/corpus-candidates.ts repository-candidates.json', + ); + expect(collector?.env).toEqual({ GITHUB_TOKEN: `\${{ github.token }}` }); + expect(steps.indexOf(collector as Step)).toBeLessThan(steps.indexOf(resolver as Step)); + expect(workflowSource('corpus.yml')).not.toContain('uniqueCandidates.sort'); + expect(workflowSource('corpus.yml')).not.toContain('gh api'); }); it('resolves the exact root-eligible sample before the scanner runs', () => { @@ -427,11 +371,22 @@ describe('corpus repository selection', () => { expect(resolverIndex).toBeGreaterThanOrEqual(0); expect(resolverIndex).toBeLessThan(scannerIndex); expect(resolver?.run).toBe( - 'pnpm exec tsx tools/corpus-resolve.ts "$CANDIDATE_FILE" repos.txt repository-sample.json "$REPO_COUNT"', + 'pnpm exec tsx tools/corpus-resolve.ts repository-candidates.json repos.txt repository-sample.json "$REPO_COUNT"', ); expect(resolver?.env).toMatchObject({ GITHUB_TOKEN: `\${{ github.token }}`, REPO_COUNT: `\${{ inputs.repo-count || '100' }}`, }); + + const scanner = steps[scannerIndex]; + expect(scanner?.env).toMatchObject({ + CORPUS_SAMPLE_METHOD: 'popularity-strata-round-robin-v1', + CORPUS_CANDIDATE_SNAPSHOT: 'repository-candidates.json', + CORPUS_SAMPLE_EVIDENCE: 'repository-sample.json', + }); + const upload = steps.find( + (step) => step.name === 'Upload draft evidence for maintainer review', + ); + expect(String(upload?.with?.path)).toContain('repository-candidates.json'); }); }); diff --git a/tools/corpus-candidates.ts b/tools/corpus-candidates.ts new file mode 100644 index 0000000..b62dbb5 --- /dev/null +++ b/tools/corpus-candidates.ts @@ -0,0 +1,229 @@ +/** Collect and persist the complete ranked candidate universe for corpus selection. */ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { redactCorpusText, sha256 } from './corpus-lib'; +import { + type GitHubFailureEvidence, + githubApiResponse, + githubFailureEvidence, + invalidGitHubResponse, +} from './github-api'; + +const GITHUB_API = 'https://api.github.com'; +export const CORPUS_SAMPLE_METHOD = 'popularity-strata-round-robin-v1' as const; + +interface SearchItem { + full_name?: string; + stargazers_count?: number; +} + +interface SearchResponse { + total_count?: number; + incomplete_results?: boolean; + items?: SearchItem[]; +} + +export interface RankedCandidate { + rank: number; + repository: string; + stars: number; +} + +export interface CandidateStratum { + id: 'typescript' | 'javascript'; + query: string; + sort: 'stars'; + order: 'desc'; + perPage: 100; + responseSha256: string; + candidates: RankedCandidate[]; +} + +export interface OrderedCandidate { + position: number; + stratum: CandidateStratum['id']; + rank: number; + repository: string; +} + +export interface CorpusCandidateSnapshot { + schemaVersion: 1; + status: 'complete' | 'failed'; + method: typeof CORPUS_SAMPLE_METHOD; + strata: CandidateStratum[]; + orderedCandidates: OrderedCandidate[]; + error?: string; + failure?: GitHubFailureEvidence; +} + +export interface CollectCorpusCandidatesOptions { + outputFile: string; + token: string; + fetchImpl?: typeof fetch; +} + +export const CORPUS_CANDIDATE_STRATA: ReadonlyArray< + Pick +> = [ + { + id: 'typescript', + query: 'language:typescript stars:>2000', + sort: 'stars', + order: 'desc', + perPage: 100, + }, + { + id: 'javascript', + query: 'language:javascript stars:>5000', + sort: 'stars', + order: 'desc', + perPage: 100, + }, +]; + +function validRepositoryName(value: string): boolean { + return ( + /^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99})$/.test(value) && + !value.includes('..') && + !value.endsWith('.') + ); +} + +export function interleaveCandidateStrata(strata: readonly CandidateStratum[]): OrderedCandidate[] { + const seen = new Set(); + const ordered: OrderedCandidate[] = []; + const maxRank = Math.max(0, ...strata.map((stratum) => stratum.candidates.length)); + for (let offset = 0; offset < maxRank; offset += 1) { + for (const stratum of strata) { + const candidate = stratum.candidates[offset]; + if (candidate === undefined || seen.has(candidate.repository)) continue; + seen.add(candidate.repository); + ordered.push({ + position: ordered.length + 1, + stratum: stratum.id, + rank: candidate.rank, + repository: candidate.repository, + }); + } + } + return ordered; +} + +function writeSnapshot(outputFile: string, snapshot: CorpusCandidateSnapshot): void { + mkdirSync(dirname(resolve(outputFile)), { recursive: true }); + writeFileSync(outputFile, `${JSON.stringify(snapshot, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + }); +} + +function searchUrl(stratum: (typeof CORPUS_CANDIDATE_STRATA)[number]): string { + const url = new URL(`${GITHUB_API}/search/repositories`); + url.searchParams.set('q', stratum.query); + url.searchParams.set('sort', stratum.sort); + url.searchParams.set('order', stratum.order); + url.searchParams.set('per_page', String(stratum.perPage)); + return url.href; +} + +export async function collectCorpusCandidates( + options: CollectCorpusCandidatesOptions, +): Promise { + if (options.token === '') throw new Error('GITHUB_TOKEN is required (read-only public access)'); + const fetchImpl = options.fetchImpl ?? fetch; + const strata: CandidateStratum[] = []; + try { + for (const definition of CORPUS_CANDIDATE_STRATA) { + const url = searchUrl(definition); + const response = await githubApiResponse( + fetchImpl, + url, + options.token, + 'scriptspect-corpus-candidates', + ); + const responseText = await response.text(); + let parsed: SearchResponse; + try { + parsed = JSON.parse(responseText) as SearchResponse; + } catch { + throw invalidGitHubResponse( + url, + `${definition.id}: GitHub search response was not JSON`, + response, + ); + } + if (parsed.incomplete_results !== false || !Array.isArray(parsed.items)) { + throw invalidGitHubResponse( + url, + `${definition.id}: GitHub search response was incomplete or invalid`, + response, + ); + } + const candidates = parsed.items.map((item, index): RankedCandidate => { + if ( + typeof item.full_name !== 'string' || + !validRepositoryName(item.full_name) || + typeof item.stargazers_count !== 'number' || + !Number.isSafeInteger(item.stargazers_count) || + item.stargazers_count < 0 + ) { + throw invalidGitHubResponse( + url, + `${definition.id}: GitHub search candidate was invalid`, + response, + ); + } + return { rank: index + 1, repository: item.full_name, stars: item.stargazers_count }; + }); + strata.push({ ...definition, responseSha256: sha256(responseText), candidates }); + } + const snapshot: CorpusCandidateSnapshot = { + schemaVersion: 1, + status: 'complete', + method: CORPUS_SAMPLE_METHOD, + strata, + orderedCandidates: interleaveCandidateStrata(strata), + }; + writeSnapshot(options.outputFile, snapshot); + return snapshot; + } catch (error) { + const message = redactCorpusText(error instanceof Error ? error.message : String(error)); + const snapshot: CorpusCandidateSnapshot = { + schemaVersion: 1, + status: 'failed', + method: CORPUS_SAMPLE_METHOD, + strata, + orderedCandidates: interleaveCandidateStrata(strata), + error: message, + ...(githubFailureEvidence(error) === undefined + ? {} + : { failure: githubFailureEvidence(error) }), + }; + writeSnapshot(options.outputFile, snapshot); + throw new Error(message); + } +} + +async function main(): Promise { + const outputFile = process.argv[2]; + if (outputFile === undefined) { + throw new Error('usage: tsx tools/corpus-candidates.ts repository-candidates.json'); + } + await collectCorpusCandidates({ + outputFile, + token: process.env.GITHUB_TOKEN ?? '', + }); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)) +) { + main().catch((error: unknown) => { + console.error( + `scriptspect corpus candidates: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + }); +} diff --git a/tools/corpus-lib.ts b/tools/corpus-lib.ts index 4f4636f..0d44c32 100644 --- a/tools/corpus-lib.ts +++ b/tools/corpus-lib.ts @@ -63,6 +63,12 @@ export function sha256(value: string | Buffer): string { return createHash('sha256').update(value).digest('hex'); } +/** Derive the canonical Git object ID for exact blob bytes. */ +export function gitBlobOid(value: string | Buffer): string { + const bytes = Buffer.isBuffer(value) ? value : Buffer.from(value); + return createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); +} + export function parseRepoLocator(value: string): RepoLocator { const match = /^([A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99}))@([a-f0-9]{40})$/.exec( diff --git a/tools/corpus-resolve.ts b/tools/corpus-resolve.ts index 3c094bd..c4f8029 100644 --- a/tools/corpus-resolve.ts +++ b/tools/corpus-resolve.ts @@ -1,49 +1,111 @@ -/** - * Resolve a deterministic corpus sample to immutable commits before scanning. - * - * A repository is eligible only when the exact resolved commit exposes a - * bounded, non-symlink root package.json. API and response-shape errors remain - * hard failures; only a verified missing/ineligible root manifest is replaced. - */ +/** Resolve the ranked corpus candidate snapshot to immutable, root-eligible commits. */ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { - DEFAULT_CORPUS_LIMITS, - redactCorpusText, - selectCorpusFiles, - type TreeEntry, -} from './corpus-lib'; + type CandidateStratum, + CORPUS_CANDIDATE_STRATA, + CORPUS_SAMPLE_METHOD, + type CorpusCandidateSnapshot, + interleaveCandidateStrata, + type OrderedCandidate, +} from './corpus-candidates'; +import { DEFAULT_CORPUS_LIMITS, redactCorpusText, sha256 } from './corpus-lib'; +import { + type GitHubFailureEvidence, + githubApiResponse, + githubFailureEvidence, + invalidGitHubResponse, +} from './github-api'; + +const GRAPHQL_URL = 'https://api.github.com/graphql'; +const GRAPHQL_BATCH_SIZE = 20; + +interface GraphQlError { + type?: unknown; + path?: unknown; + message?: unknown; +} + +interface GraphQlRateLimit { + cost?: unknown; + limit?: unknown; + remaining?: unknown; + used?: unknown; + resetAt?: unknown; +} -const GITHUB_API = 'https://api.github.com'; +interface GraphQlTreeEntry { + name?: unknown; + mode?: unknown; + type?: unknown; + oid?: unknown; + object?: { + __typename?: unknown; + oid?: unknown; + byteSize?: unknown; + isBinary?: unknown; + } | null; +} + +interface GraphQlRepository { + nameWithOwner?: unknown; + defaultBranchRef?: { + name?: unknown; + target?: { + __typename?: unknown; + oid?: unknown; + file?: GraphQlTreeEntry | null; + } | null; + } | null; +} -interface GitHubRepositoryResponse { - default_branch?: string; +interface GraphQlResponse { + data?: Record & { rateLimit?: GraphQlRateLimit }; + errors?: GraphQlError[]; +} + +interface ApiEvidence { + transport: 'github-graphql-batch-v1'; + batchSize: 20; + requests: number; + cost: number; + rateLimit?: { + limit: number; + remaining: number; + used: number; + resetAt: string; + }; } -interface GitHubCommitResponse { - sha?: string; +interface ParsedRateLimit extends NonNullable { + cost: number; } -interface GitHubTreeResponse { - tree?: TreeEntry[]; - truncated?: boolean; +interface SelectedCandidate extends OrderedCandidate { + commit: string; + rootManifestOid: string; + rootManifestBytes: number; } -interface CorpusSampleExclusion { - repository: string; +interface CorpusSampleExclusion extends OrderedCandidate { commit: string; reason: 'root-package-json-unavailable'; } export interface CorpusSampleEvidence { - schemaVersion: 1; + schemaVersion: 2; + method: typeof CORPUS_SAMPLE_METHOD; + candidateSnapshotSha256: string; requested: number; actual: number; candidatesConsidered: number; status: 'complete' | 'failed'; + api: ApiEvidence; + selected: SelectedCandidate[]; exclusions: CorpusSampleExclusion[]; error?: string; + failure?: GitHubFailureEvidence; } export interface CorpusResolveOptions { @@ -55,46 +117,231 @@ export interface CorpusResolveOptions { fetchImpl?: typeof fetch; } -function headers(token: string): Record { +function validRepositoryName(value: string): boolean { + const match = /^([A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99}))$/.exec( + value, + ); + return match !== null && !value.includes('..') && !value.endsWith('.'); +} + +function exactOid(value: unknown, description: string): string { + if (typeof value !== 'string' || !/^[a-f0-9]{40}$/.test(value)) { + throw invalidGitHubResponse(GRAPHQL_URL, `${description} was not an exact 40-character oid`); + } + return value; +} + +function safeInteger(value: unknown, description: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw invalidGitHubResponse(GRAPHQL_URL, `${description} was invalid`); + } + return value; +} + +function validateStratum(value: unknown, index: number): CandidateStratum { + if (typeof value !== 'object' || value === null) throw new Error('candidate stratum was invalid'); + const stratum = value as Partial; + const expected = CORPUS_CANDIDATE_STRATA[index]; + if ( + expected === undefined || + stratum.id !== expected.id || + stratum.query !== expected.query || + stratum.sort !== expected.sort || + stratum.order !== expected.order || + stratum.perPage !== expected.perPage || + typeof stratum.responseSha256 !== 'string' || + !/^[a-f0-9]{64}$/.test(stratum.responseSha256) || + !Array.isArray(stratum.candidates) + ) { + throw new Error('candidate stratum metadata was invalid'); + } + const candidates = stratum.candidates.map((candidate, index) => { + if ( + typeof candidate !== 'object' || + candidate === null || + candidate.rank !== index + 1 || + typeof candidate.repository !== 'string' || + !validRepositoryName(candidate.repository) || + typeof candidate.stars !== 'number' || + !Number.isSafeInteger(candidate.stars) || + candidate.stars < 0 + ) { + throw new Error(`${stratum.id}: ranked candidate ${index + 1} was invalid`); + } + return candidate; + }); + return { ...stratum, candidates } as CandidateStratum; +} + +function readCandidateSnapshot(candidateFile: string): { + snapshot: CorpusCandidateSnapshot; + digest: string; +} { + const bytes = readFileSync(candidateFile); + let parsed: unknown; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + throw new Error('candidate snapshot was not valid JSON'); + } + if (typeof parsed !== 'object' || parsed === null) + throw new Error('candidate snapshot was invalid'); + const candidate = parsed as Partial; + if ( + candidate.schemaVersion !== 1 || + candidate.status !== 'complete' || + candidate.method !== CORPUS_SAMPLE_METHOD || + !Array.isArray(candidate.strata) || + !Array.isArray(candidate.orderedCandidates) + ) { + throw new Error('candidate snapshot was incomplete or incompatible'); + } + if (candidate.strata.length !== CORPUS_CANDIDATE_STRATA.length) { + throw new Error('candidate snapshot did not contain the required popularity strata'); + } + const strata = candidate.strata.map(validateStratum); + if (new Set(strata.map((stratum) => stratum.id)).size !== strata.length) { + throw new Error('candidate snapshot contained duplicate strata'); + } + const expected = interleaveCandidateStrata(strata); + if (JSON.stringify(candidate.orderedCandidates) !== JSON.stringify(expected)) { + throw new Error('candidate snapshot ordering did not match its ranked strata'); + } return { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': '2022-11-28', - 'User-Agent': 'scriptspect-corpus-resolve', + snapshot: { ...candidate, strata, orderedCandidates: expected } as CorpusCandidateSnapshot, + digest: sha256(bytes), }; } -async function fetchJson(fetchImpl: typeof fetch, url: string, token: string): Promise { - const response = await fetchImpl(url, { headers: headers(token) }); - if (!response.ok) throw new Error(`GitHub API ${response.status} for ${url}`); - return (await response.json()) as T; +function graphQlQuery(candidates: readonly OrderedCandidate[]): string { + const repositories = candidates.map((candidate, index) => { + const [owner, name] = candidate.repository.split('/'); + return `r${index}: repository(owner: ${JSON.stringify(owner)}, name: ${JSON.stringify(name)}) { + nameWithOwner + defaultBranchRef { + name + target { + __typename + ... on Commit { + oid + file(path: "package.json") { + name + mode + type + oid + object { + __typename + ... on Blob { oid byteSize isBinary } + } + } + } + } + } + }`; + }); + return `query CorpusEligibility {\n${repositories.join('\n')}\nrateLimit { cost limit remaining used resetAt }\n}`; } -function validRepositoryName(value: string): boolean { - const match = /^([A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99}))$/.exec( - value, +function expectedMissingRoot(error: GraphQlError, alias: string): boolean { + return ( + error.type === 'NOT_FOUND' && + Array.isArray(error.path) && + JSON.stringify(error.path) === JSON.stringify([alias, 'defaultBranchRef', 'target', 'file']) ); - return match !== null && !value.includes('..') && !value.endsWith('.'); } -function readCandidates(candidateFile: string): string[] { - const candidates = readFileSync(candidateFile, 'utf8') - .split(/\r?\n/u) - .map((line) => line.trim()) - .filter((line) => line !== '' && !line.startsWith('#')); - for (const repository of candidates) { - if (!validRepositoryName(repository)) { - throw new Error(`invalid repository name: ${repository}`); - } +function graphQlRateLimit(value: unknown): ParsedRateLimit { + if (typeof value !== 'object' || value === null) { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response had no rateLimit'); } - return [...new Set(candidates)].sort(); + const rate = value as GraphQlRateLimit; + return { + cost: safeInteger(rate.cost, 'GitHub GraphQL rateLimit.cost'), + limit: safeInteger(rate.limit, 'GitHub GraphQL rateLimit.limit'), + remaining: safeInteger(rate.remaining, 'GitHub GraphQL rateLimit.remaining'), + used: safeInteger(rate.used, 'GitHub GraphQL rateLimit.used'), + resetAt: + typeof rate.resetAt === 'string' && !Number.isNaN(Date.parse(rate.resetAt)) + ? rate.resetAt + : (() => { + throw invalidGitHubResponse( + GRAPHQL_URL, + 'GitHub GraphQL rateLimit.resetAt was invalid', + ); + })(), + }; } -function exactCommit(value: unknown, repository: string): string { - if (typeof value !== 'string' || !/^[a-f0-9]{40}$/.test(value)) { - throw new Error(`${repository}: GitHub commit response had no exact commit`); +function resolveRepository( + candidate: OrderedCandidate, + alias: string, + repository: unknown, + errors: readonly GraphQlError[], +): SelectedCandidate | CorpusSampleExclusion { + if (typeof repository !== 'object' || repository === null) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}: GitHub GraphQL repository was unavailable`, + ); } - return value; + const result = repository as GraphQlRepository; + if (result.nameWithOwner !== candidate.repository) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}: canonical repository name did not match`, + ); + } + const target = result.defaultBranchRef?.target; + if (target?.__typename !== 'Commit') { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}: default branch did not resolve to a commit`, + ); + } + const commit = exactOid(target.oid, `${candidate.repository}: default branch commit`); + const aliasErrors = errors.filter( + (error) => Array.isArray(error.path) && error.path[0] === alias, + ); + if (target.file === null || target.file === undefined) { + if (aliasErrors.length !== 1 || !expectedMissingRoot(aliasErrors[0] as GraphQlError, alias)) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}@${commit}: root package.json was unresolved without the expected NOT_FOUND evidence`, + ); + } + return { ...candidate, commit, reason: 'root-package-json-unavailable' }; + } + if (aliasErrors.length !== 0) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}@${commit}: GitHub GraphQL returned a partial error`, + ); + } + const file = target.file; + const oid = exactOid(file.oid, `${candidate.repository}@${commit}: root manifest tree entry`); + const objectOid = exactOid( + file.object?.oid, + `${candidate.repository}@${commit}: root manifest blob`, + ); + const bytes = safeInteger( + file.object?.byteSize, + `${candidate.repository}@${commit}: root manifest byte size`, + ); + if ( + file.name !== 'package.json' || + file.type !== 'blob' || + (file.mode !== 33188 && file.mode !== 33261) || + file.object?.__typename !== 'Blob' || + file.object.isBinary !== false || + oid !== objectOid || + bytes > DEFAULT_CORPUS_LIMITS.maxFileBytes + ) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}@${commit}: root package.json did not satisfy immutable blob invariants`, + ); + } + return { ...candidate, commit, rootManifestOid: oid, rootManifestBytes: bytes }; } function writeEvidence( @@ -115,25 +362,7 @@ function writeEvidence( }); } -function evidence( - requested: number, - locators: readonly string[], - candidatesConsidered: number, - exclusions: CorpusSampleExclusion[], - error?: string, -): CorpusSampleEvidence { - return { - schemaVersion: 1, - requested, - actual: locators.length, - candidatesConsidered, - status: error === undefined ? 'complete' : 'failed', - exclusions, - ...(error === undefined ? {} : { error }), - }; -} - -/** Select exactly `requested` root-eligible repositories at immutable commits. */ +/** Select exactly `requested` immutable repositories from the audited ranked snapshot. */ export async function resolveCorpusSample( options: CorpusResolveOptions, ): Promise { @@ -145,85 +374,154 @@ export async function resolveCorpusSample( ) { throw new Error('requested repository count must be an integer from 1 through 100'); } - - const candidates = readCandidates(options.candidateFile); - if (candidates.length < options.requested) { + const { snapshot, digest } = readCandidateSnapshot(options.candidateFile); + if (snapshot.orderedCandidates.length < options.requested) { throw new Error( - `requested ${options.requested} repositories but only ${candidates.length} unique candidates were returned`, + `requested ${options.requested} repositories but only ${snapshot.orderedCandidates.length} ordered candidates were captured`, ); } const fetchImpl = options.fetchImpl ?? fetch; - const locators: string[] = []; + const selected: SelectedCandidate[] = []; const exclusions: CorpusSampleExclusion[] = []; + const api: ApiEvidence = { + transport: 'github-graphql-batch-v1', + batchSize: GRAPHQL_BATCH_SIZE, + requests: 0, + cost: 0, + }; let candidatesConsidered = 0; + let lastApiResponse: Response | undefined; try { - for (const repository of candidates) { - candidatesConsidered += 1; - const metadata = await fetchJson( + for (let offset = 0; offset < snapshot.orderedCandidates.length; offset += GRAPHQL_BATCH_SIZE) { + const batch = snapshot.orderedCandidates.slice(offset, offset + GRAPHQL_BATCH_SIZE); + const response = await githubApiResponse( fetchImpl, - `${GITHUB_API}/repos/${repository}`, + GRAPHQL_URL, options.token, + 'scriptspect-corpus-resolve', + { method: 'POST', body: JSON.stringify({ query: graphQlQuery(batch) }) }, ); - if (typeof metadata.default_branch !== 'string' || metadata.default_branch === '') { - throw new Error(`${repository}: GitHub repository response had no default branch`); + lastApiResponse = response; + api.requests += 1; + let payload: GraphQlResponse; + try { + payload = (await response.json()) as GraphQlResponse; + } catch { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response was not valid JSON'); } - const commits = await fetchJson( - fetchImpl, - `${GITHUB_API}/repos/${repository}/commits?sha=${encodeURIComponent(metadata.default_branch)}&per_page=1`, - options.token, - ); - const commit = exactCommit(commits[0]?.sha, repository); - const rootTree = await fetchJson( - fetchImpl, - `${GITHUB_API}/repos/${repository}/git/trees/${commit}`, - options.token, - ); - if (!Array.isArray(rootTree.tree)) { - throw new Error(`${repository}@${commit}: GitHub root tree response had no tree`); + if (typeof payload.data !== 'object' || payload.data === null) { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response had no data'); } - if (rootTree.truncated === true) { - throw new Error(`${repository}@${commit}: GitHub root tree response was truncated`); + const errors = payload.errors ?? []; + if (!Array.isArray(errors)) { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL errors were invalid'); } - - const rootManifest = selectCorpusFiles(rootTree.tree, DEFAULT_CORPUS_LIMITS).files.some( - (entry) => entry.path === 'package.json', + const unrelatedError = errors.find( + (error) => + typeof error !== 'object' || + error === null || + !batch.some((_, index) => expectedMissingRoot(error, `r${index}`)), ); - if (!rootManifest) { - exclusions.push({ - repository, - commit, - reason: 'root-package-json-unavailable', - }); - continue; + if (unrelatedError !== undefined) { + throw invalidGitHubResponse( + GRAPHQL_URL, + 'GitHub GraphQL returned an unexpected partial error', + ); } + const rate = graphQlRateLimit(payload.data.rateLimit); + api.cost += rate.cost; + api.rateLimit = { + limit: rate.limit, + remaining: rate.remaining, + used: rate.used, + resetAt: rate.resetAt, + }; - locators.push(`${repository}@${commit}`); - if (locators.length === options.requested) break; + for (const [index, candidate] of batch.entries()) { + candidatesConsidered += 1; + const resolved = resolveRepository( + candidate, + `r${index}`, + payload.data[`r${index}`], + errors, + ); + if ('reason' in resolved) exclusions.push(resolved); + else selected.push(resolved); + if (selected.length === options.requested) break; + } + if (selected.length === options.requested) break; + if (rate.remaining === 0) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `GitHub GraphQL primary budget was exhausted before ${options.requested} repositories resolved`, + ); + } } - - if (locators.length !== options.requested) { + if (selected.length !== options.requested) { throw new Error( - `requested ${options.requested} root-eligible repositories but only ${locators.length} were resolved`, + `requested ${options.requested} root-eligible repositories but only ${selected.length} were resolved`, ); } } catch (error) { - const message = redactCorpusText(error instanceof Error ? error.message : String(error)); - const failedEvidence = evidence( - options.requested, - locators, + const originalFailure = githubFailureEvidence(error); + const recordedError = + originalFailure?.kind === 'response-invalid' && + originalFailure.status === null && + lastApiResponse !== undefined + ? invalidGitHubResponse( + GRAPHQL_URL, + error instanceof Error ? error.message : String(error), + lastApiResponse, + ) + : error; + const message = redactCorpusText( + recordedError instanceof Error ? recordedError.message : String(recordedError), + ); + const failure = githubFailureEvidence(recordedError); + const failed: CorpusSampleEvidence = { + schemaVersion: 2, + method: CORPUS_SAMPLE_METHOD, + candidateSnapshotSha256: digest, + requested: options.requested, + actual: selected.length, candidatesConsidered, + status: 'failed', + api, + selected, exclusions, - message, + error: message, + ...(failure === undefined ? {} : { failure }), + }; + writeEvidence( + options.outputFile, + options.evidenceFile, + selected.map((candidate) => `${candidate.repository}@${candidate.commit}`), + failed, ); - writeEvidence(options.outputFile, options.evidenceFile, locators, failedEvidence); throw new Error(message); } - const completeEvidence = evidence(options.requested, locators, candidatesConsidered, exclusions); - writeEvidence(options.outputFile, options.evidenceFile, locators, completeEvidence); - return completeEvidence; + const complete: CorpusSampleEvidence = { + schemaVersion: 2, + method: CORPUS_SAMPLE_METHOD, + candidateSnapshotSha256: digest, + requested: options.requested, + actual: selected.length, + candidatesConsidered, + status: 'complete', + api, + selected, + exclusions, + }; + writeEvidence( + options.outputFile, + options.evidenceFile, + selected.map((candidate) => `${candidate.repository}@${candidate.commit}`), + complete, + ); + return complete; } async function main(): Promise { @@ -235,7 +533,7 @@ async function main(): Promise { requestedText === undefined ) { throw new Error( - 'usage: tsx tools/corpus-resolve.ts candidates.txt repos.txt repository-sample.json count', + 'usage: tsx tools/corpus-resolve.ts repository-candidates.json repos.txt repository-sample.json count', ); } await resolveCorpusSample({ diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index 6359655..88d238a 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -16,26 +16,29 @@ import type { Finding } from '../src/rules/types'; import { type CorpusLimits, DEFAULT_CORPUS_LIMITS, + gitBlobOid, parseRepoLocator, redactCorpusText, selectCorpusFiles, sha256, type TreeEntry, } from './corpus-lib'; +import { + checkedResponse, + type GitHubFailureEvidence, + githubApiResponse, + githubFailureEvidence, + invalidGitHubResponse, +} from './github-api'; const GITHUB_API = 'https://api.github.com'; +const GITHUB_RAW = 'https://raw.githubusercontent.com'; interface GitHubTreeResponse { tree?: TreeEntry[]; truncated?: boolean; } -interface GitHubBlobResponse { - content?: string; - encoding?: string; - size?: number; -} - interface CountSummary { repositories: number; packages: number; @@ -52,6 +55,7 @@ interface RepositoryEvidence { manifestPaths: string[]; truncations: string[]; error?: string; + failure?: GitHubFailureEvidence; rootOnly: Omit; workspaceFull: Omit; } @@ -83,7 +87,12 @@ interface CorpusRunManifest { mode: 'root-and-workspace'; targets: typeof DEFAULT_TARGETS; limits: CorpusLimits; - sampling: { method: string; seed: string }; + sampling: { + method: string; + seed: string; + candidateSnapshotSha256?: string; + sampleEvidenceSha256?: string; + }; environment: { node: string; platform: NodeJS.Platform; arch: string; runnerOs?: string }; repositories: RepositoryEvidence[]; promotedTotals: { rootOnly: CountSummary; workspaceFull: CountSummary }; @@ -100,6 +109,8 @@ export interface CorpusScanOptions { limits?: CorpusLimits; sampleMethod?: string; sampleSeed?: string; + candidateSnapshotFile?: string; + sampleEvidenceFile?: string; fetchImpl?: typeof fetch; } @@ -112,54 +123,169 @@ function exactSourceCommit(value: string): string { return value; } -function readLocators(inputFile: string): ReturnType[] { - const locators = readFileSync(inputFile, 'utf8') +function readLocatorSequence(inputFile: string): ReturnType[] { + return readFileSync(inputFile, 'utf8') .split(/\r?\n/u) .map((line) => line.trim()) .filter((line) => line !== '' && !line.startsWith('#')) .map(parseRepoLocator); +} + +function sortedUniqueLocators( + locators: readonly ReturnType[], +): ReturnType[] { const unique = new Map(locators.map((locator) => [`${locator.repo}@${locator.commit}`, locator])); return [...unique.values()].sort((left, right) => `${left.repo}@${left.commit}`.localeCompare(`${right.repo}@${right.commit}`), ); } -function headers(token: string): Record { - return { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': '2022-11-28', - 'User-Agent': 'scriptspect-corpus-scan', +function validateSampleEvidence( + bytes: Buffer, + candidateSnapshotSha256: string, + sampleMethod: string, + inputLocators: readonly ReturnType[], +): void { + let parsed: unknown; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + throw new Error('corpus sample evidence was not valid JSON'); + } + if (typeof parsed !== 'object' || parsed === null) { + throw new Error('corpus sample evidence was invalid'); + } + const evidence = parsed as { + status?: unknown; + method?: unknown; + candidateSnapshotSha256?: unknown; + selected?: unknown; }; + if (evidence.status !== 'complete') { + throw new Error('corpus sample evidence status was not complete'); + } + if (evidence.method !== sampleMethod) { + throw new Error('corpus sample evidence method did not match the scanner method'); + } + if (evidence.candidateSnapshotSha256 !== candidateSnapshotSha256) { + throw new Error('corpus sample evidence candidate snapshot digest did not match'); + } + if (!Array.isArray(evidence.selected)) { + throw new Error('corpus sample evidence selected locators were invalid'); + } + const selectedLocators = evidence.selected.map((value) => { + if (typeof value !== 'object' || value === null) { + throw new Error('corpus sample evidence selected locators were invalid'); + } + const selected = value as { repository?: unknown; commit?: unknown }; + if (typeof selected.repository !== 'string' || typeof selected.commit !== 'string') { + throw new Error('corpus sample evidence selected locators were invalid'); + } + try { + return parseRepoLocator(`${selected.repository}@${selected.commit}`); + } catch { + throw new Error('corpus sample evidence selected locators were invalid'); + } + }); + const selectedSequence = selectedLocators.map((value) => `${value.repo}@${value.commit}`); + const inputSequence = inputLocators.map((value) => `${value.repo}@${value.commit}`); + if (JSON.stringify(selectedSequence) !== JSON.stringify(inputSequence)) { + throw new Error('corpus sample evidence selected locators did not match repos.txt'); + } +} + +async function fetchJson( + fetchImpl: typeof fetch, + url: string, + token: string, +): Promise<{ data: T; response: Response }> { + const response = await githubApiResponse(fetchImpl, url, token, 'scriptspect-corpus-scan'); + try { + return { data: (await response.json()) as T, response }; + } catch { + throw invalidGitHubResponse(url, `GitHub API returned invalid JSON for ${url}`, response); + } } -async function fetchJson(fetchImpl: typeof fetch, url: string, token: string): Promise { - const response = await fetchImpl(url, { headers: headers(token) }); - if (!response.ok) throw new Error(`GitHub API ${response.status} for ${url}`); - return (await response.json()) as T; +function rawManifestUrl(repo: string, commit: string, path: string): string { + const [owner, repository] = repo.split('/'); + if (owner === undefined || repository === undefined) { + throw new Error(`invalid repository name: ${repo}`); + } + const encodedPath = path.split('/').map(encodeURIComponent).join('/'); + return `${GITHUB_RAW}/${encodeURIComponent(owner)}/${encodeURIComponent(repository)}/${commit}/${encodedPath}`; +} + +async function readBoundedBody( + response: Response, + maxBytes: number, + overflowMessage: string, +): Promise { + if (response.body === null) return Buffer.alloc(0); + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let totalBytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > maxBytes) { + await reader.cancel().catch(() => undefined); + throw new Error(overflowMessage); + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock(); + } + return Buffer.concat(chunks, totalBytes); } async function downloadSelectedFiles( repo: string, + commit: string, entries: readonly TreeEntry[], targetRoot: string, - token: string, fetchImpl: typeof fetch, limits: CorpusLimits, ): Promise { let actualTotal = 0; for (const entry of entries) { - const blob = await fetchJson( + if (!/^[a-f0-9]{40}$/.test(entry.sha)) { + throw new Error( + `${entry.path}: immutable tree Git blob OID was not 40 lowercase hex characters`, + ); + } + if (!Number.isSafeInteger(entry.size) || (entry.size as number) < 0) { + throw new Error(`${entry.path}: immutable tree entry had no valid byte size`); + } + const expectedBytes = entry.size as number; + const url = rawManifestUrl(repo, commit, entry.path); + const response = await checkedResponse( fetchImpl, - `${GITHUB_API}/repos/${repo}/git/blobs/${entry.sha}`, - token, + url, + { + headers: { + Accept: 'application/octet-stream', + 'User-Agent': 'scriptspect-corpus-scan', + }, + redirect: 'error', + }, + 'GitHub raw', ); - if (blob.encoding !== 'base64' || typeof blob.content !== 'string') { - throw new Error(`${entry.path}: GitHub blob response was not base64`); + const hardCap = Math.min( + expectedBytes, + limits.maxFileBytes, + limits.maxTotalBytes - actualTotal, + ); + const sizeMismatchMessage = `${entry.path}: raw byte length did not match the immutable tree entry`; + const bytes = await readBoundedBody(response, hardCap, sizeMismatchMessage); + if (bytes.length !== expectedBytes) { + throw new Error(`${entry.path}: raw byte length did not match the immutable tree entry`); } - const bytes = Buffer.from(blob.content.replace(/\s/gu, ''), 'base64'); - if (bytes.length !== entry.size || blob.size !== entry.size) { - throw new Error(`${entry.path}: blob size did not match the immutable tree entry`); + if (gitBlobOid(bytes) !== entry.sha) { + throw new Error(`${entry.path}: raw bytes did not match the immutable tree Git blob OID`); } actualTotal += bytes.length; if (bytes.length > limits.maxFileBytes || actualTotal > limits.maxTotalBytes) { @@ -284,8 +410,32 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise( - fetchImpl, - `${GITHUB_API}/repos/${locator.repo}/git/trees/${locator.commit}?recursive=1`, - options.token, - ); - if (!Array.isArray(treeResponse.tree)) throw new Error('GitHub tree response had no tree'); + const treeUrl = `${GITHUB_API}/repos/${locator.repo}/git/trees/${locator.commit}?recursive=1`; + const { data: treeResponse, response: treeHttpResponse } = + await fetchJson(fetchImpl, treeUrl, options.token); + if (!Array.isArray(treeResponse.tree)) { + throw invalidGitHubResponse(treeUrl, 'GitHub tree response had no tree', treeHttpResponse); + } const selected = selectCorpusFiles(treeResponse.tree, limits); truncations = [...selected.truncations]; if (treeResponse.truncated === true) truncations.unshift('github-tree-truncated'); manifestPaths = selected.files.map((entry) => entry.path); if (!manifestPaths.includes('package.json')) throw new Error('root package.json was unavailable'); + if (truncations.length !== 0) { + repositories.push({ + repository: locator.repo, + commit: locator.commit, + status: 'truncated', + manifestPaths, + truncations, + rootOnly: emptyCounts(), + workspaceFull: emptyCounts(), + }); + continue; + } await downloadSelectedFiles( locator.repo, + locator.commit, selected.files, tempRoot, - options.token, fetchImpl, limits, ); @@ -320,7 +482,7 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise { sourceCommit: process.env.SCRIPTSPECT_SOURCE_COMMIT ?? process.env.GITHUB_SHA ?? '', sampleMethod: process.env.CORPUS_SAMPLE_METHOD, sampleSeed: process.env.CORPUS_SAMPLE_SEED, + candidateSnapshotFile: process.env.CORPUS_CANDIDATE_SNAPSHOT, + sampleEvidenceFile: process.env.CORPUS_SAMPLE_EVIDENCE, }); } diff --git a/tools/github-api.ts b/tools/github-api.ts new file mode 100644 index 0000000..9136242 --- /dev/null +++ b/tools/github-api.ts @@ -0,0 +1,145 @@ +/** Shared GitHub HTTP failure evidence. Tokens stay in request headers only. */ + +export type GitHubFailureKind = + | 'primary-rate-limit-exhausted' + | 'secondary-rate-limit' + | 'authentication-failed' + | 'permission-denied' + | 'not-found' + | 'http-error' + | 'response-invalid'; + +export interface GitHubFailureEvidence { + kind: GitHubFailureKind; + status: number | null; + url: string; + rateLimit: { + limit: string | null; + remaining: string | null; + reset: string | null; + used: string | null; + resource: string | null; + }; + retryAfter: string | null; + requestId: string | null; +} + +export class GitHubRequestError extends Error { + readonly evidence: GitHubFailureEvidence; + + constructor(message: string, evidence: GitHubFailureEvidence) { + super(message); + this.name = 'GitHubRequestError'; + this.evidence = evidence; + } +} + +function responseHeaders(response: Response): Omit { + return { + status: response.status, + rateLimit: { + limit: response.headers.get('x-ratelimit-limit'), + remaining: response.headers.get('x-ratelimit-remaining'), + reset: response.headers.get('x-ratelimit-reset'), + used: response.headers.get('x-ratelimit-used'), + resource: response.headers.get('x-ratelimit-resource'), + }, + retryAfter: response.headers.get('retry-after'), + requestId: response.headers.get('x-github-request-id'), + }; +} + +function failureKind(response: Response, responseText: string): GitHubFailureKind { + const metadata = responseHeaders(response); + const lower = responseText.toLowerCase(); + if (metadata.rateLimit.remaining === '0') return 'primary-rate-limit-exhausted'; + if ( + response.status === 429 || + metadata.retryAfter !== null || + lower.includes('secondary rate limit') || + lower.includes('abuse detection') + ) { + return 'secondary-rate-limit'; + } + if (response.status === 401) return 'authentication-failed'; + if (response.status === 403) return 'permission-denied'; + if (response.status === 404) return 'not-found'; + return 'http-error'; +} + +export function githubErrorFromResponse( + response: Response, + url: string, + label: string, + responseText: string, + message = `${label} ${response.status} for ${url}`, +): GitHubRequestError { + return new GitHubRequestError(message, { + kind: failureKind(response, responseText), + url, + ...responseHeaders(response), + }); +} + +export function invalidGitHubResponse( + url: string, + message: string, + response?: Response, +): GitHubRequestError { + return new GitHubRequestError(message, { + kind: 'response-invalid', + url, + ...(response === undefined + ? { + status: null, + rateLimit: { limit: null, remaining: null, reset: null, used: null, resource: null }, + retryAfter: null, + requestId: null, + } + : responseHeaders(response)), + }); +} + +export function githubFailureEvidence(error: unknown): GitHubFailureEvidence | undefined { + return error instanceof GitHubRequestError ? error.evidence : undefined; +} + +export function githubApiHeaders(token: string, userAgent: string): Record { + return { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': userAgent, + }; +} + +export async function checkedResponse( + fetchImpl: typeof fetch, + url: string, + init: RequestInit, + label: string, +): Promise { + const response = await fetchImpl(url, init); + if (response.ok) return response; + const responseText = await response.text(); + throw githubErrorFromResponse(response, url, label, responseText); +} + +export async function githubApiResponse( + fetchImpl: typeof fetch, + url: string, + token: string, + userAgent: string, + init: RequestInit = {}, +): Promise { + return checkedResponse( + fetchImpl, + url, + { + ...init, + headers: { ...githubApiHeaders(token, userAgent), ...(init.headers ?? {}) }, + }, + 'GitHub API', + ); +} From 7d00e400f284378cf1f86cbd9c0ac7358fc11e66 Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 09:21:39 +0800 Subject: [PATCH 3/9] fix(corpus): validate durable provenance boundaries --- tests/corpus/corpus-candidates.test.ts | 59 ++- tests/corpus/corpus-resolve.test.ts | 173 ++++++++- tests/corpus/corpus-run.test.ts | 486 +++++++++++++++++++++++-- tools/corpus-candidates.ts | 202 +++++++++- tools/corpus-resolve.ts | 142 +++----- tools/corpus-scan.ts | 285 +++++++++++++-- tools/github-api.ts | 21 ++ 7 files changed, 1204 insertions(+), 164 deletions(-) diff --git a/tests/corpus/corpus-candidates.test.ts b/tests/corpus/corpus-candidates.test.ts index 9cecbc2..9288276 100644 --- a/tests/corpus/corpus-candidates.test.ts +++ b/tests/corpus/corpus-candidates.test.ts @@ -40,6 +40,7 @@ function searchApi(): typeof fetch { return (async (input: string | URL | Request) => { const url = new URL(String(input)); const query = url.searchParams.get('q'); + expect(query).toContain('is:public'); const items = query?.includes('language:typescript') ? [ { full_name: 'alpha/shared', stargazers_count: 100 }, @@ -65,6 +66,10 @@ function searchApi(): typeof fetch { }) as typeof fetch; } +function invalidSearchApi(body: unknown): typeof fetch { + return (async () => Response.json(body)) as typeof fetch; +} + it('persists both ranked strata and a deterministic round-robin candidate universe', async () => { const directory = temporaryDirectory(); const outputFile = join(directory, 'repository-candidates.json'); @@ -83,7 +88,7 @@ it('persists both ranked strata and a deterministic round-robin candidate univer strata: [ { id: 'typescript', - query: 'language:typescript stars:>2000', + query: 'is:public language:typescript stars:>2000', sort: 'stars', order: 'desc', perPage: 100, @@ -95,7 +100,7 @@ it('persists both ranked strata and a deterministic round-robin candidate univer }, { id: 'javascript', - query: 'language:javascript stars:>5000', + query: 'is:public language:javascript stars:>5000', sort: 'stars', order: 'desc', perPage: 100, @@ -191,3 +196,53 @@ it('distinguishes a permission denial from primary rate exhaustion', async () => }, }); }); + +it('rejects Search responses without a valid total_count', async () => { + const directory = temporaryDirectory(); + + await expect( + (await collector())({ + outputFile: join(directory, 'repository-candidates.json'), + token: 'read-only-test-token', + fetchImpl: invalidSearchApi({ + incomplete_results: false, + items: [{ full_name: 'alpha/project', stargazers_count: 100 }], + }), + }), + ).rejects.toThrow(/search response was incomplete or invalid/); +}); + +it('rejects Search responses whose item count does not match the bounded total', async () => { + const directory = temporaryDirectory(); + + await expect( + (await collector())({ + outputFile: join(directory, 'repository-candidates.json'), + token: 'read-only-test-token', + fetchImpl: invalidSearchApi({ + total_count: 2, + incomplete_results: false, + items: [{ full_name: 'alpha/project', stargazers_count: 100 }], + }), + }), + ).rejects.toThrow(/search response was incomplete or invalid/); +}); + +it('rejects Search candidates that are not sorted by non-increasing stars', async () => { + const directory = temporaryDirectory(); + + await expect( + (await collector())({ + outputFile: join(directory, 'repository-candidates.json'), + token: 'read-only-test-token', + fetchImpl: invalidSearchApi({ + total_count: 2, + incomplete_results: false, + items: [ + { full_name: 'alpha/project', stargazers_count: 100 }, + { full_name: 'beta/project', stargazers_count: 101 }, + ], + }), + }), + ).rejects.toThrow(/ranked by stars/); +}); diff --git a/tests/corpus/corpus-resolve.test.ts b/tests/corpus/corpus-resolve.test.ts index 4d3fdfb..f61b020 100644 --- a/tests/corpus/corpus-resolve.test.ts +++ b/tests/corpus/corpus-resolve.test.ts @@ -52,7 +52,7 @@ function candidateSnapshot(): string { strata: [ { id: 'typescript', - query: 'language:typescript stars:>2000', + query: 'is:public language:typescript stars:>2000', sort: 'stars', order: 'desc', perPage: 100, @@ -64,7 +64,7 @@ function candidateSnapshot(): string { }, { id: 'javascript', - query: 'language:javascript stars:>5000', + query: 'is:public language:javascript stars:>5000', sort: 'stars', order: 'desc', perPage: 100, @@ -262,6 +262,7 @@ it('hard-fails a rate exhaustion and persists non-secret response metadata', asy expect(JSON.parse(evidenceText)).toMatchObject({ schemaVersion: 2, status: 'failed', + api: { requests: 1 }, failure: { kind: 'primary-rate-limit-exhausted', status: 403, @@ -279,6 +280,100 @@ it('hard-fails a rate exhaustion and persists non-secret response metadata', asy }); }); +it('counts an emitted GraphQL request when GitHub responds with HTTP 429', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const outputFile = join(directory, 'repos.txt'); + const evidenceFile = join(directory, 'repository-sample.json'); + writeFileSync(candidateFile, candidateSnapshot(), 'utf8'); + + await expect( + (await resolver())({ + candidateFile, + outputFile, + evidenceFile, + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => + new Response('{"message":"slow down"}', { + status: 429, + headers: { 'retry-after': '30', 'x-github-request-id': 'REQ-429' }, + })) as typeof fetch, + }), + ).rejects.toThrow('GitHub API 429'); + + expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toMatchObject({ + status: 'failed', + api: { requests: 1 }, + failure: { + kind: 'secondary-rate-limit', + status: 429, + retryAfter: '30', + requestId: 'REQ-429', + }, + }); +}); + +it.each([ + { + graphQlType: 'RATE_LIMITED', + expectedKind: 'primary-rate-limit-exhausted', + remaining: '0', + }, + { + graphQlType: 'RATE_LIMITED', + expectedKind: 'secondary-rate-limit', + remaining: '4999', + }, + { graphQlType: 'FORBIDDEN', expectedKind: 'permission-denied', remaining: '4999' }, + { graphQlType: 'UNAUTHORIZED', expectedKind: 'authentication-failed', remaining: '4999' }, +])( + 'classifies a GraphQL 200 $graphQlType error', + async ({ graphQlType, expectedKind, remaining }) => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const evidenceFile = join(directory, 'repository-sample.json'); + writeFileSync(candidateFile, candidateSnapshot(), 'utf8'); + + await expect( + (await resolver())({ + candidateFile, + outputFile: join(directory, 'repos.txt'), + evidenceFile, + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => + Response.json( + { + data: null, + errors: [{ type: graphQlType, message: 'controlled GraphQL failure' }], + }, + { + headers: { + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': remaining, + 'x-ratelimit-reset': '1788224400', + 'x-ratelimit-used': remaining === '0' ? '5000' : '1', + 'x-ratelimit-resource': 'graphql', + 'x-github-request-id': `GRAPHQL-${graphQlType}`, + }, + }, + )) as typeof fetch, + }), + ).rejects.toThrow(/GitHub GraphQL/); + + expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toMatchObject({ + status: 'failed', + api: { requests: 1 }, + failure: { + kind: expectedKind, + status: 200, + requestId: `GRAPHQL-${graphQlType}`, + }, + }); + }, +); + it('rejects a snapshot whose ordered universe does not reproduce its ranked strata', async () => { const directory = temporaryDirectory(); const candidateFile = join(directory, 'repository-candidates.json'); @@ -304,3 +399,77 @@ it('rejects a snapshot whose ordered universe does not reproduce its ranked stra ).rejects.toThrow('candidate snapshot ordering did not match its ranked strata'); expect(called).toBe(false); }); + +it('rejects a snapshot with more than 100 candidates in one Search stratum', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const snapshot = JSON.parse(candidateSnapshot()) as { + strata: Array<{ candidates: Array> }>; + }; + const [typescript, javascript] = snapshot.strata; + if (typescript === undefined || javascript === undefined) { + throw new Error('candidate snapshot test fixture was incomplete'); + } + typescript.candidates = Array.from({ length: 101 }, (_, index) => ({ + rank: index + 1, + repository: `typescript/project-${index}`, + stars: 1_000 - index, + })); + javascript.candidates = []; + writeFileSync(candidateFile, `${JSON.stringify(snapshot)}\n`, 'utf8'); + let called = false; + + await expect( + (await resolver())({ + candidateFile, + outputFile: join(directory, 'repos.txt'), + evidenceFile: join(directory, 'repository-sample.json'), + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => { + called = true; + return new Response('unexpected'); + }) as typeof fetch, + }), + ).rejects.toThrow(/stratum exceeded 100 candidates/); + expect(called).toBe(false); +}); + +it('rejects a snapshot whose raw Search candidate budget exceeds 200', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const snapshot = JSON.parse(candidateSnapshot()) as { + strata: Array<{ candidates: Array> }>; + }; + const [typescript, javascript] = snapshot.strata; + if (typescript === undefined || javascript === undefined) { + throw new Error('candidate snapshot test fixture was incomplete'); + } + typescript.candidates = Array.from({ length: 101 }, (_, index) => ({ + rank: index + 1, + repository: `typescript/project-${index}`, + stars: 2_000 - index, + })); + javascript.candidates = Array.from({ length: 100 }, (_, index) => ({ + rank: index + 1, + repository: `javascript/project-${index}`, + stars: 1_000 - index, + })); + writeFileSync(candidateFile, `${JSON.stringify(snapshot)}\n`, 'utf8'); + let called = false; + + await expect( + (await resolver())({ + candidateFile, + outputFile: join(directory, 'repos.txt'), + evidenceFile: join(directory, 'repository-sample.json'), + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => { + called = true; + return new Response('unexpected'); + }) as typeof fetch, + }), + ).rejects.toThrow(/candidate budget exceeded 200/); + expect(called).toBe(false); +}); diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index d96df9d..93bf304 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -110,6 +110,128 @@ function fixture(): { tree: TreeEntry[]; blobs: Record; rawScrip }; } +function sensitiveDiagnosticFixture(): { + tree: TreeEntry[]; + blobs: Record; + sentinels: string[]; +} { + const sentinels = [ + 'CORPUS_ENV_SENTINEL_a71f', + 'CORPUS_SUBSTITUTION_SENTINEL_b82e', + 'CORPUS_CMD_SENTINEL_c93d', + ]; + const root = Buffer.from( + JSON.stringify({ + name: 'sensitive-message-fixture', + scripts: { + env: `echo $${sentinels[0]}`, + substitution: `echo $(printf ${sentinels[1]})`, + cmd: `echo %${sentinels[2]}%`, + }, + }), + ); + return { + sentinels, + tree: [ + { + path: 'package.json', + type: 'blob', + mode: '100644', + size: root.length, + sha: fixtureGitBlobOid(root), + }, + ], + blobs: { 'package.json': root }, + }; +} + +function completeCandidateSnapshot(): Record { + return { + schemaVersion: 1, + status: 'complete', + method: 'popularity-strata-round-robin-v1', + strata: [ + { + id: 'typescript', + query: 'is:public language:typescript stars:>2000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: 'a'.repeat(64), + candidates: [{ rank: 1, repository: 'example/project', stars: 10_000 }], + }, + { + id: 'javascript', + query: 'is:public language:javascript stars:>5000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: 'b'.repeat(64), + candidates: [], + }, + ], + orderedCandidates: [ + { position: 1, stratum: 'typescript', rank: 1, repository: 'example/project' }, + ], + }; +} + +function completeSampleEvidence( + candidateSnapshot: Buffer, + rootManifestOid: string, + rootManifestBytes: number, +): Record { + return { + schemaVersion: 2, + method: 'popularity-strata-round-robin-v1', + candidateSnapshotSha256: createHash('sha256').update(candidateSnapshot).digest('hex'), + requested: 1, + actual: 1, + candidatesConsidered: 1, + status: 'complete', + api: { + transport: 'github-graphql-batch-v1', + batchSize: 20, + requests: 1, + cost: 1, + rateLimit: { + limit: 5000, + remaining: 4999, + used: 1, + resetAt: '2026-09-01T01:00:00Z', + }, + }, + selected: [ + { + position: 1, + stratum: 'typescript', + rank: 1, + repository: 'example/project', + commit: COMMIT, + rootManifestOid, + rootManifestBytes, + }, + ], + exclusions: [], + }; +} + +function completeProvenance(data: ReturnType): { + candidateSnapshot: Buffer; + sampleEvidence: Record; +} { + const candidateSnapshot = Buffer.from( + `${JSON.stringify(completeCandidateSnapshot(), null, 2)}\n`, + 'utf8', + ); + const root = data.tree.find((entry) => entry.path === 'package.json'); + if (root?.size === undefined) throw new Error('test root manifest fixture was incomplete'); + return { + candidateSnapshot, + sampleEvidence: completeSampleEvidence(candidateSnapshot, root.sha, root.size), + }; +} + describe('immutable corpus run evidence', () => { it('uses canonical workspace analysis while persisting hashes instead of script source', async () => { const directory = temporaryDirectory(); @@ -173,6 +295,37 @@ describe('immutable corpus run evidence', () => { }); }); + it('uses source-free corpus messages for arbitrary environment and substitution findings', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = sensitiveDiagnosticFixture(); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, data.blobs), + }); + + const findingsText = readFileSync(join(outputDir, 'findings.jsonl'), 'utf8'); + const findings = findingsText + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { ruleId: string; message: string }); + expect(findings.length).toBeGreaterThan(0); + for (const sentinel of data.sentinels) expect(findingsText).not.toContain(sentinel); + expect( + findings.every( + (finding) => + finding.message === `${finding.ruleId} matched a portability rule at the recorded span.`, + ), + ).toBe(true); + }); + it('makes truncation explicit and excludes the repository from promoted totals', async () => { const directory = temporaryDirectory(); const inputFile = join(directory, 'repos.txt'); @@ -205,6 +358,77 @@ describe('immutable corpus run evidence', () => { expect(observation.rawUrls).toEqual([]); }); + it.each([ + { name: 'missing', payload: (tree: TreeEntry[]) => ({ tree }) }, + { name: 'string', payload: (tree: TreeEntry[]) => ({ tree, truncated: 'false' }) }, + { name: 'null', payload: (tree: TreeEntry[]) => ({ tree, truncated: null }) }, + ])('fails closed when tree truncated is $name', async ({ payload }) => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + const observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }; + const rawApi = fakeGitHub(data.tree, data.blobs, observation); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async (input: string | URL | Request, init?: RequestInit) => + String(input).includes('/git/trees/') + ? Response.json(payload(data.tree)) + : rawApi(input, init)) as typeof fetch, + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; failure?: { kind?: string } }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + failure: { kind: 'response-invalid' }, + }); + expect(observation.rawUrls).toEqual([]); + }); + + it('classifies a truncated GitHub tree without a root manifest as truncated', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + let rawCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + const manifest = await runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async (input: string | URL | Request) => { + if (String(input).includes('/git/trees/')) { + return Response.json({ + tree: data.tree.filter((entry) => entry.path !== 'package.json'), + truncated: true, + }); + } + rawCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + }); + + expect(manifest.repositories[0]).toMatchObject({ + status: 'truncated', + truncations: ['github-tree-truncated'], + }); + expect(manifest.promotedTotals.workspaceFull.repositories).toBe(0); + expect(rawCalled).toBe(false); + }); + it('fails closed when raw bytes do not match the immutable tree blob OID', async () => { const directory = temporaryDirectory(); const inputFile = join(directory, 'repos.txt'); @@ -452,18 +676,14 @@ describe('immutable corpus run evidence', () => { const outputDir = join(directory, 'out'); const candidateSnapshotFile = join(directory, 'repository-candidates.json'); const sampleEvidenceFile = join(directory, 'repository-sample.json'); - const candidateSnapshot = Buffer.from('{"complete":true}\r\n', 'utf8'); + const data = fixture(); + const provenance = completeProvenance(data); + const candidateSnapshot = provenance.candidateSnapshot; const candidateSnapshotSha256 = createHash('sha256').update(candidateSnapshot).digest('hex'); const sampleEvidence = Buffer.from( - `${JSON.stringify({ - status: 'complete', - method: 'popularity-strata-round-robin-v1', - candidateSnapshotSha256, - selected: [{ repository: 'example/project', commit: COMMIT }], - })}\n`, + `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`, 'utf8', ); - const data = fixture(); writeFileSync(inputFile, `example/project@${COMMIT}\n`); writeFileSync(candidateSnapshotFile, candidateSnapshot); writeFileSync(sampleEvidenceFile, sampleEvidence); @@ -494,26 +714,34 @@ describe('immutable corpus run evidence', () => { }); it('rejects mismatched sample provenance before making a network request', async () => { - const candidateSnapshot = Buffer.from('{"complete":true}\n', 'utf8'); - const candidateSnapshotSha256 = createHash('sha256').update(candidateSnapshot).digest('hex'); - const validEvidence = { - status: 'complete', - method: 'popularity-strata-round-robin-v1', - candidateSnapshotSha256, - selected: [{ repository: 'example/project', commit: COMMIT }], - }; + const data = fixture(); + const provenance = completeProvenance(data); const cases = [ - { name: 'status', evidence: { ...validEvidence, status: 'failed' } }, - { name: 'method', evidence: { ...validEvidence, method: 'wrong-method' } }, + { + name: 'status', + mutate: (evidence: Record) => { + evidence.status = 'failed'; + }, + }, + { + name: 'method', + mutate: (evidence: Record) => { + evidence.method = 'wrong-method'; + }, + }, { name: 'candidate snapshot digest', - evidence: { ...validEvidence, candidateSnapshotSha256: '0'.repeat(64) }, + mutate: (evidence: Record) => { + evidence.candidateSnapshotSha256 = '0'.repeat(64); + }, }, { name: 'selected locator sequence', - evidence: { - ...validEvidence, - selected: [{ repository: 'another/project', commit: COMMIT }], + mutate: (evidence: Record) => { + const selected = evidence.selected as Array>; + const first = selected[0]; + if (first === undefined) throw new Error('sample evidence test fixture was incomplete'); + first.repository = 'another/project'; }, }, ]; @@ -524,11 +752,12 @@ describe('immutable corpus run evidence', () => { const outputDir = join(directory, 'out'); const candidateSnapshotFile = join(directory, 'repository-candidates.json'); const sampleEvidenceFile = join(directory, 'repository-sample.json'); - const data = fixture(); + const evidence = structuredClone(provenance.sampleEvidence); + testCase.mutate(evidence); let networkCalled = false; writeFileSync(inputFile, `example/project@${COMMIT}\n`); - writeFileSync(candidateSnapshotFile, candidateSnapshot); - writeFileSync(sampleEvidenceFile, `${JSON.stringify(testCase.evidence)}\n`, 'utf8'); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(evidence)}\n`, 'utf8'); const upstream = fakeGitHub(data.tree, data.blobs); await expect( @@ -550,4 +779,211 @@ describe('immutable corpus run evidence', () => { expect(networkCalled, testCase.name).toBe(false); } }); + + it('rejects malformed candidate snapshot structure before making a network request', async () => { + const data = fixture(); + const cases = [ + { + name: 'schema', + mutate: (snapshot: Record) => { + snapshot.schemaVersion = 99; + }, + }, + { + name: 'stratum metadata', + mutate: (snapshot: Record) => { + const strata = snapshot.strata as Array>; + const first = strata[0]; + if (first === undefined) + throw new Error('candidate snapshot test fixture was incomplete'); + first.query = 'language:typescript'; + }, + }, + { + name: 'ordered universe', + mutate: (snapshot: Record) => { + snapshot.orderedCandidates = []; + }, + }, + ]; + + for (const testCase of cases) { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const snapshot = completeCandidateSnapshot(); + testCase.mutate(snapshot); + const candidateSnapshot = Buffer.from(`${JSON.stringify(snapshot)}\n`, 'utf8'); + const root = data.tree.find((entry) => entry.path === 'package.json'); + if (root?.size === undefined) throw new Error('test root manifest fixture was incomplete'); + const evidence = completeSampleEvidence(candidateSnapshot, root.sha, root.size); + let networkCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(evidence)}\n`, 'utf8'); + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/candidate snapshot/); + expect(networkCalled, testCase.name).toBe(false); + } + }); + + it('rejects incomplete sample evidence structure before making a network request', async () => { + const data = fixture(); + const provenance = completeProvenance(data); + const cases = [ + { + name: 'schemaVersion', + mutate: (evidence: Record) => { + delete evidence.schemaVersion; + }, + }, + { + name: 'requested/actual contract', + mutate: (evidence: Record) => { + evidence.requested = 2; + }, + }, + { + name: 'api', + mutate: (evidence: Record) => { + delete evidence.api; + }, + }, + { + name: 'exclusions', + mutate: (evidence: Record) => { + evidence.exclusions = [{}]; + }, + }, + { + name: 'root manifest identity', + mutate: (evidence: Record) => { + const selected = evidence.selected as Array>; + const first = selected[0]; + if (first === undefined) throw new Error('sample evidence test fixture was incomplete'); + delete first.rootManifestOid; + }, + }, + ]; + + for (const testCase of cases) { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const evidence = structuredClone(provenance.sampleEvidence); + testCase.mutate(evidence); + let networkCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(evidence)}\n`, 'utf8'); + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/corpus sample evidence/); + expect(networkCalled, testCase.name).toBe(false); + } + }); + + it.each([ + { + name: 'OID', + mutate: (selected: Record) => { + selected.rootManifestOid = 'f'.repeat(40); + }, + }, + { + name: 'byte size', + mutate: (selected: Record) => { + selected.rootManifestBytes = Number(selected.rootManifestBytes) + 1; + }, + }, + ])('cross-checks the sample root manifest $name against the REST tree', async ({ mutate }) => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + const selected = (provenance.sampleEvidence.selected as Array>)[0]; + if (selected === undefined) throw new Error('sample evidence test fixture was incomplete'); + mutate(selected); + const observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(provenance.sampleEvidence)}\n`, 'utf8'); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: fakeGitHub(data.tree, data.blobs, observation), + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: expect.stringMatching(/root package\.json.*sample evidence/), + }); + expect(observation.rawUrls).toEqual([]); + }); + + it('requires candidate and sample evidence files to be provided together', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const data = fixture(); + const provenance = completeProvenance(data); + let networkCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + }), + ).rejects.toThrow(/candidate snapshot and sample evidence must be provided together/); + expect(networkCalled).toBe(false); + }); }); diff --git a/tools/corpus-candidates.ts b/tools/corpus-candidates.ts index b62dbb5..ca34ee6 100644 --- a/tools/corpus-candidates.ts +++ b/tools/corpus-candidates.ts @@ -68,14 +68,14 @@ export const CORPUS_CANDIDATE_STRATA: ReadonlyArray< > = [ { id: 'typescript', - query: 'language:typescript stars:>2000', + query: 'is:public language:typescript stars:>2000', sort: 'stars', order: 'desc', perPage: 100, }, { id: 'javascript', - query: 'language:javascript stars:>5000', + query: 'is:public language:javascript stars:>5000', sort: 'stars', order: 'desc', perPage: 100, @@ -90,6 +90,183 @@ function validRepositoryName(value: string): boolean { ); } +function exactKeys(value: Record, keys: readonly string[]): boolean { + const actual = Object.keys(value).toSorted(); + const expected = [...keys].toSorted(); + return actual.length === expected.length && actual.every((key, index) => key === expected[index]); +} + +function validateRankedCandidate( + value: unknown, + index: number, + stratumId: CandidateStratum['id'], +): RankedCandidate { + if ( + typeof value !== 'object' || + value === null || + Array.isArray(value) || + !exactKeys(value as Record, ['rank', 'repository', 'stars']) + ) { + throw new Error(`candidate snapshot ${stratumId}: ranked candidate ${index + 1} was invalid`); + } + const candidate = value as Record; + if ( + candidate.rank !== index + 1 || + typeof candidate.repository !== 'string' || + !validRepositoryName(candidate.repository) || + typeof candidate.stars !== 'number' || + !Number.isSafeInteger(candidate.stars) || + candidate.stars < 0 + ) { + throw new Error(`candidate snapshot ${stratumId}: ranked candidate ${index + 1} was invalid`); + } + return { + rank: candidate.rank, + repository: candidate.repository, + stars: candidate.stars, + }; +} + +function validateCandidateStratum(value: unknown, index: number): CandidateStratum { + if ( + typeof value !== 'object' || + value === null || + Array.isArray(value) || + !exactKeys(value as Record, [ + 'id', + 'query', + 'sort', + 'order', + 'perPage', + 'responseSha256', + 'candidates', + ]) + ) { + throw new Error('candidate snapshot stratum metadata was invalid'); + } + const stratum = value as Record; + const expected = CORPUS_CANDIDATE_STRATA[index]; + if ( + expected === undefined || + stratum.id !== expected.id || + stratum.query !== expected.query || + stratum.sort !== expected.sort || + stratum.order !== expected.order || + stratum.perPage !== expected.perPage || + typeof stratum.responseSha256 !== 'string' || + !/^[a-f0-9]{64}$/.test(stratum.responseSha256) || + !Array.isArray(stratum.candidates) + ) { + throw new Error('candidate snapshot stratum metadata was invalid'); + } + if (stratum.candidates.length > 100) { + throw new Error(`candidate snapshot ${expected.id}: candidate stratum exceeded 100 candidates`); + } + const candidates = stratum.candidates.map((candidate, candidateIndex) => + validateRankedCandidate(candidate, candidateIndex, expected.id), + ); + if ( + candidates.some((candidate, candidateIndex) => { + const previous = candidates[candidateIndex - 1]; + return previous !== undefined && candidate.stars > previous.stars; + }) + ) { + throw new Error( + `candidate snapshot ${expected.id}: ranked candidates were not non-increasing by stars`, + ); + } + if (new Set(candidates.map((candidate) => candidate.repository)).size !== candidates.length) { + throw new Error(`candidate snapshot ${expected.id}: ranked candidates contained duplicates`); + } + return { ...expected, responseSha256: stratum.responseSha256, candidates }; +} + +/** Parse and fully validate the durable Search snapshot before any downstream API use. */ +export function parseCorpusCandidateSnapshot(bytes: Buffer): { + snapshot: CorpusCandidateSnapshot; + digest: string; +} { + let parsed: unknown; + try { + parsed = JSON.parse(bytes.toString('utf8')) as unknown; + } catch { + throw new Error('candidate snapshot was not valid JSON'); + } + if ( + typeof parsed !== 'object' || + parsed === null || + Array.isArray(parsed) || + !exactKeys(parsed as Record, [ + 'schemaVersion', + 'status', + 'method', + 'strata', + 'orderedCandidates', + ]) + ) { + throw new Error('candidate snapshot was incomplete or incompatible'); + } + const candidate = parsed as Record; + if ( + candidate.schemaVersion !== 1 || + candidate.status !== 'complete' || + candidate.method !== CORPUS_SAMPLE_METHOD || + !Array.isArray(candidate.strata) || + !Array.isArray(candidate.orderedCandidates) + ) { + throw new Error('candidate snapshot was incomplete or incompatible'); + } + if (candidate.strata.length !== CORPUS_CANDIDATE_STRATA.length) { + throw new Error('candidate snapshot did not contain the required popularity strata'); + } + const rawCandidateCount = candidate.strata.reduce((total, value) => { + if (typeof value !== 'object' || value === null || !Array.isArray(value.candidates)) { + return total; + } + return total + value.candidates.length; + }, 0); + if (rawCandidateCount > 200) { + throw new Error('candidate snapshot Search candidate budget exceeded 200'); + } + const strata = candidate.strata.map(validateCandidateStratum); + const expectedOrder = interleaveCandidateStrata(strata); + const orderedCandidates = candidate.orderedCandidates.map((value, index): OrderedCandidate => { + if ( + typeof value !== 'object' || + value === null || + Array.isArray(value) || + !exactKeys(value as Record, ['position', 'stratum', 'rank', 'repository']) + ) { + throw new Error('candidate snapshot ordering did not match its ranked strata'); + } + const ordered = value as Record; + const expected = expectedOrder[index]; + if ( + expected === undefined || + ordered.position !== expected.position || + ordered.stratum !== expected.stratum || + ordered.rank !== expected.rank || + ordered.repository !== expected.repository + ) { + throw new Error('candidate snapshot ordering did not match its ranked strata'); + } + return expected; + }); + if (orderedCandidates.length !== expectedOrder.length) { + throw new Error('candidate snapshot ordering did not match its ranked strata'); + } + return { + snapshot: { + schemaVersion: 1, + status: 'complete', + method: CORPUS_SAMPLE_METHOD, + strata, + orderedCandidates, + }, + digest: sha256(bytes), + }; +} + export function interleaveCandidateStrata(strata: readonly CandidateStratum[]): OrderedCandidate[] { const seen = new Set(); const ordered: OrderedCandidate[] = []; @@ -153,7 +330,14 @@ export async function collectCorpusCandidates( response, ); } - if (parsed.incomplete_results !== false || !Array.isArray(parsed.items)) { + if ( + parsed.incomplete_results !== false || + typeof parsed.total_count !== 'number' || + !Number.isSafeInteger(parsed.total_count) || + parsed.total_count < 0 || + !Array.isArray(parsed.items) || + parsed.items.length !== Math.min(parsed.total_count, definition.perPage) + ) { throw invalidGitHubResponse( url, `${definition.id}: GitHub search response was incomplete or invalid`, @@ -176,6 +360,18 @@ export async function collectCorpusCandidates( } return { rank: index + 1, repository: item.full_name, stars: item.stargazers_count }; }); + if ( + candidates.some((candidate, index) => { + const previous = candidates[index - 1]; + return previous !== undefined && candidate.stars > previous.stars; + }) + ) { + throw invalidGitHubResponse( + url, + `${definition.id}: GitHub search candidates were not ranked by stars`, + response, + ); + } strata.push({ ...definition, responseSha256: sha256(responseText), candidates }); } const snapshot: CorpusCandidateSnapshot = { diff --git a/tools/corpus-resolve.ts b/tools/corpus-resolve.ts index c4f8029..30d0cf9 100644 --- a/tools/corpus-resolve.ts +++ b/tools/corpus-resolve.ts @@ -3,15 +3,13 @@ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { - type CandidateStratum, - CORPUS_CANDIDATE_STRATA, CORPUS_SAMPLE_METHOD, - type CorpusCandidateSnapshot, - interleaveCandidateStrata, type OrderedCandidate, + parseCorpusCandidateSnapshot, } from './corpus-candidates'; -import { DEFAULT_CORPUS_LIMITS, redactCorpusText, sha256 } from './corpus-lib'; +import { DEFAULT_CORPUS_LIMITS, redactCorpusText } from './corpus-lib'; import { + classifiedGitHubError, type GitHubFailureEvidence, githubApiResponse, githubFailureEvidence, @@ -65,6 +63,39 @@ interface GraphQlResponse { errors?: GraphQlError[]; } +function classifiedGraphQlFailure( + errors: readonly GraphQlError[], + response: Response, +): + | { + kind: + | 'primary-rate-limit-exhausted' + | 'secondary-rate-limit' + | 'authentication-failed' + | 'permission-denied'; + type: 'RATE_LIMITED' | 'UNAUTHORIZED' | 'FORBIDDEN'; + } + | undefined { + for (const error of errors) { + if (error.type === 'RATE_LIMITED') { + return { + kind: + response.headers.get('x-ratelimit-remaining') === '0' + ? 'primary-rate-limit-exhausted' + : 'secondary-rate-limit', + type: error.type, + }; + } + if (error.type === 'UNAUTHORIZED') { + return { kind: 'authentication-failed', type: error.type }; + } + if (error.type === 'FORBIDDEN') { + return { kind: 'permission-denied', type: error.type }; + } + } + return undefined; +} + interface ApiEvidence { transport: 'github-graphql-batch-v1'; batchSize: 20; @@ -117,13 +148,6 @@ export interface CorpusResolveOptions { fetchImpl?: typeof fetch; } -function validRepositoryName(value: string): boolean { - const match = /^([A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99}))$/.exec( - value, - ); - return match !== null && !value.includes('..') && !value.endsWith('.'); -} - function exactOid(value: unknown, description: string): string { if (typeof value !== 'string' || !/^[a-f0-9]{40}$/.test(value)) { throw invalidGitHubResponse(GRAPHQL_URL, `${description} was not an exact 40-character oid`); @@ -138,81 +162,6 @@ function safeInteger(value: unknown, description: string): number { return value; } -function validateStratum(value: unknown, index: number): CandidateStratum { - if (typeof value !== 'object' || value === null) throw new Error('candidate stratum was invalid'); - const stratum = value as Partial; - const expected = CORPUS_CANDIDATE_STRATA[index]; - if ( - expected === undefined || - stratum.id !== expected.id || - stratum.query !== expected.query || - stratum.sort !== expected.sort || - stratum.order !== expected.order || - stratum.perPage !== expected.perPage || - typeof stratum.responseSha256 !== 'string' || - !/^[a-f0-9]{64}$/.test(stratum.responseSha256) || - !Array.isArray(stratum.candidates) - ) { - throw new Error('candidate stratum metadata was invalid'); - } - const candidates = stratum.candidates.map((candidate, index) => { - if ( - typeof candidate !== 'object' || - candidate === null || - candidate.rank !== index + 1 || - typeof candidate.repository !== 'string' || - !validRepositoryName(candidate.repository) || - typeof candidate.stars !== 'number' || - !Number.isSafeInteger(candidate.stars) || - candidate.stars < 0 - ) { - throw new Error(`${stratum.id}: ranked candidate ${index + 1} was invalid`); - } - return candidate; - }); - return { ...stratum, candidates } as CandidateStratum; -} - -function readCandidateSnapshot(candidateFile: string): { - snapshot: CorpusCandidateSnapshot; - digest: string; -} { - const bytes = readFileSync(candidateFile); - let parsed: unknown; - try { - parsed = JSON.parse(bytes.toString('utf8')); - } catch { - throw new Error('candidate snapshot was not valid JSON'); - } - if (typeof parsed !== 'object' || parsed === null) - throw new Error('candidate snapshot was invalid'); - const candidate = parsed as Partial; - if ( - candidate.schemaVersion !== 1 || - candidate.status !== 'complete' || - candidate.method !== CORPUS_SAMPLE_METHOD || - !Array.isArray(candidate.strata) || - !Array.isArray(candidate.orderedCandidates) - ) { - throw new Error('candidate snapshot was incomplete or incompatible'); - } - if (candidate.strata.length !== CORPUS_CANDIDATE_STRATA.length) { - throw new Error('candidate snapshot did not contain the required popularity strata'); - } - const strata = candidate.strata.map(validateStratum); - if (new Set(strata.map((stratum) => stratum.id)).size !== strata.length) { - throw new Error('candidate snapshot contained duplicate strata'); - } - const expected = interleaveCandidateStrata(strata); - if (JSON.stringify(candidate.orderedCandidates) !== JSON.stringify(expected)) { - throw new Error('candidate snapshot ordering did not match its ranked strata'); - } - return { - snapshot: { ...candidate, strata, orderedCandidates: expected } as CorpusCandidateSnapshot, - digest: sha256(bytes), - }; -} - function graphQlQuery(candidates: readonly OrderedCandidate[]): string { const repositories = candidates.map((candidate, index) => { const [owner, name] = candidate.repository.split('/'); @@ -374,7 +323,7 @@ export async function resolveCorpusSample( ) { throw new Error('requested repository count must be an integer from 1 through 100'); } - const { snapshot, digest } = readCandidateSnapshot(options.candidateFile); + const { snapshot, digest } = parseCorpusCandidateSnapshot(readFileSync(options.candidateFile)); if (snapshot.orderedCandidates.length < options.requested) { throw new Error( `requested ${options.requested} repositories but only ${snapshot.orderedCandidates.length} ordered candidates were captured`, @@ -396,6 +345,7 @@ export async function resolveCorpusSample( try { for (let offset = 0; offset < snapshot.orderedCandidates.length; offset += GRAPHQL_BATCH_SIZE) { const batch = snapshot.orderedCandidates.slice(offset, offset + GRAPHQL_BATCH_SIZE); + api.requests += 1; const response = await githubApiResponse( fetchImpl, GRAPHQL_URL, @@ -404,20 +354,28 @@ export async function resolveCorpusSample( { method: 'POST', body: JSON.stringify({ query: graphQlQuery(batch) }) }, ); lastApiResponse = response; - api.requests += 1; let payload: GraphQlResponse; try { payload = (await response.json()) as GraphQlResponse; } catch { throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response was not valid JSON'); } - if (typeof payload.data !== 'object' || payload.data === null) { - throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response had no data'); - } const errors = payload.errors ?? []; if (!Array.isArray(errors)) { throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL errors were invalid'); } + const classifiedFailure = classifiedGraphQlFailure(errors, response); + if (classifiedFailure !== undefined) { + throw classifiedGitHubError( + classifiedFailure.kind, + GRAPHQL_URL, + `GitHub GraphQL returned ${classifiedFailure.type}`, + response, + ); + } + if (typeof payload.data !== 'object' || payload.data === null) { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response had no data'); + } const unrelatedError = errors.find( (error) => typeof error !== 'object' || diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index 88d238a..90e57d7 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -13,6 +13,12 @@ import { type AnalysisResult, analyze } from '../src/core/analyze'; import { DEFAULT_TARGETS } from '../src/core/targets'; import { RULES } from '../src/rules'; import type { Finding } from '../src/rules/types'; +import { + CORPUS_SAMPLE_METHOD, + type CorpusCandidateSnapshot, + type OrderedCandidate, + parseCorpusCandidateSnapshot, +} from './corpus-candidates'; import { type CorpusLimits, DEFAULT_CORPUS_LIMITS, @@ -36,7 +42,7 @@ const GITHUB_RAW = 'https://raw.githubusercontent.com'; interface GitHubTreeResponse { tree?: TreeEntry[]; - truncated?: boolean; + truncated?: unknown; } interface CountSummary { @@ -77,6 +83,12 @@ interface FindingEvidence { message: string; } +interface ValidatedSampleSelection extends OrderedCandidate { + commit: string; + rootManifestOid: string; + rootManifestBytes: number; +} + interface CorpusRunManifest { schemaVersion: 1; generatedAt: string; @@ -140,58 +152,219 @@ function sortedUniqueLocators( ); } +function exactEvidenceRecord( + value: unknown, + keys: readonly string[], + description: string, +): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`corpus sample evidence ${description} was invalid`); + } + const record = value as Record; + const actualKeys = Object.keys(record).toSorted(); + const expectedKeys = [...keys].toSorted(); + if ( + actualKeys.length !== expectedKeys.length || + !actualKeys.every((key, index) => key === expectedKeys[index]) + ) { + throw new Error(`corpus sample evidence ${description} was invalid`); + } + return record; +} + +function evidenceInteger(value: unknown, description: string, minimum = 0): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) { + throw new Error(`corpus sample evidence ${description} was invalid`); + } + return value; +} + +function evidenceOid(value: unknown, description: string): string { + if (typeof value !== 'string' || !/^[a-f0-9]{40}$/.test(value)) { + throw new Error(`corpus sample evidence ${description} was invalid`); + } + return value; +} + +function evidenceCandidateIdentity( + record: Record, + snapshot: CorpusCandidateSnapshot, + description: string, +): OrderedCandidate { + const position = evidenceInteger(record.position, `${description} position`, 1); + const expected = snapshot.orderedCandidates[position - 1]; + if ( + expected === undefined || + record.stratum !== expected.stratum || + record.rank !== expected.rank || + record.repository !== expected.repository + ) { + throw new Error(`corpus sample evidence ${description} did not match the candidate snapshot`); + } + return expected; +} + function validateSampleEvidence( bytes: Buffer, candidateSnapshotSha256: string, sampleMethod: string, inputLocators: readonly ReturnType[], -): void { + candidateSnapshot: CorpusCandidateSnapshot, +): Map { let parsed: unknown; try { parsed = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('corpus sample evidence was not valid JSON'); } - if (typeof parsed !== 'object' || parsed === null) { - throw new Error('corpus sample evidence was invalid'); - } - const evidence = parsed as { - status?: unknown; - method?: unknown; - candidateSnapshotSha256?: unknown; - selected?: unknown; - }; - if (evidence.status !== 'complete') { - throw new Error('corpus sample evidence status was not complete'); + const evidence = exactEvidenceRecord( + parsed, + [ + 'schemaVersion', + 'method', + 'candidateSnapshotSha256', + 'requested', + 'actual', + 'candidatesConsidered', + 'status', + 'api', + 'selected', + 'exclusions', + ], + 'root object', + ); + if ( + evidence.schemaVersion !== 2 || + evidence.status !== 'complete' || + evidence.method !== sampleMethod || + evidence.method !== CORPUS_SAMPLE_METHOD || + evidence.candidateSnapshotSha256 !== candidateSnapshotSha256 + ) { + throw new Error('corpus sample evidence header did not match the scanner provenance'); } - if (evidence.method !== sampleMethod) { - throw new Error('corpus sample evidence method did not match the scanner method'); + const requested = evidenceInteger(evidence.requested, 'requested count', 1); + const actual = evidenceInteger(evidence.actual, 'actual count', 1); + const candidatesConsidered = evidenceInteger( + evidence.candidatesConsidered, + 'candidatesConsidered', + 1, + ); + if (requested > 100 || actual !== requested || candidatesConsidered < actual) { + throw new Error('corpus sample evidence requested/actual contract was invalid'); } - if (evidence.candidateSnapshotSha256 !== candidateSnapshotSha256) { - throw new Error('corpus sample evidence candidate snapshot digest did not match'); + + const api = exactEvidenceRecord( + evidence.api, + ['transport', 'batchSize', 'requests', 'cost', 'rateLimit'], + 'api object', + ); + const requests = evidenceInteger(api.requests, 'api.requests', 1); + const cost = evidenceInteger(api.cost, 'api.cost', 1); + const rateLimit = exactEvidenceRecord( + api.rateLimit, + ['limit', 'remaining', 'used', 'resetAt'], + 'api.rateLimit object', + ); + const limit = evidenceInteger(rateLimit.limit, 'api.rateLimit.limit', 1); + const remaining = evidenceInteger(rateLimit.remaining, 'api.rateLimit.remaining'); + const used = evidenceInteger(rateLimit.used, 'api.rateLimit.used'); + if ( + api.transport !== 'github-graphql-batch-v1' || + api.batchSize !== 20 || + requests !== Math.ceil(candidatesConsidered / 20) || + cost < requests || + remaining > limit || + used > limit || + typeof rateLimit.resetAt !== 'string' || + Number.isNaN(Date.parse(rateLimit.resetAt)) + ) { + throw new Error('corpus sample evidence api contract was invalid'); } - if (!Array.isArray(evidence.selected)) { - throw new Error('corpus sample evidence selected locators were invalid'); + if (!Array.isArray(evidence.selected) || !Array.isArray(evidence.exclusions)) { + throw new Error('corpus sample evidence selected/exclusions were invalid'); } - const selectedLocators = evidence.selected.map((value) => { - if (typeof value !== 'object' || value === null) { - throw new Error('corpus sample evidence selected locators were invalid'); - } - const selected = value as { repository?: unknown; commit?: unknown }; - if (typeof selected.repository !== 'string' || typeof selected.commit !== 'string') { - throw new Error('corpus sample evidence selected locators were invalid'); + + const selected = evidence.selected.map((value, index): ValidatedSampleSelection => { + const record = exactEvidenceRecord( + value, + [ + 'position', + 'stratum', + 'rank', + 'repository', + 'commit', + 'rootManifestOid', + 'rootManifestBytes', + ], + `selected item ${index + 1}`, + ); + const identity = evidenceCandidateIdentity( + record, + candidateSnapshot, + `selected item ${index + 1}`, + ); + const commit = evidenceOid(record.commit, `selected item ${index + 1} commit`); + const rootManifestOid = evidenceOid( + record.rootManifestOid, + `selected item ${index + 1} rootManifestOid`, + ); + const rootManifestBytes = evidenceInteger( + record.rootManifestBytes, + `selected item ${index + 1} rootManifestBytes`, + ); + if (rootManifestBytes > DEFAULT_CORPUS_LIMITS.maxFileBytes) { + throw new Error( + `corpus sample evidence selected item ${index + 1} root manifest was oversized`, + ); } - try { - return parseRepoLocator(`${selected.repository}@${selected.commit}`); - } catch { - throw new Error('corpus sample evidence selected locators were invalid'); + return { ...identity, commit, rootManifestOid, rootManifestBytes }; + }); + const exclusions = evidence.exclusions.map((value, index) => { + const record = exactEvidenceRecord( + value, + ['position', 'stratum', 'rank', 'repository', 'commit', 'reason'], + `exclusion item ${index + 1}`, + ); + const identity = evidenceCandidateIdentity( + record, + candidateSnapshot, + `exclusion item ${index + 1}`, + ); + const commit = evidenceOid(record.commit, `exclusion item ${index + 1} commit`); + if (record.reason !== 'root-package-json-unavailable') { + throw new Error(`corpus sample evidence exclusion item ${index + 1} reason was invalid`); } + return { ...identity, commit }; }); - const selectedSequence = selectedLocators.map((value) => `${value.repo}@${value.commit}`); + if ( + selected.length !== actual || + selected.length + exclusions.length !== candidatesConsidered || + candidatesConsidered > candidateSnapshot.orderedCandidates.length + ) { + throw new Error('corpus sample evidence candidate accounting was invalid'); + } + const considered = [...selected, ...exclusions].toSorted( + (left, right) => left.position - right.position, + ); + if ( + considered.some((candidate, index) => candidate.position !== index + 1) || + selected.some((candidate, index) => { + const previous = selected[index - 1]; + return previous !== undefined && candidate.position <= previous.position; + }) || + exclusions.some((candidate, index) => { + const previous = exclusions[index - 1]; + return previous !== undefined && candidate.position <= previous.position; + }) + ) { + throw new Error('corpus sample evidence candidate sequence was invalid'); + } + const selectedSequence = selected.map((value) => `${value.repository}@${value.commit}`); const inputSequence = inputLocators.map((value) => `${value.repo}@${value.commit}`); if (JSON.stringify(selectedSequence) !== JSON.stringify(inputSequence)) { throw new Error('corpus sample evidence selected locators did not match repos.txt'); } + return new Map(selected.map((value) => [`${value.repository}@${value.commit}`, value])); } async function fetchJson( @@ -356,7 +529,7 @@ function findingEvidence( confidence: finding.confidence, affectedTargets: finding.affectedTargets, span: finding.span, - message: redactCorpusText(finding.message), + message: `${finding.ruleId} matched a portability rule at the recorded span.`, }; } @@ -410,29 +583,40 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise(); if ( - candidateSnapshotBytes !== undefined && + parsedCandidateSnapshot !== undefined && candidateSnapshotSha256 !== undefined && sampleEvidenceBytes !== undefined ) { - validateSampleEvidence( + sampleSelections = validateSampleEvidence( sampleEvidenceBytes, candidateSnapshotSha256, sampleMethod, locatorSequence, + parsedCandidateSnapshot.snapshot, ); } const locators = sortedUniqueLocators(locatorSequence); @@ -452,12 +636,17 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise entry.path); - if (!manifestPaths.includes('package.json')) - throw new Error('root package.json was unavailable'); if (truncations.length !== 0) { repositories.push({ repository: locator.repo, @@ -470,6 +659,22 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise entry.path === 'package.json'); + const rootEntry = rootEntries[0]; + if ( + rootEntries.length !== 1 || + rootEntry?.type !== 'blob' || + (rootEntry.mode !== '100644' && rootEntry.mode !== '100755') || + rootEntry.sha !== sampleSelection.rootManifestOid || + rootEntry.size !== sampleSelection.rootManifestBytes + ) { + throw new Error('root package.json tree entry did not match corpus sample evidence'); + } + } + if (!manifestPaths.includes('package.json')) + throw new Error('root package.json was unavailable'); await downloadSelectedFiles( locator.repo, locator.commit, diff --git a/tools/github-api.ts b/tools/github-api.ts index 9136242..773d4eb 100644 --- a/tools/github-api.ts +++ b/tools/github-api.ts @@ -100,6 +100,27 @@ export function invalidGitHubResponse( }); } +/** Persist a semantic GitHub failure even when GraphQL returns HTTP 200. */ +export function classifiedGitHubError( + kind: GitHubFailureKind, + url: string, + message: string, + response?: Response, +): GitHubRequestError { + return new GitHubRequestError(message, { + kind, + url, + ...(response === undefined + ? { + status: null, + rateLimit: { limit: null, remaining: null, reset: null, used: null, resource: null }, + retryAfter: null, + requestId: null, + } + : responseHeaders(response)), + }); +} + export function githubFailureEvidence(error: unknown): GitHubFailureEvidence | undefined { return error instanceof GitHubRequestError ? error.evidence : undefined; } From 42e645676deeabdaf678f1f4664c5b7e504011a9 Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 09:56:29 +0800 Subject: [PATCH 4/9] fix(corpus): make provenance replay complete --- docs/evidence/corpus-method.md | 17 +++++++++-- tests/corpus/corpus-run.test.ts | 46 +++++++++++++++++++++++++++++ tools/corpus-scan.ts | 52 +++++++++++++++++++++++++++++++-- 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/docs/evidence/corpus-method.md b/docs/evidence/corpus-method.md index 8622384..1dfffe1 100644 --- a/docs/evidence/corpus-method.md +++ b/docs/evidence/corpus-method.md @@ -54,13 +54,26 @@ The workflow artifact contains: commits/root-manifest blobs; - `repos.txt`: the exact immutable sample; - `findings.jsonl`: stable finding IDs, immutable source URLs, script SHA-256, - rule metadata, spans, and redacted messages—never raw script source; + rule metadata, spans, and source-free rule summaries—never raw script source; - `corpus-run.json`: selected manifest paths, scanner/source commit and hashes, rule-registry hash, limits, sample method/seed, hashes of the full candidate snapshot and sample evidence, environment, per-repository status, separate - scan modes, artifact hashes, and reproduction command; + scan modes, artifact hashes, and a directly copyable POSIX-shell reproduction + command; - `summary.md`: an explicitly unverified summary for maintainers. +To replay a run, place its `repository-candidates.json`, +`repository-sample.json`, and `repos.txt` beside the repository checkout, set +`GITHUB_TOKEN` externally to a read-only public-repository token, and run the +command recorded in `corpus-run.json`. The command checks out the exact source +commit, activates the pinned pnpm version, installs from the frozen lockfile, +and refuses to reuse its deterministic independent output directory. It binds +the original generation timestamp, sample method and seed, candidate snapshot, +sample evidence, repository list, and output directory to the scanner's actual +environment variables and positional arguments. Only evidence basenames and a +token-variable reference are recorded; neither the credential nor a local +absolute path is persisted. + The run fails if any repository fails, while still leaving `corpus-run.json` for diagnosis. Truncation is visible and excluded rather than silently treated as a complete sample. Expiring workflow artifacts supplement; they do not diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index 93bf304..4d30d9b 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -713,6 +713,52 @@ describe('immutable corpus run evidence', () => { }); }); + it('emits a safe path-independent replay command for the complete provenance contract', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos copy.txt'); + const outputDir = join(directory, 'initial output'); + const candidateSnapshotFile = join(directory, "repository candidate's.json"); + const sampleEvidenceFile = join(directory, 'repository sample;ignored.json'); + const data = fixture(); + const provenance = completeProvenance(data); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync( + sampleEvidenceFile, + `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`, + 'utf8', + ); + + const manifest = await runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token-must-not-be-persisted', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, data.blobs), + sampleMethod: 'popularity-strata-round-robin-v1', + sampleSeed: 'candidate-seed', + candidateSnapshotFile, + sampleEvidenceFile, + }); + + const replayOutput = `corpus-reproduction-${SOURCE_COMMIT}`; + expect(manifest.reproduction).toBe( + [ + `: "\${GITHUB_TOKEN:?set GITHUB_TOKEN to a read-only public-repository token}"`, + `git -c advice.detachedHead=false checkout --detach '${SOURCE_COMMIT}'`, + 'corepack enable', + "corepack prepare 'pnpm@11.24.0' --activate", + 'pnpm install --frozen-lockfile', + `test ! -e '${replayOutput}'`, + `mkdir -- '${replayOutput}'`, + `SCRIPTSPECT_SOURCE_COMMIT='${SOURCE_COMMIT}' CORPUS_GENERATED_AT='2026-09-01T00:00:00.000Z' CORPUS_SAMPLE_METHOD='popularity-strata-round-robin-v1' CORPUS_SAMPLE_SEED='candidate-seed' CORPUS_CANDIDATE_SNAPSHOT='repository candidate'"'"'s.json' CORPUS_SAMPLE_EVIDENCE='repository sample;ignored.json' pnpm exec tsx tools/corpus-scan.ts 'repos copy.txt' '${replayOutput}'`, + ].join(' && '), + ); + expect(manifest.reproduction).not.toContain('read-only-test-token-must-not-be-persisted'); + expect(manifest.reproduction).not.toContain(directory); + }); + it('rejects mismatched sample provenance before making a network request', async () => { const data = fixture(); const provenance = completeProvenance(data); diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index 90e57d7..0992c73 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -135,6 +135,44 @@ function exactSourceCommit(value: string): string { return value; } +function posixShellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function reproductionCommand(options: { + sourceCommit: string; + generatedAt: string; + sampleMethod: string; + sampleSeed: string; + inputFile: string; + candidateSnapshotFile?: string; + sampleEvidenceFile?: string; +}): string { + const outputDirectory = `corpus-reproduction-${options.sourceCommit}`; + const environment = [ + `SCRIPTSPECT_SOURCE_COMMIT=${posixShellQuote(options.sourceCommit)}`, + `CORPUS_GENERATED_AT=${posixShellQuote(options.generatedAt)}`, + `CORPUS_SAMPLE_METHOD=${posixShellQuote(options.sampleMethod)}`, + `CORPUS_SAMPLE_SEED=${posixShellQuote(options.sampleSeed)}`, + ...(options.candidateSnapshotFile === undefined + ? [] + : [`CORPUS_CANDIDATE_SNAPSHOT=${posixShellQuote(basename(options.candidateSnapshotFile))}`]), + ...(options.sampleEvidenceFile === undefined + ? [] + : [`CORPUS_SAMPLE_EVIDENCE=${posixShellQuote(basename(options.sampleEvidenceFile))}`]), + ]; + return [ + `: "\${GITHUB_TOKEN:?set GITHUB_TOKEN to a read-only public-repository token}"`, + `git -c advice.detachedHead=false checkout --detach ${posixShellQuote(options.sourceCommit)}`, + 'corepack enable', + "corepack prepare 'pnpm@11.24.0' --activate", + 'pnpm install --frozen-lockfile', + `test ! -e ${posixShellQuote(outputDirectory)}`, + `mkdir -- ${posixShellQuote(outputDirectory)}`, + `${environment.join(' ')} pnpm exec tsx tools/corpus-scan.ts ${posixShellQuote(basename(options.inputFile))} ${posixShellQuote(outputDirectory)}`, + ].join(' && '); +} + function readLocatorSequence(inputFile: string): ReturnType[] { return readFileSync(inputFile, 'utf8') .split(/\r?\n/u) @@ -581,6 +619,7 @@ function renderSummary(manifest: CorpusRunManifest): string { export async function runCorpusScan(options: CorpusScanOptions): Promise { if (options.token === '') throw new Error('GITHUB_TOKEN is required (read-only public access)'); const sourceCommit = exactSourceCommit(options.sourceCommit); + const generatedAt = options.generatedAt ?? new Date().toISOString(); const limits = options.limits ?? DEFAULT_CORPUS_LIMITS; const fetchImpl = options.fetchImpl ?? fetch; const sampleMethod = options.sampleMethod ?? CORPUS_SAMPLE_METHOD; @@ -732,7 +771,7 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise = { schemaVersion: 1, - generatedAt: options.generatedAt ?? new Date().toISOString(), + generatedAt, sourceCommit, scannerSha256: sha256(readFileSync(scannerPath)), registrySha256: sha256(JSON.stringify(registryPayload)), @@ -757,7 +796,15 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise { outputDir: process.argv[3] ?? process.cwd(), token: process.env.GITHUB_TOKEN ?? '', sourceCommit: process.env.SCRIPTSPECT_SOURCE_COMMIT ?? process.env.GITHUB_SHA ?? '', + generatedAt: process.env.CORPUS_GENERATED_AT, sampleMethod: process.env.CORPUS_SAMPLE_METHOD, sampleSeed: process.env.CORPUS_SAMPLE_SEED, candidateSnapshotFile: process.env.CORPUS_CANDIDATE_SNAPSHOT, From 9738a0eab040a9770f15cde0f3f3185309dfc54f Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 10:09:54 +0800 Subject: [PATCH 5/9] fix(corpus): fail closed on replay drift --- docs/evidence/corpus-method.md | 18 +- tests/corpus/corpus-run.test.ts | 286 +++++++++++++++++++++++++++++++- tools/corpus-scan.ts | 121 +++++++++++--- 3 files changed, 395 insertions(+), 30 deletions(-) diff --git a/docs/evidence/corpus-method.md b/docs/evidence/corpus-method.md index 1dfffe1..e42db93 100644 --- a/docs/evidence/corpus-method.md +++ b/docs/evidence/corpus-method.md @@ -66,13 +66,19 @@ To replay a run, place its `repository-candidates.json`, `repository-sample.json`, and `repos.txt` beside the repository checkout, set `GITHUB_TOKEN` externally to a read-only public-repository token, and run the command recorded in `corpus-run.json`. The command checks out the exact source -commit, activates the pinned pnpm version, installs from the frozen lockfile, -and refuses to reuse its deterministic independent output directory. It binds -the original generation timestamp, sample method and seed, candidate snapshot, -sample evidence, repository list, and output directory to the scanner's actual -environment variables and positional arguments. Only evidence basenames and a +commit and fails unless HEAD, the index, and every tracked file are clean. Apart +from the three named evidence inputs, any nonignored untracked file is also a +failure. These Git checks run both before and after the frozen-lockfile install, +before the scanner starts. Replay requires the exact recorded Node version, +platform, and architecture; it restores the recorded `RUNNER_OS` value or +explicitly unsets it. It also binds the complete canonical limits JSON, original +generation timestamp, sample method and seed, candidate snapshot, sample +evidence, repository list, and a new deterministic output directory to the +scanner's actual environment variables and positional arguments. The command +refuses to reuse that output directory. Only evidence basenames and a token-variable reference are recorded; neither the credential nor a local -absolute path is persisted. +absolute path is persisted. A run recorded on another platform must therefore +be replayed in a matching environment with the recorded Node patch version. The run fails if any repository fails, while still leaving `corpus-run.json` for diagnosis. Truncation is visible and excluded rather than silently treated diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index 4d30d9b..b86b513 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -1,10 +1,11 @@ +import { execFileSync, spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; -import { DEFAULT_CORPUS_LIMITS, type TreeEntry } from '../../tools/corpus-lib'; -import { runCorpusScan } from '../../tools/corpus-scan'; +import { type CorpusLimits, DEFAULT_CORPUS_LIMITS, type TreeEntry } from '../../tools/corpus-lib'; +import { corpusScanOptionsFromCli, runCorpusScan } from '../../tools/corpus-scan'; const COMMIT = '0123456789abcdef0123456789abcdef01234567'; const SOURCE_COMMIT = '89abcdef0123456789abcdef0123456789abcdef'; @@ -232,6 +233,105 @@ function completeProvenance(data: ReturnType): { }; } +function git(directory: string, ...arguments_: string[]): string { + return execFileSync('git', ['-C', directory, ...arguments_], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +function posixShell(): string { + if (process.platform !== 'win32') return 'sh'; + const gitExecPath = execFileSync('git', ['--exec-path'], { encoding: 'utf8' }).trim(); + const gitRoot = dirname(dirname(dirname(gitExecPath))); + return join(gitRoot, 'bin', 'sh.exe'); +} + +async function replayFixture( + options: { limits?: CorpusLimits; runnerOs?: string } = {}, +): Promise<{ directory: string; reproduction: string }> { + const directory = temporaryDirectory(); + const outputRoot = temporaryDirectory(); + mkdirSync(join(directory, 'tools'), { recursive: true }); + writeFileSync(join(directory, 'tools', 'corpus-scan.ts'), 'export const committed = true;\n'); + writeFileSync(join(directory, 'package.json'), '{"packageManager":"pnpm@11.24.0"}\n'); + git(directory, 'init', '--quiet'); + git(directory, 'config', 'user.name', 'Corpus Replay Test'); + git(directory, 'config', 'user.email', 'corpus-replay@example.invalid'); + git(directory, 'config', 'core.autocrlf', 'false'); + git(directory, 'add', '--', 'tools/corpus-scan.ts', 'package.json'); + git(directory, 'commit', '--quiet', '-m', 'fixture'); + const sourceCommit = git(directory, 'rev-parse', 'HEAD'); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync( + sampleEvidenceFile, + `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`, + 'utf8', + ); + + const previousRunnerOs = process.env.RUNNER_OS; + if (options.runnerOs === undefined) delete process.env.RUNNER_OS; + else process.env.RUNNER_OS = options.runnerOs; + try { + const manifest = await runCorpusScan({ + inputFile, + outputDir: join(outputRoot, 'initial-output'), + token: 'read-only-test-token-must-not-be-persisted', + sourceCommit, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, data.blobs), + limits: options.limits, + sampleMethod: 'popularity-strata-round-robin-v1', + sampleSeed: 'candidate-seed', + candidateSnapshotFile, + sampleEvidenceFile, + }); + return { directory, reproduction: manifest.reproduction }; + } finally { + if (previousRunnerOs === undefined) delete process.env.RUNNER_OS; + else process.env.RUNNER_OS = previousRunnerOs; + } +} + +function executeReplay( + directory: string, + reproduction: string, + options: { shellSetup?: string; runnerOs?: string } = {}, +): ReturnType { + const script = [ + 'corepack() { return 0; }', + [ + 'pnpm() {', + ' if [ "$1" = "exec" ]; then', + ` printf "RUNNER_OS=%s\\n" "\${RUNNER_OS-}" > replay-observation.txt`, + ` printf "CORPUS_LIMITS_JSON=%s\\n" "\${CORPUS_LIMITS_JSON-}" >> replay-observation.txt`, + ' printf "ARGS=" >> replay-observation.txt', + ' printf "<%s>" "$@" >> replay-observation.txt', + ' printf "\\n" >> replay-observation.txt', + ' fi', + ' return 0', + '}', + ].join('\n'), + options.shellSetup ?? '', + reproduction, + ].join('\n'); + return spawnSync(posixShell(), ['-c', script], { + cwd: directory, + encoding: 'utf8', + env: { + ...process.env, + GITHUB_TOKEN: 'ephemeral-replay-test-token', + ...(options.runnerOs === undefined ? {} : { RUNNER_OS: options.runnerOs }), + }, + }); +} + describe('immutable corpus run evidence', () => { it('uses canonical workspace analysis while persisting hashes instead of script source', async () => { const directory = temporaryDirectory(); @@ -743,22 +843,198 @@ describe('immutable corpus run evidence', () => { }); const replayOutput = `corpus-reproduction-${SOURCE_COMMIT}`; + const cleanCheckout = [ + `test "$(git rev-parse --verify HEAD)" = '${SOURCE_COMMIT}'`, + 'git diff --quiet --', + 'git diff --cached --quiet --', + `test -z "$(git status --porcelain=v1 --untracked-files=all -- '.' ':(top,literal,exclude)repos copy.txt' ':(top,literal,exclude)repository candidate'"'"'s.json' ':(top,literal,exclude)repository sample;ignored.json')"`, + ]; expect(manifest.reproduction).toBe( [ `: "\${GITHUB_TOKEN:?set GITHUB_TOKEN to a read-only public-repository token}"`, `git -c advice.detachedHead=false checkout --detach '${SOURCE_COMMIT}'`, + ...cleanCheckout, + `test "$(node --version)" = '${process.version}'`, + `test "$(node -p 'process.platform')" = '${process.platform}'`, + `test "$(node -p 'process.arch')" = '${process.arch}'`, 'corepack enable', "corepack prepare 'pnpm@11.24.0' --activate", 'pnpm install --frozen-lockfile', + ...cleanCheckout, `test ! -e '${replayOutput}'`, `mkdir -- '${replayOutput}'`, - `SCRIPTSPECT_SOURCE_COMMIT='${SOURCE_COMMIT}' CORPUS_GENERATED_AT='2026-09-01T00:00:00.000Z' CORPUS_SAMPLE_METHOD='popularity-strata-round-robin-v1' CORPUS_SAMPLE_SEED='candidate-seed' CORPUS_CANDIDATE_SNAPSHOT='repository candidate'"'"'s.json' CORPUS_SAMPLE_EVIDENCE='repository sample;ignored.json' pnpm exec tsx tools/corpus-scan.ts 'repos copy.txt' '${replayOutput}'`, + 'unset RUNNER_OS', + `SCRIPTSPECT_SOURCE_COMMIT='${SOURCE_COMMIT}' CORPUS_GENERATED_AT='2026-09-01T00:00:00.000Z' CORPUS_SAMPLE_METHOD='popularity-strata-round-robin-v1' CORPUS_SAMPLE_SEED='candidate-seed' CORPUS_LIMITS_JSON='{"maxTreeEntries":20000,"maxManifests":500,"maxDepth":12,"maxFileBytes":1048576,"maxTotalBytes":10485760}' CORPUS_CANDIDATE_SNAPSHOT='repository candidate'"'"'s.json' CORPUS_SAMPLE_EVIDENCE='repository sample;ignored.json' pnpm exec tsx tools/corpus-scan.ts 'repos copy.txt' '${replayOutput}'`, ].join(' && '), ); expect(manifest.reproduction).not.toContain('read-only-test-token-must-not-be-persisted'); expect(manifest.reproduction).not.toContain(directory); }); + it('fails closed before scanning a checkout with dirty tracked, staged, or untracked files', async () => { + const cases = [ + { + name: 'tracked worktree', + dirty: (directory: string) => { + writeFileSync(join(directory, 'tools', 'corpus-scan.ts'), 'export const dirty = true;\n'); + }, + }, + { + name: 'staged index', + dirty: (directory: string) => { + writeFileSync( + join(directory, 'tools', 'corpus-scan.ts'), + 'export const staged = true;\n', + ); + git(directory, 'add', '--', 'tools/corpus-scan.ts'); + }, + }, + { + name: 'nonignored untracked code', + dirty: (directory: string) => { + writeFileSync(join(directory, 'rogue-config.ts'), 'export const rogue = true;\n'); + }, + }, + ]; + + for (const testCase of cases) { + const replay = await replayFixture(); + testCase.dirty(replay.directory); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, `${testCase.name}: ${result.stderr}`).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + } + }); + + it('fails closed before scanning under a different Node runtime', async () => { + const cases = [ + { name: 'version', version: 'v0.0.0', platform: process.platform, arch: process.arch }, + { name: 'platform', version: process.version, platform: 'foreign-os', arch: process.arch }, + { + name: 'architecture', + version: process.version, + platform: process.platform, + arch: 'foreign-arch', + }, + ]; + + for (const testCase of cases) { + const replay = await replayFixture(); + const shellSetup = [ + 'node() {', + ` if [ "$1" = "--version" ]; then printf '%s\\n' '${testCase.version}'`, + ` elif [ "$1" = "-p" ] && [ "$2" = "process.platform" ]; then printf '%s\\n' '${testCase.platform}'`, + ` elif [ "$1" = "-p" ] && [ "$2" = "process.arch" ]; then printf '%s\\n' '${testCase.arch}'`, + ' else return 64', + ' fi', + '}', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + + expect(result.status, `${testCase.name}: ${result.stderr}`).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + } + }); + + it('restores defined and unset RUNNER_OS state plus exact default and custom limits', async () => { + const customLimits: CorpusLimits = { + maxTotalBytes: 50_000, + maxFileBytes: 4_000, + maxDepth: 3, + maxManifests: 2, + maxTreeEntries: 100, + }; + const cases = [ + { + name: 'unset runner and default limits', + replay: await replayFixture(), + outerRunnerOs: 'ConflictingRunner', + expectedRunnerOs: '', + expectedLimits: DEFAULT_CORPUS_LIMITS, + }, + { + name: 'defined runner and custom limits', + replay: await replayFixture({ limits: customLimits, runnerOs: 'RecordedRunner' }), + outerRunnerOs: 'ConflictingRunner', + expectedRunnerOs: 'RecordedRunner', + expectedLimits: customLimits, + }, + ]; + + for (const testCase of cases) { + const result = executeReplay(testCase.replay.directory, testCase.replay.reproduction, { + runnerOs: testCase.outerRunnerOs, + }); + + expect(result.status, `${testCase.name}: ${result.stderr}`).toBe(0); + const observation = readFileSync( + join(testCase.replay.directory, 'replay-observation.txt'), + 'utf8', + ); + expect(observation).toContain(`RUNNER_OS=${testCase.expectedRunnerOs}\n`); + expect(observation).toContain( + `CORPUS_LIMITS_JSON=${JSON.stringify({ + maxTreeEntries: testCase.expectedLimits.maxTreeEntries, + maxManifests: testCase.expectedLimits.maxManifests, + maxDepth: testCase.expectedLimits.maxDepth, + maxFileBytes: testCase.expectedLimits.maxFileBytes, + maxTotalBytes: testCase.expectedLimits.maxTotalBytes, + })}\n`, + ); + expect(observation).toContain( + ``, + ); + expect(testCase.replay.reproduction).not.toContain('ephemeral-replay-test-token'); + expect(testCase.replay.reproduction).not.toContain(testCase.replay.directory); + } + }); + + it('parses default and exact custom limits from the scanner CLI environment', () => { + const baseEnvironment = { + GITHUB_TOKEN: 'read-only-test-token', + SCRIPTSPECT_SOURCE_COMMIT: SOURCE_COMMIT, + }; + const defaults = corpusScanOptionsFromCli(['repos.txt', 'default-output'], baseEnvironment); + expect(defaults).toMatchObject({ + inputFile: 'repos.txt', + outputDir: 'default-output', + limits: DEFAULT_CORPUS_LIMITS, + }); + + const customJson = + '{"maxTotalBytes":50000,"maxDepth":3,"maxTreeEntries":100,"maxFileBytes":4000,"maxManifests":2}'; + const custom = corpusScanOptionsFromCli(['repos.txt', 'custom-output'], { + ...baseEnvironment, + CORPUS_LIMITS_JSON: customJson, + }); + expect(JSON.stringify(custom.limits)).toBe( + '{"maxTreeEntries":100,"maxManifests":2,"maxDepth":3,"maxFileBytes":4000,"maxTotalBytes":50000}', + ); + + for (const malformed of [ + '{}', + '[]', + '{"maxTreeEntries":1,"maxManifests":2,"maxDepth":3,"maxFileBytes":4,"maxTotalBytes":5,"extra":6}', + '{"maxTreeEntries":1,"maxManifests":2,"maxDepth":-1,"maxFileBytes":4,"maxTotalBytes":5}', + '{"maxTreeEntries":1,"maxManifests":2,"maxDepth":3.5,"maxFileBytes":4,"maxTotalBytes":5}', + 'not-json', + ]) { + expect(() => + corpusScanOptionsFromCli(['repos.txt'], { + ...baseEnvironment, + CORPUS_LIMITS_JSON: malformed, + }), + ).toThrow(/CORPUS_LIMITS_JSON/); + } + }); + it('rejects mismatched sample provenance before making a network request', async () => { const data = fixture(); const provenance = completeProvenance(data); diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index 0992c73..bf9b061 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -39,6 +39,13 @@ import { const GITHUB_API = 'https://api.github.com'; const GITHUB_RAW = 'https://raw.githubusercontent.com'; +const CORPUS_LIMIT_KEYS = [ + 'maxTreeEntries', + 'maxManifests', + 'maxDepth', + 'maxFileBytes', + 'maxTotalBytes', +] as const satisfies readonly (keyof CorpusLimits)[]; interface GitHubTreeResponse { tree?: TreeEntry[]; @@ -139,21 +146,78 @@ function posixShellQuote(value: string): string { return `'${value.replaceAll("'", `'"'"'`)}'`; } +function normalizeCorpusLimits(value: unknown, source: string): CorpusLimits { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`${source} must be an object with the complete corpus limit contract`); + } + const record = value as Record; + const actualKeys = Object.keys(record).sort(); + const expectedKeys = [...CORPUS_LIMIT_KEYS].sort(); + if (JSON.stringify(actualKeys) !== JSON.stringify(expectedKeys)) { + throw new Error(`${source} must contain exactly ${CORPUS_LIMIT_KEYS.join(', ')}`); + } + for (const key of CORPUS_LIMIT_KEYS) { + if (!Number.isSafeInteger(record[key]) || (record[key] as number) < 0) { + throw new Error(`${source}.${key} must be a non-negative safe integer`); + } + } + return { + maxTreeEntries: record.maxTreeEntries as number, + maxManifests: record.maxManifests as number, + maxDepth: record.maxDepth as number, + maxFileBytes: record.maxFileBytes as number, + maxTotalBytes: record.maxTotalBytes as number, + }; +} + +function corpusLimitsFromEnvironment(value: string | undefined): CorpusLimits { + if (value === undefined) return normalizeCorpusLimits(DEFAULT_CORPUS_LIMITS, 'default limits'); + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error('CORPUS_LIMITS_JSON must be valid JSON'); + } + return normalizeCorpusLimits(parsed, 'CORPUS_LIMITS_JSON'); +} + function reproductionCommand(options: { sourceCommit: string; generatedAt: string; sampleMethod: string; sampleSeed: string; + environment: CorpusRunManifest['environment']; + limits: CorpusLimits; inputFile: string; candidateSnapshotFile?: string; sampleEvidenceFile?: string; }): string { const outputDirectory = `corpus-reproduction-${options.sourceCommit}`; + const evidenceFiles = [ + basename(options.inputFile), + ...(options.candidateSnapshotFile === undefined + ? [] + : [basename(options.candidateSnapshotFile)]), + ...(options.sampleEvidenceFile === undefined ? [] : [basename(options.sampleEvidenceFile)]), + ]; + const cleanStatusPathspec = [ + '.', + ...new Set(evidenceFiles.map((file) => `:(top,literal,exclude)${file}`)), + ] + .map(posixShellQuote) + .join(' '); + const cleanCheckout = [ + `test "$(git rev-parse --verify HEAD)" = ${posixShellQuote(options.sourceCommit)}`, + 'git diff --quiet --', + 'git diff --cached --quiet --', + `test -z "$(git status --porcelain=v1 --untracked-files=all -- ${cleanStatusPathspec})"`, + ]; const environment = [ `SCRIPTSPECT_SOURCE_COMMIT=${posixShellQuote(options.sourceCommit)}`, `CORPUS_GENERATED_AT=${posixShellQuote(options.generatedAt)}`, `CORPUS_SAMPLE_METHOD=${posixShellQuote(options.sampleMethod)}`, `CORPUS_SAMPLE_SEED=${posixShellQuote(options.sampleSeed)}`, + `CORPUS_LIMITS_JSON=${posixShellQuote(JSON.stringify(options.limits))}`, ...(options.candidateSnapshotFile === undefined ? [] : [`CORPUS_CANDIDATE_SNAPSHOT=${posixShellQuote(basename(options.candidateSnapshotFile))}`]), @@ -164,11 +228,19 @@ function reproductionCommand(options: { return [ `: "\${GITHUB_TOKEN:?set GITHUB_TOKEN to a read-only public-repository token}"`, `git -c advice.detachedHead=false checkout --detach ${posixShellQuote(options.sourceCommit)}`, + ...cleanCheckout, + `test "$(node --version)" = ${posixShellQuote(options.environment.node)}`, + `test "$(node -p 'process.platform')" = ${posixShellQuote(options.environment.platform)}`, + `test "$(node -p 'process.arch')" = ${posixShellQuote(options.environment.arch)}`, 'corepack enable', "corepack prepare 'pnpm@11.24.0' --activate", 'pnpm install --frozen-lockfile', + ...cleanCheckout, `test ! -e ${posixShellQuote(outputDirectory)}`, `mkdir -- ${posixShellQuote(outputDirectory)}`, + options.environment.runnerOs === undefined + ? 'unset RUNNER_OS' + : `export RUNNER_OS=${posixShellQuote(options.environment.runnerOs)}`, `${environment.join(' ')} pnpm exec tsx tools/corpus-scan.ts ${posixShellQuote(basename(options.inputFile))} ${posixShellQuote(outputDirectory)}`, ].join(' && '); } @@ -620,7 +692,13 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise { - const inputFile = process.argv[2]; +export function corpusScanOptionsFromCli( + arguments_: readonly string[], + environment: NodeJS.ProcessEnv, +): CorpusScanOptions { + const inputFile = arguments_[0]; if (inputFile === undefined) { throw new Error('usage: tsx tools/corpus-scan.ts repos.txt [output-directory]'); } - await runCorpusScan({ + return { inputFile, - outputDir: process.argv[3] ?? process.cwd(), - token: process.env.GITHUB_TOKEN ?? '', - sourceCommit: process.env.SCRIPTSPECT_SOURCE_COMMIT ?? process.env.GITHUB_SHA ?? '', - generatedAt: process.env.CORPUS_GENERATED_AT, - sampleMethod: process.env.CORPUS_SAMPLE_METHOD, - sampleSeed: process.env.CORPUS_SAMPLE_SEED, - candidateSnapshotFile: process.env.CORPUS_CANDIDATE_SNAPSHOT, - sampleEvidenceFile: process.env.CORPUS_SAMPLE_EVIDENCE, - }); + outputDir: arguments_[1] ?? process.cwd(), + token: environment.GITHUB_TOKEN ?? '', + sourceCommit: environment.SCRIPTSPECT_SOURCE_COMMIT ?? environment.GITHUB_SHA ?? '', + generatedAt: environment.CORPUS_GENERATED_AT, + limits: corpusLimitsFromEnvironment(environment.CORPUS_LIMITS_JSON), + sampleMethod: environment.CORPUS_SAMPLE_METHOD, + sampleSeed: environment.CORPUS_SAMPLE_SEED, + candidateSnapshotFile: environment.CORPUS_CANDIDATE_SNAPSHOT, + sampleEvidenceFile: environment.CORPUS_SAMPLE_EVIDENCE, + }; +} + +async function main(): Promise { + await runCorpusScan(corpusScanOptionsFromCli(process.argv.slice(2), process.env)); } if ( From 063ef176092e8f8f87a3764708fc0b79cb60d10f Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 10:20:22 +0800 Subject: [PATCH 6/9] fix(corpus): reject hidden replay index state --- docs/evidence/corpus-method.md | 28 +++++++++++++++------------- tests/corpus/corpus-run.test.ts | 19 +++++++++++++++++++ tools/corpus-scan.ts | 2 ++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/docs/evidence/corpus-method.md b/docs/evidence/corpus-method.md index e42db93..4d87074 100644 --- a/docs/evidence/corpus-method.md +++ b/docs/evidence/corpus-method.md @@ -66,19 +66,21 @@ To replay a run, place its `repository-candidates.json`, `repository-sample.json`, and `repos.txt` beside the repository checkout, set `GITHUB_TOKEN` externally to a read-only public-repository token, and run the command recorded in `corpus-run.json`. The command checks out the exact source -commit and fails unless HEAD, the index, and every tracked file are clean. Apart -from the three named evidence inputs, any nonignored untracked file is also a -failure. These Git checks run both before and after the frozen-lockfile install, -before the scanner starts. Replay requires the exact recorded Node version, -platform, and architecture; it restores the recorded `RUNNER_OS` value or -explicitly unsets it. It also binds the complete canonical limits JSON, original -generation timestamp, sample method and seed, candidate snapshot, sample -evidence, repository list, and a new deterministic output directory to the -scanner's actual environment variables and positional arguments. The command -refuses to reuse that output directory. Only evidence basenames and a -token-variable reference are recorded; neither the credential nor a local -absolute path is persisted. A run recorded on another platform must therefore -be replayed in a matching environment with the recorded Node patch version. +commit and fails unless HEAD, the index, and every tracked file are clean. Every +tracked-index tag other than ordinary `H` is rejected, including Git's +`assume-unchanged` and `skip-worktree` hiding flags. Apart from the three named +evidence inputs, any nonignored untracked file is also a failure. These Git +checks run both before and after the frozen-lockfile install, before the scanner +starts. Replay requires the exact recorded Node version, platform, and +architecture; it restores the recorded `RUNNER_OS` value or explicitly unsets +it. It also binds the complete canonical limits JSON, original generation +timestamp, sample method and seed, candidate snapshot, sample evidence, +repository list, and a new deterministic output directory to the scanner's +actual environment variables and positional arguments. The command refuses to +reuse that output directory. Only evidence basenames and a token-variable +reference are recorded; neither the credential nor a local absolute path is +persisted. A run recorded on another platform must therefore be replayed in a +matching environment with the recorded Node patch version. The run fails if any repository fails, while still leaving `corpus-run.json` for diagnosis. Truncation is visible and excluded rather than silently treated diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index b86b513..416ce7a 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -845,6 +845,8 @@ describe('immutable corpus run evidence', () => { const replayOutput = `corpus-reproduction-${SOURCE_COMMIT}`; const cleanCheckout = [ `test "$(git rev-parse --verify HEAD)" = '${SOURCE_COMMIT}'`, + 'git ls-files -v >/dev/null', + 'git ls-files -v | while IFS= read -r entry; do case "$entry" in H\\ *) ;; *) exit 1 ;; esac; done', 'git diff --quiet --', 'git diff --cached --quiet --', `test -z "$(git status --porcelain=v1 --untracked-files=all -- '.' ':(top,literal,exclude)repos copy.txt' ':(top,literal,exclude)repository candidate'"'"'s.json' ':(top,literal,exclude)repository sample;ignored.json')"`, @@ -908,6 +910,23 @@ describe('immutable corpus run evidence', () => { } }); + it.each([ + ['assume-unchanged', '--assume-unchanged'], + ['skip-worktree', '--skip-worktree'], + ])('fails closed when %s hides a dirty tracked scanner', async (_name, indexFlag) => { + const replay = await replayFixture(); + git(replay.directory, 'update-index', indexFlag, '--', 'tools/corpus-scan.ts'); + writeFileSync( + join(replay.directory, 'tools', 'corpus-scan.ts'), + 'export const hiddenDirtyScanner = true;\n', + ); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + it('fails closed before scanning under a different Node runtime', async () => { const cases = [ { name: 'version', version: 'v0.0.0', platform: process.platform, arch: process.arch }, diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index bf9b061..306b93b 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -208,6 +208,8 @@ function reproductionCommand(options: { .join(' '); const cleanCheckout = [ `test "$(git rev-parse --verify HEAD)" = ${posixShellQuote(options.sourceCommit)}`, + 'git ls-files -v >/dev/null', + 'git ls-files -v | while IFS= read -r entry; do case "$entry" in H\\ *) ;; *) exit 1 ;; esac; done', 'git diff --quiet --', 'git diff --cached --quiet --', `test -z "$(git status --porcelain=v1 --untracked-files=all -- ${cleanStatusPathspec})"`, From 842c43d11eee1fdf8bdb37cc42232f9b01b41d4e Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 10:37:41 +0800 Subject: [PATCH 7/9] fix(corpus): verify replay worktree bytes --- docs/evidence/corpus-method.md | 20 +++- tests/corpus/corpus-run.test.ts | 157 ++++++++++++++++++++++++++------ tools/corpus-replay-check.mjs | 154 +++++++++++++++++++++++++++++++ tools/corpus-scan.ts | 19 ++-- 4 files changed, 305 insertions(+), 45 deletions(-) create mode 100644 tools/corpus-replay-check.mjs diff --git a/docs/evidence/corpus-method.md b/docs/evidence/corpus-method.md index 4d87074..27c47ad 100644 --- a/docs/evidence/corpus-method.md +++ b/docs/evidence/corpus-method.md @@ -67,10 +67,22 @@ To replay a run, place its `repository-candidates.json`, `GITHUB_TOKEN` externally to a read-only public-repository token, and run the command recorded in `corpus-run.json`. The command checks out the exact source commit and fails unless HEAD, the index, and every tracked file are clean. Every -tracked-index tag other than ordinary `H` is rejected, including Git's -`assume-unchanged` and `skip-worktree` hiding flags. Apart from the three named -evidence inputs, any nonignored untracked file is also a failure. These Git -checks run both before and after the frozen-lockfile install, before the scanner +tracked-index tag other than ordinary `H` is rejected. The preflight reads +separate NUL-delimited `git ls-files -v` and `git ls-files -f` results so Git's +`assume-unchanged`, `skip-worktree`, and `fsmonitor-valid` hiding flags cannot +mask a changed file. The validator source is embedded once in the reproduction +command rather than loaded from the checkout it is validating. It parses the +raw NUL-delimited `HEAD` tree and hashes every regular file's worktree bytes +with the repository's Git object algorithm; clean/smudge filters, EOL or +working-tree encodings, symlink emulation, racy stat data, and hidden index bits +therefore cannot substitute different bytes. POSIX executable bits and raw +symlink targets must match their tree modes. Gitlinks are rejected rather than +trusted as submodules. Git is invoked without an interpolating shell, and +NUL-delimited output plus literal pathspec arguments preserve unusual evidence +filenames. Working-tree, cached, and status checks explicitly do not ignore +submodules; any Git or filesystem failure is fatal. Apart from the three named +evidence inputs, any nonignored untracked file is also a failure. These checks +run both before and after the frozen-lockfile install, before the scanner starts. Replay requires the exact recorded Node version, platform, and architecture; it restores the recorded `RUNNER_OS` value or explicitly unsets it. It also binds the complete canonical limits JSON, original generation diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index 416ce7a..0a06b77 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -1,6 +1,16 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; @@ -248,19 +258,62 @@ function posixShell(): string { } async function replayFixture( - options: { limits?: CorpusLimits; runnerOs?: string } = {}, + options: { gitlink?: boolean; limits?: CorpusLimits; runnerOs?: string } = {}, ): Promise<{ directory: string; reproduction: string }> { const directory = temporaryDirectory(); const outputRoot = temporaryDirectory(); mkdirSync(join(directory, 'tools'), { recursive: true }); writeFileSync(join(directory, 'tools', 'corpus-scan.ts'), 'export const committed = true;\n'); + copyFileSync( + join(process.cwd(), 'tools', 'corpus-replay-check.mjs'), + join(directory, 'tools', 'corpus-replay-check.mjs'), + ); writeFileSync(join(directory, 'package.json'), '{"packageManager":"pnpm@11.24.0"}\n'); + writeFileSync(join(directory, 'README.md'), 'committed replay fixture\n'); + writeFileSync(join(directory, '.gitattributes'), 'filtered.txt filter=replay-clean\n'); + writeFileSync(join(directory, 'filtered.txt'), 'canonical\n'); + writeFileSync(join(directory, "special ' [x] ;.txt"), 'special filename\n'); + writeFileSync(join(directory, 'executable.sh'), '#!/bin/sh\nexit 0\n'); + if (process.platform !== 'win32') { + chmodSync(join(directory, 'executable.sh'), 0o755); + symlinkSync('README.md', join(directory, 'readme-link')); + } git(directory, 'init', '--quiet'); git(directory, 'config', 'user.name', 'Corpus Replay Test'); git(directory, 'config', 'user.email', 'corpus-replay@example.invalid'); git(directory, 'config', 'core.autocrlf', 'false'); - git(directory, 'add', '--', 'tools/corpus-scan.ts', 'package.json'); + git(directory, 'config', 'filter.replay-clean.clean', "sed 's/.*/canonical/'"); + git(directory, 'config', 'filter.replay-clean.smudge', 'cat'); + git( + directory, + 'add', + '--', + 'tools/corpus-scan.ts', + 'tools/corpus-replay-check.mjs', + 'package.json', + 'README.md', + '.gitattributes', + 'filtered.txt', + "special ' [x] ;.txt", + 'executable.sh', + ...(process.platform === 'win32' ? [] : ['readme-link']), + ); git(directory, 'commit', '--quiet', '-m', 'fixture'); + if (options.gitlink === true) { + const submodule = join(directory, 'vendor', 'submodule'); + execFileSync('git', ['clone', '--quiet', '--no-hardlinks', directory, submodule], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + const gitlinkCommit = git(submodule, 'rev-parse', 'HEAD'); + git( + directory, + 'update-index', + '--add', + '--cacheinfo', + `160000,${gitlinkCommit},vendor/submodule`, + ); + git(directory, 'commit', '--quiet', '-m', 'add gitlink fixture'); + } const sourceCommit = git(directory, 'rev-parse', 'HEAD'); const inputFile = join(directory, 'repos.txt'); const candidateSnapshotFile = join(directory, 'repository-candidates.json'); @@ -842,32 +895,14 @@ describe('immutable corpus run evidence', () => { sampleEvidenceFile, }); - const replayOutput = `corpus-reproduction-${SOURCE_COMMIT}`; - const cleanCheckout = [ - `test "$(git rev-parse --verify HEAD)" = '${SOURCE_COMMIT}'`, - 'git ls-files -v >/dev/null', - 'git ls-files -v | while IFS= read -r entry; do case "$entry" in H\\ *) ;; *) exit 1 ;; esac; done', - 'git diff --quiet --', - 'git diff --cached --quiet --', - `test -z "$(git status --porcelain=v1 --untracked-files=all -- '.' ':(top,literal,exclude)repos copy.txt' ':(top,literal,exclude)repository candidate'"'"'s.json' ':(top,literal,exclude)repository sample;ignored.json')"`, - ]; - expect(manifest.reproduction).toBe( - [ - `: "\${GITHUB_TOKEN:?set GITHUB_TOKEN to a read-only public-repository token}"`, - `git -c advice.detachedHead=false checkout --detach '${SOURCE_COMMIT}'`, - ...cleanCheckout, - `test "$(node --version)" = '${process.version}'`, - `test "$(node -p 'process.platform')" = '${process.platform}'`, - `test "$(node -p 'process.arch')" = '${process.arch}'`, - 'corepack enable', - "corepack prepare 'pnpm@11.24.0' --activate", - 'pnpm install --frozen-lockfile', - ...cleanCheckout, - `test ! -e '${replayOutput}'`, - `mkdir -- '${replayOutput}'`, - 'unset RUNNER_OS', - `SCRIPTSPECT_SOURCE_COMMIT='${SOURCE_COMMIT}' CORPUS_GENERATED_AT='2026-09-01T00:00:00.000Z' CORPUS_SAMPLE_METHOD='popularity-strata-round-robin-v1' CORPUS_SAMPLE_SEED='candidate-seed' CORPUS_LIMITS_JSON='{"maxTreeEntries":20000,"maxManifests":500,"maxDepth":12,"maxFileBytes":1048576,"maxTotalBytes":10485760}' CORPUS_CANDIDATE_SNAPSHOT='repository candidate'"'"'s.json' CORPUS_SAMPLE_EVIDENCE='repository sample;ignored.json' pnpm exec tsx tools/corpus-scan.ts 'repos copy.txt' '${replayOutput}'`, - ].join(' && '), + const preflight = `node --input-type=module -e "$SCRIPTSPECT_REPLAY_CHECK" -- '${SOURCE_COMMIT}' 'repos copy.txt' 'repository candidate'"'"'s.json' 'repository sample;ignored.json'`; + expect(manifest.reproduction).toMatch(/^SCRIPTSPECT_REPLAY_CHECK='/u); + expect(manifest.reproduction.split(preflight)).toHaveLength(3); + expect(manifest.reproduction).toContain( + `CORPUS_LIMITS_JSON='{"maxTreeEntries":20000,"maxManifests":500,"maxDepth":12,"maxFileBytes":1048576,"maxTotalBytes":10485760}'`, + ); + expect(manifest.reproduction).toContain( + `pnpm exec tsx tools/corpus-scan.ts 'repos copy.txt' 'corpus-reproduction-${SOURCE_COMMIT}'`, ); expect(manifest.reproduction).not.toContain('read-only-test-token-must-not-be-persisted'); expect(manifest.reproduction).not.toContain(directory); @@ -908,7 +943,7 @@ describe('immutable corpus run evidence', () => { expect(result.status, `${testCase.name}: ${result.stderr}`).not.toBe(0); expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); } - }); + }, 15_000); it.each([ ['assume-unchanged', '--assume-unchanged'], @@ -927,6 +962,68 @@ describe('immutable corpus run evidence', () => { expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); }); + it('fails closed when fsmonitor-valid hides a dirty tracked file', async () => { + const replay = await replayFixture(); + git(replay.directory, 'config', 'core.fsmonitor', 'true'); + writeFileSync(join(replay.directory, 'README.md'), 'dirty content hidden by fsmonitor\n'); + git(replay.directory, 'update-index', '--fsmonitor-valid', '--', 'README.md'); + expect(git(replay.directory, 'ls-files', '-v', '--', 'README.md')).toBe('H README.md'); + expect(git(replay.directory, 'ls-files', '-f', '--', 'README.md')).toBe('h README.md'); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('fails closed when a clean filter hides different worktree bytes', async () => { + const replay = await replayFixture(); + writeFileSync(join(replay.directory, 'filtered.txt'), 'malicious worktree bytes\n'); + git(replay.directory, 'add', '--', 'filtered.txt'); + git(replay.directory, 'diff', '--quiet', '--', 'filtered.txt'); + git(replay.directory, 'diff', '--cached', '--quiet', '--', 'filtered.txt'); + expect(git(replay.directory, 'status', '--porcelain=v1', '--', 'filtered.txt')).toBe(''); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('propagates a Git status failure instead of treating it as clean', async () => { + const replay = await replayFixture(); + git(replay.directory, 'config', 'status.showUntrackedFiles', 'invalid-mode'); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('rejects a gitlink even when its checked-out submodule is clean', async () => { + const replay = await replayFixture({ gitlink: true }); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it.skipIf(process.platform === 'win32')( + 'fails closed when core.filemode hides an executable-bit mismatch', + async () => { + const replay = await replayFixture(); + git(replay.directory, 'config', 'core.filemode', 'false'); + chmodSync(join(replay.directory, 'executable.sh'), 0o644); + git(replay.directory, 'diff', '--quiet', '--', 'executable.sh'); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }, + ); + it('fails closed before scanning under a different Node runtime', async () => { const cases = [ { name: 'version', version: 'v0.0.0', platform: process.platform, arch: process.arch }, diff --git a/tools/corpus-replay-check.mjs b/tools/corpus-replay-check.mjs new file mode 100644 index 0000000..6536208 --- /dev/null +++ b/tools/corpus-replay-check.mjs @@ -0,0 +1,154 @@ +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { lstatSync, readFileSync, readlinkSync } from 'node:fs'; +import { basename, dirname, isAbsolute } from 'node:path'; + +const MAX_GIT_OUTPUT_BYTES = 64 * 1024 * 1024; + +function fail(message) { + throw new Error(`corpus replay preflight failed: ${message}`); +} + +function git(arguments_) { + try { + return execFileSync('git', arguments_, { + encoding: null, + maxBuffer: MAX_GIT_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch { + fail(`git ${arguments_[0] ?? 'command'} did not complete successfully`); + } +} + +function nulRecords(output, description) { + const records = []; + let recordStart = 0; + for (let index = 0; index < output.length; index += 1) { + if (output[index] !== 0) continue; + records.push(output.subarray(recordStart, index)); + recordStart = index + 1; + } + if (recordStart !== output.length) fail(`${description} was not NUL terminated`); + return records; +} + +function validateIndexTags(flag, description) { + for (const record of nulRecords(git(['ls-files', '-z', flag, '--']), description)) { + if (record.length < 3 || record[0] !== 0x48 || record[1] !== 0x20) { + fail(`${description} contains a nonordinary tracked index tag`); + } + } +} + +function validateEvidenceBasenames(values) { + if (values.length === 0) fail('at least one evidence basename is required'); + for (const value of values) { + if ( + value === '' || + value === '.' || + value === '..' || + value.includes('\0') || + isAbsolute(value) || + dirname(value) !== '.' || + basename(value) !== value + ) { + fail('evidence inputs must be root-level basenames'); + } + } +} + +function worktreeBytes(mode, path) { + let stat; + try { + stat = lstatSync(path); + } catch { + fail('a tracked worktree path could not be inspected'); + } + + if (mode === '120000') { + if (!stat.isSymbolicLink()) fail('a tracked symlink is not a worktree symlink'); + try { + return readlinkSync(path, { encoding: 'buffer' }); + } catch { + fail('a tracked symlink target could not be read'); + } + } + + if (mode !== '100644' && mode !== '100755') fail('unsupported tracked blob mode'); + if (!stat.isFile()) fail('a tracked blob is not a regular worktree file'); + if (process.platform !== 'win32') { + const executable = (stat.mode & 0o111) !== 0; + if (executable !== (mode === '100755')) fail('tracked executable mode differs from HEAD'); + } + try { + return readFileSync(path); + } catch { + fail('a tracked worktree file could not be read'); + } +} + +function validateRawHeadTree() { + const tree = git(['ls-tree', '-r', '-z', '--full-tree', 'HEAD']); + for (const record of nulRecords(tree, 'HEAD tree')) { + const tab = record.indexOf(0x09); + if (tab <= 0 || tab === record.length - 1) fail('HEAD tree record is malformed'); + const metadata = record.subarray(0, tab).toString('ascii').split(' '); + if (metadata.length !== 3) fail('HEAD tree metadata is malformed'); + const [mode, type, oid] = metadata; + if (mode === '160000' || type !== 'blob') fail('gitlinks and non-blob entries are unsupported'); + if (oid === undefined || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(oid)) { + fail('HEAD tree object ID is malformed'); + } + const path = record.subarray(tab + 1); + const bytes = worktreeBytes(mode, path); + const algorithm = oid.length === 40 ? 'sha1' : 'sha256'; + const actual = createHash(algorithm) + .update(Buffer.from(`blob ${bytes.length}\0`, 'ascii')) + .update(bytes) + .digest('hex'); + if (actual !== oid) fail('tracked worktree bytes differ from HEAD'); + } +} + +function main() { + const [sourceCommit, ...evidenceBasenames] = process.argv.slice(1); + if (sourceCommit === undefined || !/^[a-f0-9]{40}$/u.test(sourceCommit)) { + fail('source commit must be an exact 40-character lowercase SHA'); + } + validateEvidenceBasenames(evidenceBasenames); + + const head = git(['rev-parse', '--verify', 'HEAD']).toString('ascii').trim(); + if (head !== sourceCommit) fail('HEAD does not match the recorded source commit'); + + validateIndexTags('-v', 'assume-unchanged/skip-worktree check'); + validateIndexTags('-f', 'fsmonitor-valid check'); + validateRawHeadTree(); + git(['diff', '--quiet', '--ignore-submodules=none', '--']); + git(['diff', '--cached', '--quiet', '--ignore-submodules=none', '--']); + + const statusPathspec = [ + '.', + ...new Set(evidenceBasenames.map((value) => `:(top,literal,exclude)${value}`)), + ]; + const unexpectedStatus = git([ + 'status', + '--porcelain=v1', + '-z', + '--untracked-files=all', + '--ignore-submodules=none', + '--', + ...statusPathspec, + ]); + if (unexpectedStatus.length !== 0) { + fail('checkout contains tracked changes or non-evidence untracked files'); + } +} + +try { + main(); +} catch (error) { + const message = error instanceof Error ? error.message : 'unknown preflight failure'; + console.error(message); + process.exitCode = 1; +} diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index 306b93b..a2bbf17 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -39,6 +39,10 @@ import { const GITHUB_API = 'https://api.github.com'; const GITHUB_RAW = 'https://raw.githubusercontent.com'; +const CORPUS_REPLAY_CHECK_SOURCE = readFileSync( + fileURLToPath(new URL('./corpus-replay-check.mjs', import.meta.url)), + 'utf8', +); const CORPUS_LIMIT_KEYS = [ 'maxTreeEntries', 'maxManifests', @@ -200,19 +204,11 @@ function reproductionCommand(options: { : [basename(options.candidateSnapshotFile)]), ...(options.sampleEvidenceFile === undefined ? [] : [basename(options.sampleEvidenceFile)]), ]; - const cleanStatusPathspec = [ - '.', - ...new Set(evidenceFiles.map((file) => `:(top,literal,exclude)${file}`)), - ] + const replayCheckArguments = [options.sourceCommit, ...new Set(evidenceFiles)] .map(posixShellQuote) .join(' '); const cleanCheckout = [ - `test "$(git rev-parse --verify HEAD)" = ${posixShellQuote(options.sourceCommit)}`, - 'git ls-files -v >/dev/null', - 'git ls-files -v | while IFS= read -r entry; do case "$entry" in H\\ *) ;; *) exit 1 ;; esac; done', - 'git diff --quiet --', - 'git diff --cached --quiet --', - `test -z "$(git status --porcelain=v1 --untracked-files=all -- ${cleanStatusPathspec})"`, + `node --input-type=module -e "$SCRIPTSPECT_REPLAY_CHECK" -- ${replayCheckArguments}`, ]; const environment = [ `SCRIPTSPECT_SOURCE_COMMIT=${posixShellQuote(options.sourceCommit)}`, @@ -228,12 +224,13 @@ function reproductionCommand(options: { : [`CORPUS_SAMPLE_EVIDENCE=${posixShellQuote(basename(options.sampleEvidenceFile))}`]), ]; return [ + `SCRIPTSPECT_REPLAY_CHECK=${posixShellQuote(CORPUS_REPLAY_CHECK_SOURCE)}`, `: "\${GITHUB_TOKEN:?set GITHUB_TOKEN to a read-only public-repository token}"`, `git -c advice.detachedHead=false checkout --detach ${posixShellQuote(options.sourceCommit)}`, - ...cleanCheckout, `test "$(node --version)" = ${posixShellQuote(options.environment.node)}`, `test "$(node -p 'process.platform')" = ${posixShellQuote(options.environment.platform)}`, `test "$(node -p 'process.arch')" = ${posixShellQuote(options.environment.arch)}`, + ...cleanCheckout, 'corepack enable', "corepack prepare 'pnpm@11.24.0' --activate", 'pnpm install --frozen-lockfile', From 94fe973d862a30e7303ef06a61a9aaf4a0ca2e31 Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 11:24:23 +0800 Subject: [PATCH 8/9] fix(corpus): harden replay evidence boundary --- docs/evidence/corpus-method.md | 84 ++++--- tests/corpus/corpus-run.test.ts | 411 +++++++++++++++++++++++++++++++- tools/corpus-replay-check.mjs | 166 ++++++++++--- tools/corpus-scan.ts | 294 +++++++++++++++++++---- 4 files changed, 836 insertions(+), 119 deletions(-) diff --git a/docs/evidence/corpus-method.md b/docs/evidence/corpus-method.md index 27c47ad..c9cf0b5 100644 --- a/docs/evidence/corpus-method.md +++ b/docs/evidence/corpus-method.md @@ -63,36 +63,62 @@ The workflow artifact contains: - `summary.md`: an explicitly unverified summary for maintainers. To replay a run, place its `repository-candidates.json`, -`repository-sample.json`, and `repos.txt` beside the repository checkout, set -`GITHUB_TOKEN` externally to a read-only public-repository token, and run the -command recorded in `corpus-run.json`. The command checks out the exact source -commit and fails unless HEAD, the index, and every tracked file are clean. Every -tracked-index tag other than ordinary `H` is rejected. The preflight reads -separate NUL-delimited `git ls-files -v` and `git ls-files -f` results so Git's -`assume-unchanged`, `skip-worktree`, and `fsmonitor-valid` hiding flags cannot -mask a changed file. The validator source is embedded once in the reproduction -command rather than loaded from the checkout it is validating. It parses the +`repository-sample.json`, and `repos.txt` as nonignored, untracked regular files +at the root of a **fresh dedicated checkout already positioned at the recorded +commit**. Their basenames must be safe and unique and cannot collide with +`.git`, `node_modules`, `findings.jsonl`, `summary.md`, `corpus-run.json`, or the +deterministic replay output directory. Before the original scan makes a network +request, it also requires its source checkout's `HEAD` to equal the recorded +source commit and compares the evidence basenames with that commit's raw root +tree. A basename already tracked there is rejected instead of emitting an +intrinsically unreplayable command. Set `GITHUB_TOKEN` externally to a read-only +public-repository token, then run the command recorded in `corpus-run.json`. +The command never invokes `git checkout`: a different HEAD fails closed and no +smudge/process filter or `post-checkout` hook gets an opportunity to run first. + +The validator source is gzip/base64-embedded once in the reproduction command +and decoded with Node's built-in `node:zlib` into a data-module; it is never +loaded from the checkout being validated. Every Git subprocess forces +`core.fsmonitor=false` and `core.hooksPath=/dev/null`, disables replacement +objects, binds the worktree to the current checkout, and discards inherited Git +repository/index/object/config redirection variables. The validator parses the raw NUL-delimited `HEAD` tree and hashes every regular file's worktree bytes -with the repository's Git object algorithm; clean/smudge filters, EOL or -working-tree encodings, symlink emulation, racy stat data, and hidden index bits -therefore cannot substitute different bytes. POSIX executable bits and raw -symlink targets must match their tree modes. Gitlinks are rejected rather than -trusted as submodules. Git is invoked without an interpolating shell, and -NUL-delimited output plus literal pathspec arguments preserve unusual evidence -filenames. Working-tree, cached, and status checks explicitly do not ignore -submodules; any Git or filesystem failure is fatal. Apart from the three named -evidence inputs, any nonignored untracked file is also a failure. These checks -run both before and after the frozen-lockfile install, before the scanner -starts. Replay requires the exact recorded Node version, platform, and -architecture; it restores the recorded `RUNNER_OS` value or explicitly unsets -it. It also binds the complete canonical limits JSON, original generation -timestamp, sample method and seed, candidate snapshot, sample evidence, -repository list, and a new deterministic output directory to the scanner's -actual environment variables and positional arguments. The command refuses to -reuse that output directory. Only evidence basenames and a token-variable -reference are recorded; neither the credential nor a local absolute path is -persisted. A run recorded on another platform must therefore be replayed in a -matching environment with the recorded Node patch version. +with the repository's Git object algorithm. It separately parses +NUL-delimited `git ls-files --stage` records and requires every path, mode, blob +OID, and stage to match the `HEAD` tree exactly. This direct comparison makes +`assume-unchanged`, `skip-worktree`, `fsmonitor-valid`, clean/smudge filters, +EOL or working-tree encodings, and racy stat data unable to substitute other +bytes without invoking `git diff` or `git status`. POSIX executable bits and +raw symlink targets must match their tree modes. Gitlinks are rejected rather +than trusted as submodules. A NUL-delimited +`git ls-files --others --exclude-standard` enumeration must contain exactly the +three named evidence inputs; any other nonignored untracked file or any Git or +filesystem failure is fatal. Raw path buffers preserve unusual filenames. + +Before **and** after the frozen-lockfile install, each evidence input must still +be a regular file with the exact SHA-256 recorded by the original manifest, and +the tracked worktree/index/untracked checks must all pass. `node_modules` must +not exist before package tooling starts (including as a broken link), while the +second gate permits the frozen install's ignored `node_modules` output. The +complete replay runs in a POSIX subshell. It first disables inherited shell +`allexport`, copies `GITHUB_TOKEN` into a non-exported shell variable, and +removes the credential from the environment before Git, Corepack, and pnpm +install run; the token is injected as a +single-command environment assignment +only for the final scanner invocation. Caller environment values therefore +survive the replay unchanged. + +Replay also requires the exact recorded Node version, platform, and +architecture; inside its subshell it restores the recorded `RUNNER_OS` value or +explicitly unsets it. It binds the complete canonical limits JSON, original +generation timestamp, sample method and seed, candidate snapshot, sample +evidence, repository list, and a new deterministic output directory to the +scanner's actual environment variables and positional arguments. The command +refuses to reuse that output directory. Only evidence basenames and their +digests plus a token-variable reference are recorded; neither the credential +nor a local absolute path is persisted. A run recorded on another platform +must therefore be replayed in a matching environment with the recorded Node +patch version. The run fails if any repository fails, while still leaving `corpus-run.json` for diagnosis. Truncation is visible and excluded rather than silently treated diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index 0a06b77..6387c0c 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -18,7 +18,7 @@ import { type CorpusLimits, DEFAULT_CORPUS_LIMITS, type TreeEntry } from '../../ import { corpusScanOptionsFromCli, runCorpusScan } from '../../tools/corpus-scan'; const COMMIT = '0123456789abcdef0123456789abcdef01234567'; -const SOURCE_COMMIT = '89abcdef0123456789abcdef0123456789abcdef'; +const SOURCE_COMMIT = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); const temporaryDirectories: string[] = []; function temporaryDirectory(): string { @@ -270,6 +270,7 @@ async function replayFixture( ); writeFileSync(join(directory, 'package.json'), '{"packageManager":"pnpm@11.24.0"}\n'); writeFileSync(join(directory, 'README.md'), 'committed replay fixture\n'); + writeFileSync(join(directory, '.gitignore'), 'node_modules/\n'); writeFileSync(join(directory, '.gitattributes'), 'filtered.txt filter=replay-clean\n'); writeFileSync(join(directory, 'filtered.txt'), 'canonical\n'); writeFileSync(join(directory, "special ' [x] ;.txt"), 'special filename\n'); @@ -292,6 +293,7 @@ async function replayFixture( 'tools/corpus-replay-check.mjs', 'package.json', 'README.md', + '.gitignore', '.gitattributes', 'filtered.txt', "special ' [x] ;.txt", @@ -344,6 +346,7 @@ async function replayFixture( sampleSeed: 'candidate-seed', candidateSnapshotFile, sampleEvidenceFile, + sourceCheckout: directory, }); return { directory, reproduction: manifest.reproduction }; } finally { @@ -355,7 +358,12 @@ async function replayFixture( function executeReplay( directory: string, reproduction: string, - options: { shellSetup?: string; runnerOs?: string } = {}, + options: { + shellSetup?: string; + shellAfter?: string; + runnerOs?: string; + environment?: NodeJS.ProcessEnv; + } = {}, ): ReturnType { const script = [ 'corepack() { return 0; }', @@ -373,12 +381,14 @@ function executeReplay( ].join('\n'), options.shellSetup ?? '', reproduction, + options.shellAfter ?? '', ].join('\n'); return spawnSync(posixShell(), ['-c', script], { cwd: directory, encoding: 'utf8', env: { ...process.env, + ...options.environment, GITHUB_TOKEN: 'ephemeral-replay-test-token', ...(options.runnerOs === undefined ? {} : { RUNNER_OS: options.runnerOs }), }, @@ -895,9 +905,19 @@ describe('immutable corpus run evidence', () => { sampleEvidenceFile, }); - const preflight = `node --input-type=module -e "$SCRIPTSPECT_REPLAY_CHECK" -- '${SOURCE_COMMIT}' 'repos copy.txt' 'repository candidate'"'"'s.json' 'repository sample;ignored.json'`; - expect(manifest.reproduction).toMatch(/^SCRIPTSPECT_REPLAY_CHECK='/u); - expect(manifest.reproduction.split(preflight)).toHaveLength(3); + expect(manifest.reproduction).toMatch(/^\(set \+a && SCRIPTSPECT_REPLAY_CHECK='/u); + expect(manifest.reproduction.split('node --input-type=module -e')).toHaveLength(3); + for (const digest of [ + manifest.inputSha256, + manifest.sampling.candidateSnapshotSha256, + manifest.sampling.sampleEvidenceSha256, + ]) { + expect(digest).toMatch(/^[a-f0-9]{64}$/u); + expect(manifest.reproduction.split(String(digest))).toHaveLength(3); + } + expect(manifest.reproduction).toContain('test ! -e node_modules'); + expect(manifest.reproduction).not.toMatch(/\bgit\b[^&]*\bcheckout\b/u); + expect(manifest.reproduction).not.toContain('tools/corpus-replay-check.mjs'); expect(manifest.reproduction).toContain( `CORPUS_LIMITS_JSON='{"maxTreeEntries":20000,"maxManifests":500,"maxDepth":12,"maxFileBytes":1048576,"maxTotalBytes":10485760}'`, ); @@ -908,6 +928,212 @@ describe('immutable corpus run evidence', () => { expect(manifest.reproduction).not.toContain(directory); }); + it('does not execute a local clean filter while validating a replay checkout', async () => { + const replay = await replayFixture(); + const candidateEvidence = join(replay.directory, 'repository-candidates.json'); + const originalEvidence = readFileSync(candidateEvidence); + git( + replay.directory, + 'config', + 'filter.replay-clean.clean', + 'sh -c \'printf tampered > repository-candidates.json; cat >/dev/null; printf \\"canonical\\n\\"\'', + ); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).toBe(0); + expect(readFileSync(candidateEvidence)).toEqual(originalEvidence); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(true); + }); + + it('requires callers to prepare the exact commit without invoking checkout hooks', async () => { + const replay = await replayFixture(); + const hook = join(replay.directory, '.git', 'hooks', 'post-checkout'); + writeFileSync(hook, '#!/bin/sh\nprintf hook-ran > checkout-hook-ran\n'); + chmodSync(hook, 0o755); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).toBe(0); + expect(existsSync(join(replay.directory, 'checkout-hook-ran'))).toBe(false); + expect(replay.reproduction).not.toMatch(/\bgit\b[^&]*\bcheckout\b/u); + }); + + it('fails closed on a different clean HEAD without changing the caller checkout', async () => { + const replay = await replayFixture(); + writeFileSync(join(replay.directory, 'later-commit.txt'), 'later clean commit\n'); + git(replay.directory, 'add', '--', 'later-commit.txt'); + git(replay.directory, 'commit', '--quiet', '-m', 'later fixture commit'); + const laterHead = git(replay.directory, 'rev-parse', 'HEAD'); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(git(replay.directory, 'rev-parse', 'HEAD')).toBe(laterHead); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('keeps replay environment changes inside a subshell', async () => { + const replay = await replayFixture(); + + const result = executeReplay(replay.directory, replay.reproduction, { + shellAfter: `printf "GITHUB_TOKEN=%s\\nRUNNER_OS=%s\\n" "\${GITHUB_TOKEN-}" "\${RUNNER_OS-}" > caller-environment.txt`, + runnerOs: 'CallerRunner', + }); + + expect(result.status, String(result.stderr)).toBe(0); + expect(readFileSync(join(replay.directory, 'caller-environment.txt'), 'utf8')).toBe( + 'GITHUB_TOKEN=ephemeral-replay-test-token\nRUNNER_OS=CallerRunner\n', + ); + }); + + it('rejects a pre-existing dependency install before running package tooling', async () => { + const replay = await replayFixture(); + mkdirSync(join(replay.directory, 'node_modules', 'malicious'), { recursive: true }); + writeFileSync( + join(replay.directory, 'node_modules', 'malicious', 'index.js'), + 'throw new Error("pre-existing dependency executed");\n', + ); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('rejects a broken node_modules link before running package tooling', async () => { + const replay = await replayFixture(); + const nodeModules = join(replay.directory, 'node_modules'); + if (process.platform === 'win32') { + symlinkSync(join(replay.directory, 'missing-dependency-target'), nodeModules, 'junction'); + } else { + symlinkSync('missing-dependency-target', nodeModules); + } + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('allows the frozen install to create an ignored dependency directory', async () => { + const replay = await replayFixture(); + const shellSetup = [ + 'pnpm() {', + ' if [ "$1" = "install" ]; then', + ' mkdir -p node_modules/installed', + ' printf installed > node_modules/installed/package.json', + ' elif [ "$1" = "exec" ]; then', + ' printf scanner-ran > replay-observation.txt', + ' fi', + ' return 0', + '}', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + + expect(result.status, String(result.stderr)).toBe(0); + expect(readFileSync(join(replay.directory, 'replay-observation.txt'), 'utf8')).toBe( + 'scanner-ran', + ); + }); + + it('withholds the token from package setup and injects it only into the scanner command', async () => { + const replay = await replayFixture(); + const shellSetup = [ + 'set -a', + 'pnpm() {', + ' if [ "$1" = "install" ]; then', + ' if env | grep -q "^GITHUB_TOKEN=" || env | grep -q "^SCRIPTSPECT_REPLAY_TOKEN="; then', + ' printf leaked > package-setup-token-leak.txt', + ' return 97', + ' fi', + ' elif [ "$1" = "exec" ]; then', + ' env | grep "^GITHUB_TOKEN=" > scanner-token.txt', + ' fi', + ' return 0', + '}', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + + expect(result.status, String(result.stderr)).toBe(0); + expect(existsSync(join(replay.directory, 'package-setup-token-leak.txt'))).toBe(false); + expect(readFileSync(join(replay.directory, 'scanner-token.txt'), 'utf8')).toBe( + 'GITHUB_TOKEN=ephemeral-replay-test-token\n', + ); + }); + + it('ignores inherited Git repository redirection while validating the checkout', async () => { + const replay = await replayFixture(); + const redirectedWorktree = temporaryDirectory(); + for (const evidenceName of [ + 'repos.txt', + 'repository-candidates.json', + 'repository-sample.json', + ]) { + copyFileSync(join(replay.directory, evidenceName), join(redirectedWorktree, evidenceName)); + } + writeFileSync( + join(replay.directory, 'unexpected-untracked.ts'), + 'export const hidden = true;\n', + ); + + const result = executeReplay(replay.directory, replay.reproduction, { + environment: { GIT_WORK_TREE: redirectedWorktree }, + }); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it.each(['repos.txt', 'repository-candidates.json', 'repository-sample.json'])( + 'rejects changed %s evidence before the frozen install', + async (evidenceName) => { + const replay = await replayFixture(); + writeFileSync(join(replay.directory, evidenceName), `tampered ${evidenceName}\n`); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }, + ); + + it.each(['repos.txt', 'repository-candidates.json', 'repository-sample.json'])( + 'rejects lifecycle changes to %s evidence after the frozen install', + async (evidenceName) => { + const replay = await replayFixture(); + const shellSetup = [ + 'pnpm() {', + ' if [ "$1" = "install" ]; then', + ` printf tampered > '${evidenceName}'`, + ' elif [ "$1" = "exec" ]; then', + ' printf scanner-ran > replay-observation.txt', + ' fi', + ' return 0', + '}', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }, + ); + + it('rejects a non-regular evidence input before the frozen install', async () => { + const replay = await replayFixture(); + const evidencePath = join(replay.directory, 'repository-sample.json'); + rmSync(evidencePath); + mkdirSync(evidencePath); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + it('fails closed before scanning a checkout with dirty tracked, staged, or untracked files', async () => { const cases = [ { @@ -990,9 +1216,9 @@ describe('immutable corpus run evidence', () => { expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); }); - it('propagates a Git status failure instead of treating it as clean', async () => { + it('propagates a Git metadata failure instead of treating it as clean', async () => { const replay = await replayFixture(); - git(replay.directory, 'config', 'status.showUntrackedFiles', 'invalid-mode'); + git(replay.directory, 'config', 'core.repositoryFormatVersion', '999'); const result = executeReplay(replay.directory, replay.reproduction); @@ -1424,4 +1650,175 @@ describe('immutable corpus run evidence', () => { ).rejects.toThrow(/candidate snapshot and sample evidence must be provided together/); expect(networkCalled).toBe(false); }); + + it('rejects evidence paths that collapse to duplicate replay basenames before network access', async () => { + const directory = temporaryDirectory(); + const inputDirectory = join(directory, 'input'); + const candidateDirectory = join(directory, 'candidate'); + const sampleDirectory = join(directory, 'sample'); + mkdirSync(inputDirectory); + mkdirSync(candidateDirectory); + mkdirSync(sampleDirectory); + const inputFile = join(inputDirectory, 'evidence.json'); + const candidateSnapshotFile = join(candidateDirectory, 'evidence.json'); + const sampleEvidenceFile = join(sampleDirectory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`); + let networkCalled = false; + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/unique.*basename|basename.*unique/iu); + expect(networkCalled).toBe(false); + }); + + it('rejects an evidence basename tracked at the recorded source commit before network access', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'package.json'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + const sourceCommit = git(process.cwd(), 'rev-parse', 'HEAD'); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`); + let networkCalled = false; + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/basename.*tracked.*source commit|tracked.*source commit.*basename/iu); + expect(networkCalled).toBe(false); + }); + + it.each([ + ['repository list', 'findings.jsonl', 'repository-candidates.json', 'repository-sample.json'], + ['candidate snapshot', 'repos.txt', 'summary.md', 'repository-sample.json'], + ['sample evidence', 'repos.txt', 'repository-candidates.json', 'corpus-run.json'], + [ + 'repository list', + `corpus-reproduction-${SOURCE_COMMIT}`, + 'repository-candidates.json', + 'repository-sample.json', + ], + ['repository list', 'node_modules', 'repository-candidates.json', 'repository-sample.json'], + ])( + 'rejects a %s basename reserved for corpus output before network access', + async (_role, inputName, candidateName, sampleName) => { + const directory = temporaryDirectory(); + const inputFile = join(directory, inputName); + const candidateSnapshotFile = join(directory, candidateName); + const sampleEvidenceFile = join(directory, sampleName); + const data = fixture(); + const provenance = completeProvenance(data); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`); + let networkCalled = false; + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/reserved.*corpus output|corpus output.*reserved/iu); + expect(networkCalled).toBe(false); + }, + ); + + it('applies replay basename collision checks to CLI-derived options', () => { + const environment = { + GITHUB_TOKEN: 'read-only-test-token', + SCRIPTSPECT_SOURCE_COMMIT: SOURCE_COMMIT, + CORPUS_CANDIDATE_SNAPSHOT: 'candidate/evidence.json', + CORPUS_SAMPLE_EVIDENCE: 'sample/evidence.json', + }; + expect(() => corpusScanOptionsFromCli(['input/repos.txt'], environment)).toThrow( + /unique.*basename|basename.*unique/iu, + ); + expect(() => + corpusScanOptionsFromCli(['input/findings.jsonl'], { + ...environment, + CORPUS_CANDIDATE_SNAPSHOT: 'repository-candidates.json', + CORPUS_SAMPLE_EVIDENCE: 'repository-sample.json', + }), + ).toThrow(/reserved.*corpus output|corpus output.*reserved/iu); + }); + + it.each(['repository list', 'candidate snapshot', 'sample evidence'])( + 'requires the %s input to be a regular file before network access', + async (role) => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + const paths = new Map([ + ['repository list', inputFile], + ['candidate snapshot', candidateSnapshotFile], + ['sample evidence', sampleEvidenceFile], + ]); + for (const [currentRole, path] of paths) { + if (currentRole === role) mkdirSync(path); + else if (currentRole === 'repository list') { + writeFileSync(path, `example/project@${COMMIT}\n`); + } else if (currentRole === 'candidate snapshot') { + writeFileSync(path, provenance.candidateSnapshot); + } else { + writeFileSync(path, `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`); + } + } + let networkCalled = false; + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(new RegExp(`${role}.*regular file`, 'iu')); + expect(networkCalled).toBe(false); + }, + ); }); diff --git a/tools/corpus-replay-check.mjs b/tools/corpus-replay-check.mjs index 6536208..1609e72 100644 --- a/tools/corpus-replay-check.mjs +++ b/tools/corpus-replay-check.mjs @@ -4,15 +4,55 @@ import { lstatSync, readFileSync, readlinkSync } from 'node:fs'; import { basename, dirname, isAbsolute } from 'node:path'; const MAX_GIT_OUTPUT_BYTES = 64 * 1024 * 1024; +const GIT_SAFE_CONFIG = ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null']; +const GIT_REDIRECT_ENVIRONMENT = new Set([ + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_CEILING_DIRECTORIES', + 'GIT_COMMON_DIR', + 'GIT_CONFIG', + 'GIT_CONFIG_COUNT', + 'GIT_CONFIG_GLOBAL', + 'GIT_CONFIG_NOSYSTEM', + 'GIT_CONFIG_PARAMETERS', + 'GIT_CONFIG_SYSTEM', + 'GIT_DIR', + 'GIT_DISCOVERY_ACROSS_FILESYSTEM', + 'GIT_INDEX_FILE', + 'GIT_NAMESPACE', + 'GIT_NO_REPLACE_OBJECTS', + 'GIT_OBJECT_DIRECTORY', + 'GIT_PREFIX', + 'GIT_QUARANTINE_PATH', + 'GIT_REPLACE_REF_BASE', + 'GIT_SHALLOW_FILE', + 'GIT_WORK_TREE', +]); function fail(message) { throw new Error(`corpus replay preflight failed: ${message}`); } +function gitEnvironment(worktree) { + const environment = { ...process.env }; + for (const key of Object.keys(environment)) { + const normalized = key.toUpperCase(); + if ( + GIT_REDIRECT_ENVIRONMENT.has(normalized) || + /^GIT_CONFIG_(?:KEY|VALUE)_\d+$/u.test(normalized) + ) { + delete environment[key]; + } + } + environment.GIT_NO_REPLACE_OBJECTS = '1'; + environment.GIT_WORK_TREE = worktree; + return environment; +} + function git(arguments_) { try { - return execFileSync('git', arguments_, { + return execFileSync('git', [...GIT_SAFE_CONFIG, ...arguments_], { encoding: null, + env: gitEnvironment(process.cwd()), maxBuffer: MAX_GIT_OUTPUT_BYTES, stdio: ['ignore', 'pipe', 'pipe'], }); @@ -33,18 +73,17 @@ function nulRecords(output, description) { return records; } -function validateIndexTags(flag, description) { - for (const record of nulRecords(git(['ls-files', '-z', flag, '--']), description)) { - if (record.length < 3 || record[0] !== 0x48 || record[1] !== 0x20) { - fail(`${description} contains a nonordinary tracked index tag`); - } +function validateEvidenceArguments(values) { + if (values.length === 0 || values.length % 2 !== 0) { + fail('evidence basenames and SHA-256 digests must be paired'); } -} - -function validateEvidenceBasenames(values) { - if (values.length === 0) fail('at least one evidence basename is required'); - for (const value of values) { + const evidence = []; + const seen = new Set(); + for (let index = 0; index < values.length; index += 2) { + const value = values[index]; + const digest = values[index + 1]; if ( + value === undefined || value === '' || value === '.' || value === '..' || @@ -55,6 +94,33 @@ function validateEvidenceBasenames(values) { ) { fail('evidence inputs must be root-level basenames'); } + if (seen.has(value)) fail('evidence basenames must be unique'); + if (digest === undefined || !/^[a-f0-9]{64}$/u.test(digest)) { + fail('evidence SHA-256 digest was invalid'); + } + seen.add(value); + evidence.push({ name: value, sha256: digest }); + } + return evidence; +} + +function validateEvidenceFiles(evidence) { + for (const expected of evidence) { + let stat; + try { + stat = lstatSync(expected.name); + } catch { + fail('an evidence input could not be inspected'); + } + if (!stat.isFile()) fail('evidence inputs must be regular files'); + let bytes; + try { + bytes = readFileSync(expected.name); + } catch { + fail('an evidence input could not be read'); + } + const actual = createHash('sha256').update(bytes).digest('hex'); + if (actual !== expected.sha256) fail('evidence input bytes differ from the recorded run'); } } @@ -89,6 +155,7 @@ function worktreeBytes(mode, path) { } function validateRawHeadTree() { + const entries = new Map(); const tree = git(['ls-tree', '-r', '-z', '--full-tree', 'HEAD']); for (const record of nulRecords(tree, 'HEAD tree')) { const tab = record.indexOf(0x09); @@ -101,6 +168,8 @@ function validateRawHeadTree() { fail('HEAD tree object ID is malformed'); } const path = record.subarray(tab + 1); + const pathKey = path.toString('hex'); + if (entries.has(pathKey)) fail('HEAD tree contains a duplicate path'); const bytes = worktreeBytes(mode, path); const algorithm = oid.length === 40 ? 'sha1' : 'sha256'; const actual = createHash(algorithm) @@ -108,41 +177,68 @@ function validateRawHeadTree() { .update(bytes) .digest('hex'); if (actual !== oid) fail('tracked worktree bytes differ from HEAD'); + entries.set(pathKey, { mode, oid }); + } + return entries; +} + +function validateIndexAgainstHead(headEntries) { + const indexEntries = new Map(); + const index = git(['ls-files', '--stage', '-z', '--full-name', '--']); + for (const record of nulRecords(index, 'index')) { + const tab = record.indexOf(0x09); + if (tab <= 0 || tab === record.length - 1) fail('index record is malformed'); + const metadata = record.subarray(0, tab).toString('ascii').split(' '); + if (metadata.length !== 3) fail('index metadata is malformed'); + const [mode, oid, stage] = metadata; + if ( + stage !== '0' || + (mode !== '100644' && mode !== '100755' && mode !== '120000') || + oid === undefined || + !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(oid) + ) { + fail('index entry is unsupported or malformed'); + } + const pathKey = record.subarray(tab + 1).toString('hex'); + if (indexEntries.has(pathKey)) fail('index contains a duplicate path'); + indexEntries.set(pathKey, { mode, oid }); + } + if (indexEntries.size !== headEntries.size) fail('index tree differs from HEAD'); + for (const [pathKey, expected] of headEntries) { + const actual = indexEntries.get(pathKey); + if (actual?.mode !== expected.mode || actual.oid !== expected.oid) { + fail('index tree differs from HEAD'); + } + } +} + +function validateUntrackedEvidence(evidence) { + const expectedPaths = new Set(evidence.map((value) => Buffer.from(value.name).toString('hex'))); + const untracked = git(['ls-files', '--others', '--exclude-standard', '-z', '--full-name', '--']); + for (const path of nulRecords(untracked, 'untracked file list')) { + if (!expectedPaths.delete(path.toString('hex'))) { + fail('checkout contains a non-evidence untracked file'); + } + } + if (expectedPaths.size !== 0) { + fail('evidence inputs must be nonignored untracked root files'); } } function main() { - const [sourceCommit, ...evidenceBasenames] = process.argv.slice(1); + const [sourceCommit, ...evidenceArguments] = process.argv.slice(1); if (sourceCommit === undefined || !/^[a-f0-9]{40}$/u.test(sourceCommit)) { fail('source commit must be an exact 40-character lowercase SHA'); } - validateEvidenceBasenames(evidenceBasenames); + const evidence = validateEvidenceArguments(evidenceArguments); + validateEvidenceFiles(evidence); const head = git(['rev-parse', '--verify', 'HEAD']).toString('ascii').trim(); if (head !== sourceCommit) fail('HEAD does not match the recorded source commit'); - validateIndexTags('-v', 'assume-unchanged/skip-worktree check'); - validateIndexTags('-f', 'fsmonitor-valid check'); - validateRawHeadTree(); - git(['diff', '--quiet', '--ignore-submodules=none', '--']); - git(['diff', '--cached', '--quiet', '--ignore-submodules=none', '--']); - - const statusPathspec = [ - '.', - ...new Set(evidenceBasenames.map((value) => `:(top,literal,exclude)${value}`)), - ]; - const unexpectedStatus = git([ - 'status', - '--porcelain=v1', - '-z', - '--untracked-files=all', - '--ignore-submodules=none', - '--', - ...statusPathspec, - ]); - if (unexpectedStatus.length !== 0) { - fail('checkout contains tracked changes or non-evidence untracked files'); - } + const headEntries = validateRawHeadTree(); + validateIndexAgainstHead(headEntries); + validateUntrackedEvidence(evidence); } try { diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index a2bbf17..f33b247 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -5,10 +5,12 @@ * package manifests are downloaded, scripts are never executed, and raw * script source is never written to evidence artifacts. */ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; import { type AnalysisResult, analyze } from '../src/core/analyze'; import { DEFAULT_TARGETS } from '../src/core/targets'; import { RULES } from '../src/rules'; @@ -39,10 +41,40 @@ import { const GITHUB_API = 'https://api.github.com'; const GITHUB_RAW = 'https://raw.githubusercontent.com'; +const CORPUS_SOURCE_CHECKOUT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const MAX_GIT_OUTPUT_BYTES = 16 * 1024 * 1024; +const GIT_SAFE_CONFIG = ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null']; +const GIT_REDIRECT_ENVIRONMENT = new Set([ + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_CEILING_DIRECTORIES', + 'GIT_COMMON_DIR', + 'GIT_CONFIG', + 'GIT_CONFIG_COUNT', + 'GIT_CONFIG_GLOBAL', + 'GIT_CONFIG_NOSYSTEM', + 'GIT_CONFIG_PARAMETERS', + 'GIT_CONFIG_SYSTEM', + 'GIT_DIR', + 'GIT_DISCOVERY_ACROSS_FILESYSTEM', + 'GIT_INDEX_FILE', + 'GIT_NAMESPACE', + 'GIT_NO_REPLACE_OBJECTS', + 'GIT_OBJECT_DIRECTORY', + 'GIT_PREFIX', + 'GIT_QUARANTINE_PATH', + 'GIT_REPLACE_REF_BASE', + 'GIT_SHALLOW_FILE', + 'GIT_WORK_TREE', +]); const CORPUS_REPLAY_CHECK_SOURCE = readFileSync( fileURLToPath(new URL('./corpus-replay-check.mjs', import.meta.url)), 'utf8', ); +const CORPUS_REPLAY_CHECK_GZIP_BASE64 = gzipSync(Buffer.from(CORPUS_REPLAY_CHECK_SOURCE), { + level: 9, +}).toString('base64'); +const CORPUS_REPLAY_BOOTSTRAP = + 'const encoded = process.argv.splice(1, 1)[0]; if (encoded === undefined) throw new Error("missing replay check"); const { gunzipSync } = await import("node:zlib"); await import("data:text/javascript;base64," + gunzipSync(Buffer.from(encoded, "base64")).toString("base64"));'; const CORPUS_LIMIT_KEYS = [ 'maxTreeEntries', 'maxManifests', @@ -50,6 +82,26 @@ const CORPUS_LIMIT_KEYS = [ 'maxFileBytes', 'maxTotalBytes', ] as const satisfies readonly (keyof CorpusLimits)[]; +const CORPUS_RESERVED_BASENAMES = new Set([ + '.git', + 'node_modules', + 'findings.jsonl', + 'summary.md', + 'corpus-run.json', +]); + +type CorpusEvidenceRole = 'repository list' | 'candidate snapshot' | 'sample evidence'; + +interface CorpusEvidencePath { + role: CorpusEvidenceRole; + path: string; + name: string; +} + +interface CorpusEvidenceInput extends CorpusEvidencePath { + bytes: Buffer; + sha256: string; +} interface GitHubTreeResponse { tree?: TreeEntry[]; @@ -134,6 +186,7 @@ export interface CorpusScanOptions { sampleSeed?: string; candidateSnapshotFile?: string; sampleEvidenceFile?: string; + sourceCheckout?: string; fetchImpl?: typeof fetch; } @@ -150,6 +203,144 @@ function posixShellQuote(value: string): string { return `'${value.replaceAll("'", `'"'"'`)}'`; } +function corpusFilenameKey(value: string | Buffer): string { + const bytes = typeof value === 'string' ? Buffer.from(value) : value; + if (process.platform !== 'win32' && process.platform !== 'darwin') { + return bytes.toString('hex'); + } + let decoded: string; + try { + decoded = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error('recorded source commit contains a filename unsupported on this platform'); + } + return decoded.normalize('NFC').toLocaleLowerCase('en-US'); +} + +function sourceGit(sourceCheckout: string, arguments_: string[], description: string): Buffer { + const environment = { ...process.env }; + for (const key of Object.keys(environment)) { + const normalized = key.toUpperCase(); + if ( + GIT_REDIRECT_ENVIRONMENT.has(normalized) || + /^GIT_CONFIG_(?:KEY|VALUE)_\d+$/u.test(normalized) + ) { + delete environment[key]; + } + } + environment.GIT_NO_REPLACE_OBJECTS = '1'; + environment.GIT_WORK_TREE = sourceCheckout; + try { + return execFileSync('git', [...GIT_SAFE_CONFIG, '-C', sourceCheckout, ...arguments_], { + encoding: null, + env: environment, + maxBuffer: MAX_GIT_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch { + throw new Error(`recorded source checkout ${description} could not be verified`); + } +} + +function nulRecords(output: Buffer, description: string): Buffer[] { + if (output.length === 0) return []; + if (output.at(-1) !== 0) throw new Error(`${description} was not NUL terminated`); + const records: Buffer[] = []; + let start = 0; + for (let index = 0; index < output.length; index += 1) { + if (output[index] !== 0) continue; + if (index === start) throw new Error(`${description} contained an empty filename`); + records.push(output.subarray(start, index)); + start = index + 1; + } + return records; +} + +function validateEvidenceAgainstSourceTree( + evidence: CorpusEvidencePath[], + sourceCommit: string, + sourceCheckout = CORPUS_SOURCE_CHECKOUT, +): void { + const checkout = resolve(sourceCheckout); + const head = sourceGit(checkout, ['rev-parse', '--verify', 'HEAD^{commit}'], 'HEAD') + .toString('ascii') + .trim(); + if (head !== sourceCommit) { + throw new Error('recorded source checkout HEAD does not match the source commit'); + } + const trackedRootKeys = new Set( + nulRecords( + sourceGit(checkout, ['ls-tree', '-z', '--name-only', sourceCommit], 'root tree'), + 'recorded source root tree', + ).map(corpusFilenameKey), + ); + for (const input of evidence) { + if (trackedRootKeys.has(corpusFilenameKey(input.name))) { + throw new Error(`${input.role} basename is tracked at the recorded source commit root`); + } + } +} + +function corpusEvidencePaths( + options: Pick< + CorpusScanOptions, + 'inputFile' | 'candidateSnapshotFile' | 'sampleEvidenceFile' | 'sourceCommit' + >, +): CorpusEvidencePath[] { + if ( + (options.candidateSnapshotFile === undefined) !== + (options.sampleEvidenceFile === undefined) + ) { + throw new Error('candidate snapshot and sample evidence must be provided together'); + } + const paths: Array<{ role: CorpusEvidenceRole; path: string | undefined }> = [ + { role: 'repository list', path: options.inputFile }, + { role: 'candidate snapshot', path: options.candidateSnapshotFile }, + { role: 'sample evidence', path: options.sampleEvidenceFile }, + ]; + const reserved = new Set(CORPUS_RESERVED_BASENAMES); + if (/^[a-f0-9]{40}$/u.test(options.sourceCommit)) { + reserved.add(`corpus-reproduction-${options.sourceCommit}`); + } + const reservedKeys = new Set([...reserved].map(corpusFilenameKey)); + const seen = new Set(); + const evidence: CorpusEvidencePath[] = []; + for (const entry of paths) { + if (entry.path === undefined) continue; + const name = entry.path.includes('\0') ? '' : basename(entry.path); + if (name === '' || name === '.' || name === '..') { + throw new Error(`${entry.role} must have a safe replay basename`); + } + const key = corpusFilenameKey(name); + if (reservedKeys.has(key)) { + throw new Error(`${entry.role} basename is reserved for corpus output or replay state`); + } + if (seen.has(key)) { + throw new Error('corpus evidence inputs must have unique replay basenames'); + } + seen.add(key); + evidence.push({ role: entry.role, path: entry.path, name }); + } + return evidence; +} + +function readCorpusEvidence(input: CorpusEvidencePath): CorpusEvidenceInput { + let stat: ReturnType; + try { + stat = lstatSync(input.path); + } catch { + throw new Error(`${input.role} could not be inspected`); + } + if (!stat.isFile()) throw new Error(`${input.role} must be a regular file`); + let bytes: Buffer; + try { + bytes = readFileSync(input.path); + } catch { + throw new Error(`${input.role} could not be read`); + } + return { ...input, bytes, sha256: sha256(bytes) }; +} + function normalizeCorpusLimits(value: unknown, source: string): CorpusLimits { if (typeof value !== 'object' || value === null || Array.isArray(value)) { throw new Error(`${source} must be an object with the complete corpus limit contract`); @@ -192,23 +383,24 @@ function reproductionCommand(options: { sampleSeed: string; environment: CorpusRunManifest['environment']; limits: CorpusLimits; - inputFile: string; - candidateSnapshotFile?: string; - sampleEvidenceFile?: string; + input: CorpusEvidenceInput; + candidateSnapshot?: CorpusEvidenceInput; + sampleEvidence?: CorpusEvidenceInput; }): string { const outputDirectory = `corpus-reproduction-${options.sourceCommit}`; - const evidenceFiles = [ - basename(options.inputFile), - ...(options.candidateSnapshotFile === undefined - ? [] - : [basename(options.candidateSnapshotFile)]), - ...(options.sampleEvidenceFile === undefined ? [] : [basename(options.sampleEvidenceFile)]), + const evidence = [ + options.input, + ...(options.candidateSnapshot === undefined ? [] : [options.candidateSnapshot]), + ...(options.sampleEvidence === undefined ? [] : [options.sampleEvidence]), ]; - const replayCheckArguments = [options.sourceCommit, ...new Set(evidenceFiles)] + const replayCheckArguments = [ + options.sourceCommit, + ...evidence.flatMap((value) => [value.name, value.sha256]), + ] .map(posixShellQuote) .join(' '); const cleanCheckout = [ - `node --input-type=module -e "$SCRIPTSPECT_REPLAY_CHECK" -- ${replayCheckArguments}`, + `node --input-type=module -e ${posixShellQuote(CORPUS_REPLAY_BOOTSTRAP)} -- "$SCRIPTSPECT_REPLAY_CHECK" ${replayCheckArguments}`, ]; const environment = [ `SCRIPTSPECT_SOURCE_COMMIT=${posixShellQuote(options.sourceCommit)}`, @@ -216,21 +408,24 @@ function reproductionCommand(options: { `CORPUS_SAMPLE_METHOD=${posixShellQuote(options.sampleMethod)}`, `CORPUS_SAMPLE_SEED=${posixShellQuote(options.sampleSeed)}`, `CORPUS_LIMITS_JSON=${posixShellQuote(JSON.stringify(options.limits))}`, - ...(options.candidateSnapshotFile === undefined + ...(options.candidateSnapshot === undefined ? [] - : [`CORPUS_CANDIDATE_SNAPSHOT=${posixShellQuote(basename(options.candidateSnapshotFile))}`]), - ...(options.sampleEvidenceFile === undefined + : [`CORPUS_CANDIDATE_SNAPSHOT=${posixShellQuote(options.candidateSnapshot.name)}`]), + ...(options.sampleEvidence === undefined ? [] - : [`CORPUS_SAMPLE_EVIDENCE=${posixShellQuote(basename(options.sampleEvidenceFile))}`]), + : [`CORPUS_SAMPLE_EVIDENCE=${posixShellQuote(options.sampleEvidence.name)}`]), ]; - return [ - `SCRIPTSPECT_REPLAY_CHECK=${posixShellQuote(CORPUS_REPLAY_CHECK_SOURCE)}`, - `: "\${GITHUB_TOKEN:?set GITHUB_TOKEN to a read-only public-repository token}"`, - `git -c advice.detachedHead=false checkout --detach ${posixShellQuote(options.sourceCommit)}`, + const commands = [ + 'set +a', + `SCRIPTSPECT_REPLAY_CHECK=${posixShellQuote(CORPUS_REPLAY_CHECK_GZIP_BASE64)}`, + 'unset SCRIPTSPECT_REPLAY_TOKEN', + `SCRIPTSPECT_REPLAY_TOKEN="\${GITHUB_TOKEN-}"`, + 'unset GITHUB_TOKEN', `test "$(node --version)" = ${posixShellQuote(options.environment.node)}`, `test "$(node -p 'process.platform')" = ${posixShellQuote(options.environment.platform)}`, `test "$(node -p 'process.arch')" = ${posixShellQuote(options.environment.arch)}`, ...cleanCheckout, + 'test ! -e node_modules && test ! -L node_modules', 'corepack enable', "corepack prepare 'pnpm@11.24.0' --activate", 'pnpm install --frozen-lockfile', @@ -240,12 +435,15 @@ function reproductionCommand(options: { options.environment.runnerOs === undefined ? 'unset RUNNER_OS' : `export RUNNER_OS=${posixShellQuote(options.environment.runnerOs)}`, - `${environment.join(' ')} pnpm exec tsx tools/corpus-scan.ts ${posixShellQuote(basename(options.inputFile))} ${posixShellQuote(outputDirectory)}`, - ].join(' && '); + 'test -n "$SCRIPTSPECT_REPLAY_TOKEN"', + `GITHUB_TOKEN="$SCRIPTSPECT_REPLAY_TOKEN" ${environment.join(' ')} pnpm exec tsx tools/corpus-scan.ts ${posixShellQuote(options.input.name)} ${posixShellQuote(outputDirectory)}`, + ]; + return `(${commands.join(' && ')})`; } -function readLocatorSequence(inputFile: string): ReturnType[] { - return readFileSync(inputFile, 'utf8') +function readLocatorSequence(input: Buffer): ReturnType[] { + return input + .toString('utf8') .split(/\r?\n/u) .map((line) => line.trim()) .filter((line) => line !== '' && !line.startsWith('#')) @@ -690,6 +888,15 @@ function renderSummary(manifest: CorpusRunManifest): string { export async function runCorpusScan(options: CorpusScanOptions): Promise { if (options.token === '') throw new Error('GITHUB_TOKEN is required (read-only public access)'); const sourceCommit = exactSourceCommit(options.sourceCommit); + const evidencePaths = corpusEvidencePaths({ ...options, sourceCommit }); + validateEvidenceAgainstSourceTree(evidencePaths, sourceCommit, options.sourceCheckout); + const evidenceByRole = new Map( + evidencePaths.map(readCorpusEvidence).map((input) => [input.role, input] as const), + ); + const inputEvidence = evidenceByRole.get('repository list'); + if (inputEvidence === undefined) throw new Error('repository list evidence was unavailable'); + const candidateSnapshotEvidence = evidenceByRole.get('candidate snapshot'); + const sampleEvidenceInput = evidenceByRole.get('sample evidence'); const generatedAt = options.generatedAt ?? new Date().toISOString(); const environment: CorpusRunManifest['environment'] = { node: process.version, @@ -700,26 +907,15 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise(); if ( @@ -852,7 +1048,7 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise { From 1bf6503be3cb21e976950b9b9d1e24a3488693b3 Mon Sep 17 00:00:00 2001 From: Tom409114 Date: Tue, 1 Sep 2026 11:37:58 +0800 Subject: [PATCH 9/9] fix(corpus): isolate replay package commands --- docs/evidence/corpus-method.md | 13 +-- tests/corpus/corpus-run.test.ts | 146 +++++++++++++++++++++++--------- tools/corpus-scan.ts | 10 +-- 3 files changed, 117 insertions(+), 52 deletions(-) diff --git a/docs/evidence/corpus-method.md b/docs/evidence/corpus-method.md index c9cf0b5..fd29c12 100644 --- a/docs/evidence/corpus-method.md +++ b/docs/evidence/corpus-method.md @@ -102,11 +102,14 @@ not exist before package tooling starts (including as a broken link), while the second gate permits the frozen install's ignored `node_modules` output. The complete replay runs in a POSIX subshell. It first disables inherited shell `allexport`, copies `GITHUB_TOKEN` into a non-exported shell variable, and -removes the credential from the environment before Git, Corepack, and pnpm -install run; the token is injected as a -single-command environment assignment -only for the final scanner invocation. Caller environment values therefore -survive the replay unchanged. +removes the credential from the environment. An empty token fails before +package setup or output-directory creation, so adding a token and retrying does +not require cleanup. Corepack, pnpm install, and the final pnpm scanner command +are resolved as external commands through `command env`, bypassing caller shell +functions and aliases that could otherwise read shell-local replay state. The +token is injected as a single-command environment assignment only for the final +scanner invocation. Caller environment values therefore survive the replay +unchanged. Replay also requires the exact recorded Node version, platform, and architecture; inside its subshell it restores the recorded `RUNNER_OS` value or diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index 6387c0c..45c1e5d 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -363,22 +363,41 @@ function executeReplay( shellAfter?: string; runnerOs?: string; environment?: NodeJS.ProcessEnv; + token?: string; + corepackBody?: string; + pnpmBody?: string; } = {}, ): ReturnType { + const commandDirectory = temporaryDirectory(); + const corepackPath = join(commandDirectory, 'corepack'); + const pnpmPath = join(commandDirectory, 'pnpm'); + writeFileSync(corepackPath, `#!/bin/sh\nset -eu\n${options.corepackBody ?? 'exit 0'}\n`); + writeFileSync( + pnpmPath, + `#!/bin/sh\nset -eu\n${ + options.pnpmBody ?? + [ + `if [ "\${1-}" = "exec" ]; then`, + ` printf "RUNNER_OS=%s\\n" "\${RUNNER_OS-}" > replay-observation.txt`, + ` printf "CORPUS_LIMITS_JSON=%s\\n" "\${CORPUS_LIMITS_JSON-}" >> replay-observation.txt`, + ' printf "ARGS=" >> replay-observation.txt', + ' printf "<%s>" "$@" >> replay-observation.txt', + ' printf "\\n" >> replay-observation.txt', + 'fi', + 'exit 0', + ].join('\n') + }\n`, + ); + chmodSync(corepackPath, 0o755); + chmodSync(pnpmPath, 0o755); + const shellCommandDirectory = execFileSync(posixShell(), ['-c', 'pwd'], { + cwd: commandDirectory, + encoding: 'utf8', + }).trim(); + const quotedCommandDirectory = `'${shellCommandDirectory.replaceAll("'", `'"'"'`)}'`; const script = [ - 'corepack() { return 0; }', - [ - 'pnpm() {', - ' if [ "$1" = "exec" ]; then', - ` printf "RUNNER_OS=%s\\n" "\${RUNNER_OS-}" > replay-observation.txt`, - ` printf "CORPUS_LIMITS_JSON=%s\\n" "\${CORPUS_LIMITS_JSON-}" >> replay-observation.txt`, - ' printf "ARGS=" >> replay-observation.txt', - ' printf "<%s>" "$@" >> replay-observation.txt', - ' printf "\\n" >> replay-observation.txt', - ' fi', - ' return 0', - '}', - ].join('\n'), + `PATH=${quotedCommandDirectory}:"$PATH"`, + 'export PATH', options.shellSetup ?? '', reproduction, options.shellAfter ?? '', @@ -389,7 +408,7 @@ function executeReplay( env: { ...process.env, ...options.environment, - GITHUB_TOKEN: 'ephemeral-replay-test-token', + GITHUB_TOKEN: options.token ?? 'ephemeral-replay-test-token', ...(options.runnerOs === undefined ? {} : { RUNNER_OS: options.runnerOs }), }, }); @@ -1018,19 +1037,17 @@ describe('immutable corpus run evidence', () => { it('allows the frozen install to create an ignored dependency directory', async () => { const replay = await replayFixture(); - const shellSetup = [ - 'pnpm() {', - ' if [ "$1" = "install" ]; then', + const pnpmBody = [ + `if [ "\${1-}" = "install" ]; then`, ' mkdir -p node_modules/installed', ' printf installed > node_modules/installed/package.json', - ' elif [ "$1" = "exec" ]; then', + `elif [ "\${1-}" = "exec" ]; then`, ' printf scanner-ran > replay-observation.txt', - ' fi', - ' return 0', - '}', + 'fi', + 'exit 0', ].join('\n'); - const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + const result = executeReplay(replay.directory, replay.reproduction, { pnpmBody }); expect(result.status, String(result.stderr)).toBe(0); expect(readFileSync(join(replay.directory, 'replay-observation.txt'), 'utf8')).toBe( @@ -1040,22 +1057,22 @@ describe('immutable corpus run evidence', () => { it('withholds the token from package setup and injects it only into the scanner command', async () => { const replay = await replayFixture(); - const shellSetup = [ - 'set -a', - 'pnpm() {', - ' if [ "$1" = "install" ]; then', - ' if env | grep -q "^GITHUB_TOKEN=" || env | grep -q "^SCRIPTSPECT_REPLAY_TOKEN="; then', + const pnpmBody = [ + `if [ "\${1-}" = "install" ]; then`, + ' if env | grep -q "^GITHUB_TOKEN=" || env | grep -q "^SCRIPTSPECT_REPLAY_TOKEN="; then', ' printf leaked > package-setup-token-leak.txt', - ' return 97', - ' fi', - ' elif [ "$1" = "exec" ]; then', - ' env | grep "^GITHUB_TOKEN=" > scanner-token.txt', + ' exit 97', ' fi', - ' return 0', - '}', + `elif [ "\${1-}" = "exec" ]; then`, + ' env | grep "^GITHUB_TOKEN=" > scanner-token.txt', + 'fi', + 'exit 0', ].join('\n'); - const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + const result = executeReplay(replay.directory, replay.reproduction, { + shellSetup: 'set -a', + pnpmBody, + }); expect(result.status, String(result.stderr)).toBe(0); expect(existsSync(join(replay.directory, 'package-setup-token-leak.txt'))).toBe(false); @@ -1064,6 +1081,53 @@ describe('immutable corpus run evidence', () => { ); }); + it.each(['corepack', 'pnpm'])( + 'bypasses a caller-defined %s function that can read shell-local replay state', + async (command) => { + const replay = await replayFixture(); + const shellSetup = [ + `${command}() {`, + ` if [ -n "\${SCRIPTSPECT_REPLAY_TOKEN-}" ]; then`, + ' printf leaked > caller-function-token-leak.txt', + ' return 97', + ' fi', + ' return 96', + '}', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + + expect(result.status, String(result.stderr)).toBe(0); + expect(existsSync(join(replay.directory, 'caller-function-token-leak.txt'))).toBe(false); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(true); + }, + ); + + it('rejects an empty token before package setup or output creation and remains retryable', async () => { + const replay = await replayFixture(); + const markerDirectory = temporaryDirectory(); + const packageSetupMarker = join(markerDirectory, 'package-setup-called.txt'); + const outputDirectory = join( + replay.directory, + `corpus-reproduction-${git(replay.directory, 'rev-parse', 'HEAD')}`, + ); + const markerBody = 'printf called > "$PACKAGE_SETUP_MARKER"\nexit 0'; + + const missingToken = executeReplay(replay.directory, replay.reproduction, { + token: '', + environment: { PACKAGE_SETUP_MARKER: packageSetupMarker.replaceAll('\\', '/') }, + corepackBody: markerBody, + pnpmBody: markerBody, + }); + + expect(missingToken.status, String(missingToken.stderr)).not.toBe(0); + expect(existsSync(packageSetupMarker)).toBe(false); + expect(existsSync(outputDirectory)).toBe(false); + + const retry = executeReplay(replay.directory, replay.reproduction); + expect(retry.status, String(retry.stderr)).toBe(0); + }); + it('ignores inherited Git repository redirection while validating the checkout', async () => { const replay = await replayFixture(); const redirectedWorktree = temporaryDirectory(); @@ -1104,18 +1168,16 @@ describe('immutable corpus run evidence', () => { 'rejects lifecycle changes to %s evidence after the frozen install', async (evidenceName) => { const replay = await replayFixture(); - const shellSetup = [ - 'pnpm() {', - ' if [ "$1" = "install" ]; then', + const pnpmBody = [ + `if [ "\${1-}" = "install" ]; then`, ` printf tampered > '${evidenceName}'`, - ' elif [ "$1" = "exec" ]; then', + `elif [ "\${1-}" = "exec" ]; then`, ' printf scanner-ran > replay-observation.txt', - ' fi', - ' return 0', - '}', + 'fi', + 'exit 0', ].join('\n'); - const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + const result = executeReplay(replay.directory, replay.reproduction, { pnpmBody }); expect(result.status, String(result.stderr)).not.toBe(0); expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index f33b247..7fea8d9 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -421,22 +421,22 @@ function reproductionCommand(options: { 'unset SCRIPTSPECT_REPLAY_TOKEN', `SCRIPTSPECT_REPLAY_TOKEN="\${GITHUB_TOKEN-}"`, 'unset GITHUB_TOKEN', + 'test -n "$SCRIPTSPECT_REPLAY_TOKEN"', `test "$(node --version)" = ${posixShellQuote(options.environment.node)}`, `test "$(node -p 'process.platform')" = ${posixShellQuote(options.environment.platform)}`, `test "$(node -p 'process.arch')" = ${posixShellQuote(options.environment.arch)}`, ...cleanCheckout, 'test ! -e node_modules && test ! -L node_modules', - 'corepack enable', - "corepack prepare 'pnpm@11.24.0' --activate", - 'pnpm install --frozen-lockfile', + 'command env corepack enable', + "command env corepack prepare 'pnpm@11.24.0' --activate", + 'command env pnpm install --frozen-lockfile', ...cleanCheckout, `test ! -e ${posixShellQuote(outputDirectory)}`, `mkdir -- ${posixShellQuote(outputDirectory)}`, options.environment.runnerOs === undefined ? 'unset RUNNER_OS' : `export RUNNER_OS=${posixShellQuote(options.environment.runnerOs)}`, - 'test -n "$SCRIPTSPECT_REPLAY_TOKEN"', - `GITHUB_TOKEN="$SCRIPTSPECT_REPLAY_TOKEN" ${environment.join(' ')} pnpm exec tsx tools/corpus-scan.ts ${posixShellQuote(options.input.name)} ${posixShellQuote(outputDirectory)}`, + `command env GITHUB_TOKEN="$SCRIPTSPECT_REPLAY_TOKEN" ${environment.join(' ')} pnpm exec tsx tools/corpus-scan.ts ${posixShellQuote(options.input.name)} ${posixShellQuote(outputDirectory)}`, ]; return `(${commands.join(' && ')})`; }