diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index e90f5f8..91d297f 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -27,88 +27,22 @@ jobs: corepack enable corepack prepare pnpm@11.24.0 --activate pnpm install --frozen-lockfile - - name: Collect repository candidates + - name: Collect the ranked repository candidate snapshot env: - GH_TOKEN: ${{ github.token }} - CANDIDATE_FILE: ${{ runner.temp }}/repository-candidates.txt - run: | - set -euo pipefail - gh api 'search/repositories?q=language:typescript+stars:%3E2000&sort=stars&order=desc&per_page=100' \ - --jq '.items[].full_name' > "$CANDIDATE_FILE" - gh api 'search/repositories?q=language:javascript+stars:%3E5000&sort=stars&order=desc&per_page=100' \ - --jq '.items[].full_name' >> "$CANDIDATE_FILE" - - name: Select the exact deterministic repository sample - env: - REPO_COUNT: ${{ inputs.repo-count || '100' }} - CANDIDATE_FILE: ${{ runner.temp }}/repository-candidates.txt - SELECTED_FILE: ${{ runner.temp }}/repository-names.txt - SELECTION_FILE: repository-sample.json - run: | - set -euo pipefail - [[ "$REPO_COUNT" =~ ^([1-9]|[1-9][0-9]|100)$ ]] - [[ "$REPO_COUNT" -le 100 ]] - node --input-type=module <<'NODE' - import { readFileSync, writeFileSync } from 'node:fs'; - - const countText = process.env.REPO_COUNT ?? ''; - if (!/^(?:[1-9]|[1-9][0-9]|100)$/u.test(countText)) { - throw new Error('REPO_COUNT must be an integer from 1 through 100'); - } - const requested = Number(countText); - const candidateFile = process.env.CANDIDATE_FILE; - const selectedFile = process.env.SELECTED_FILE; - const selectionFile = process.env.SELECTION_FILE; - if (!candidateFile || !selectedFile || !selectionFile) { - throw new Error('corpus selection file paths are required'); - } - - const candidates = readFileSync(candidateFile, 'utf8') - .split(/\r?\n/u) - .filter(Boolean); - for (const repository of candidates) { - if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { - throw new Error(`invalid repository name: ${repository}`); - } - } - const uniqueCandidates = [...new Set(candidates)].sort(); - if (uniqueCandidates.length < requested) { - throw new Error( - `requested ${requested} repositories but only ${uniqueCandidates.length} unique candidates were returned`, - ); - } - const selected = uniqueCandidates.slice(0, requested); - writeFileSync(selectedFile, `${selected.join('\n')}\n`, 'utf8'); - writeFileSync( - selectionFile, - `${JSON.stringify({ schemaVersion: 1, requested, actual: selected.length }, null, 2)}\n`, - 'utf8', - ); - NODE - - name: Resolve immutable repository commits + GITHUB_TOKEN: ${{ github.token }} + run: pnpm exec tsx tools/corpus-candidates.ts repository-candidates.json + - name: Resolve the exact root-eligible repository sample env: - GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} REPO_COUNT: ${{ inputs.repo-count || '100' }} - SELECTED_FILE: ${{ runner.temp }}/repository-names.txt - run: | - set -euo pipefail - : > repos.txt - while IFS= read -r repository; do - [[ "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] - default_branch=$(gh api "repos/$repository" --jq '.default_branch') - [[ -n "$default_branch" ]] - commit=$(gh api -X GET "repos/$repository/commits" \ - -f sha="$default_branch" -f per_page=1 --jq '.[0].sha') - [[ "$commit" =~ ^[0-9a-f]{40}$ ]] - printf '%s@%s\n' "$repository" "$commit" >> repos.txt - done < "$SELECTED_FILE" - sort -u repos.txt -o repos.txt - awk '!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+@[0-9a-f]{40}$/ { exit 1 }' repos.txt - ACTUAL_COUNT=$(wc -l < repos.txt) - test "$ACTUAL_COUNT" -eq "$REPO_COUNT" + run: pnpm exec tsx tools/corpus-resolve.ts repository-candidates.json repos.txt repository-sample.json "$REPO_COUNT" - name: Scan scripts without writing to sampled repositories env: GITHUB_TOKEN: ${{ github.token }} SCRIPTSPECT_SOURCE_COMMIT: ${{ github.sha }} + CORPUS_SAMPLE_METHOD: popularity-strata-round-robin-v1 + CORPUS_CANDIDATE_SNAPSHOT: repository-candidates.json + CORPUS_SAMPLE_EVIDENCE: repository-sample.json run: pnpm exec tsx tools/corpus-scan.ts repos.txt - name: Create a deterministic 100-finding adjudication draft when available if: always() @@ -126,6 +60,7 @@ jobs: with: name: corpus-draft-${{ github.run_id }} path: | + repository-candidates.json repos.txt repository-sample.json findings.jsonl diff --git a/docs/evidence/corpus-method.md b/docs/evidence/corpus-method.md index 7d6508a..fd29c12 100644 --- a/docs/evidence/corpus-method.md +++ b/docs/evidence/corpus-method.md @@ -5,22 +5,40 @@ public code. A machine scan produces a **data draft**, not a precision claim. ## Immutable scan -1. The manual/monthly corpus workflow selects public JavaScript and TypeScript - repositories, resolves each default branch once, and records only - `owner/repository@40-character-commit` locators. -2. `tools/corpus-scan.ts` reads those immutable locators through GitHub's tree - and blob APIs. It never clones with credentials, executes scripts, or writes - to a sampled repository. -3. Only the root `package.json`, `pnpm-workspace.yaml`, and candidate workspace +1. The manual/monthly corpus workflow captures the complete first page of two + popularity-ranked GitHub Search strata (JavaScript and TypeScript), including + each query, response hash, rank, star count, and repository. It then uses a + deterministic rank-by-rank round robin and de-duplicates a repository at its + first appearance. The full ordered snapshot is preserved rather than sorting + away its rank or stratum. +2. The resolver batches candidates through GitHub GraphQL. One response anchors + the default-branch commit and root `package.json` blob together. Only an exact + `NOT_FOUND` error at that candidate's root-file field is an eligibility + exclusion; every other partial error or response mismatch fails closed. The + resolver records only `owner/repository@40-character-commit` locators and + hashes the complete ordered candidate snapshot into its evidence. +3. `tools/corpus-scan.ts` makes one bounded recursive-tree REST request per + selected repository. It reads each selected manifest from + `raw.githubusercontent.com` at the exact commit without an Authorization + header, then verifies byte length and the Git blob OID from the immutable tree + before analysis. Manifest downloads therefore do not consume per-blob GitHub + REST core requests. The scanner never clones, executes scripts, or writes to + a sampled repository. +4. Only the root `package.json`, `pnpm-workspace.yaml`, and candidate workspace `package.json` files are materialized in a fresh temporary directory. The normal CLI analyzer then applies the same canonical-root, workspace glob, dependency visibility, and symlink-boundary policy used for local projects. -4. Dependency/VCS/vendor/generated/build/distribution directories and symlink +5. Dependency/VCS/vendor/generated/build/distribution directories and symlink tree entries are excluded. The default ceilings are 20,000 tree entries, 500 manifests, depth 12, 1 MiB per file, and 10 MiB decoded bytes per repository. A GitHub-truncated tree or any local limit marks the repository - `truncated`; API, decoding, or analysis errors mark it `failed`. Neither - status contributes to promoted totals. + `truncated`; API, decoding, immutable-blob verification, or analysis errors + mark it `failed`. Neither status contributes to promoted totals. GitHub HTTP + failures retain their status, rate-limit limit/remaining/reset/used/resource, + Retry-After, request ID, and a rate/auth/permission classification; request + credentials are never persisted. Findings from truncated or failed + repositories are also excluded from `findings.jsonl`, so the adjudication + draft cannot silently sample incomplete repositories. The scan reports root-only and workspace-full counts separately. This prevents root-only PS040 results from being presented as monorepo truth. @@ -29,14 +47,82 @@ root-only PS040 results from being presented as monorepo truth. The workflow artifact contains: +- `repository-candidates.json`: the complete ordered popularity-strata snapshot, + including query metadata, ranks, repositories, response hashes, and status; +- `repository-sample.json`: candidate-snapshot SHA-256, deterministic method, + GraphQL request/cost evidence, rootless replacements, and selected immutable + commits/root-manifest blobs; - `repos.txt`: the exact immutable sample; - `findings.jsonl`: stable finding IDs, immutable source URLs, script SHA-256, - rule metadata, spans, and redacted messages—never raw script source; + rule metadata, spans, and source-free rule summaries—never raw script source; - `corpus-run.json`: selected manifest paths, scanner/source commit and hashes, - rule-registry hash, limits, sample method/seed, environment, per-repository - status, separate scan modes, artifact hashes, and reproduction command; + rule-registry hash, limits, sample method/seed, hashes of the full candidate + snapshot and sample evidence, environment, per-repository status, separate + scan modes, artifact hashes, and a directly copyable POSIX-shell reproduction + command; - `summary.md`: an explicitly unverified summary for maintainers. +To replay a run, place its `repository-candidates.json`, +`repository-sample.json`, and `repos.txt` as nonignored, untracked regular files +at the root of a **fresh dedicated checkout already positioned at the recorded +commit**. Their basenames must be safe and unique and cannot collide with +`.git`, `node_modules`, `findings.jsonl`, `summary.md`, `corpus-run.json`, or the +deterministic replay output directory. Before the original scan makes a network +request, it also requires its source checkout's `HEAD` to equal the recorded +source commit and compares the evidence basenames with that commit's raw root +tree. A basename already tracked there is rejected instead of emitting an +intrinsically unreplayable command. Set `GITHUB_TOKEN` externally to a read-only +public-repository token, then run the command recorded in `corpus-run.json`. +The command never invokes `git checkout`: a different HEAD fails closed and no +smudge/process filter or `post-checkout` hook gets an opportunity to run first. + +The validator source is gzip/base64-embedded once in the reproduction command +and decoded with Node's built-in `node:zlib` into a data-module; it is never +loaded from the checkout being validated. Every Git subprocess forces +`core.fsmonitor=false` and `core.hooksPath=/dev/null`, disables replacement +objects, binds the worktree to the current checkout, and discards inherited Git +repository/index/object/config redirection variables. The validator parses the +raw NUL-delimited `HEAD` tree and hashes every regular file's worktree bytes +with the repository's Git object algorithm. It separately parses +NUL-delimited `git ls-files --stage` records and requires every path, mode, blob +OID, and stage to match the `HEAD` tree exactly. This direct comparison makes +`assume-unchanged`, `skip-worktree`, `fsmonitor-valid`, clean/smudge filters, +EOL or working-tree encodings, and racy stat data unable to substitute other +bytes without invoking `git diff` or `git status`. POSIX executable bits and +raw symlink targets must match their tree modes. Gitlinks are rejected rather +than trusted as submodules. A NUL-delimited +`git ls-files --others --exclude-standard` enumeration must contain exactly the +three named evidence inputs; any other nonignored untracked file or any Git or +filesystem failure is fatal. Raw path buffers preserve unusual filenames. + +Before **and** after the frozen-lockfile install, each evidence input must still +be a regular file with the exact SHA-256 recorded by the original manifest, and +the tracked worktree/index/untracked checks must all pass. `node_modules` must +not exist before package tooling starts (including as a broken link), while the +second gate permits the frozen install's ignored `node_modules` output. The +complete replay runs in a POSIX subshell. It first disables inherited shell +`allexport`, copies `GITHUB_TOKEN` into a non-exported shell variable, and +removes the credential from the environment. An empty token fails before +package setup or output-directory creation, so adding a token and retrying does +not require cleanup. Corepack, pnpm install, and the final pnpm scanner command +are resolved as external commands through `command env`, bypassing caller shell +functions and aliases that could otherwise read shell-local replay state. The +token is injected as a single-command environment assignment only for the final +scanner invocation. Caller environment values therefore survive the replay +unchanged. + +Replay also requires the exact recorded Node version, platform, and +architecture; inside its subshell it restores the recorded `RUNNER_OS` value or +explicitly unsets it. It binds the complete canonical limits JSON, original +generation timestamp, sample method and seed, candidate snapshot, sample +evidence, repository list, and a new deterministic output directory to the +scanner's actual environment variables and positional arguments. The command +refuses to reuse that output directory. Only evidence basenames and their +digests plus a token-variable reference are recorded; neither the credential +nor a local absolute path is persisted. A run recorded on another platform +must therefore be replayed in a matching environment with the recorded Node +patch version. + The run fails if any repository fails, while still leaving `corpus-run.json` for diagnosis. Truncation is visible and excluded rather than silently treated as a complete sample. Expiring workflow artifacts supplement; they do not @@ -77,4 +163,3 @@ exist, scriptspect makes no head-to-head superiority claim. - Draft, partial, overdue, or failed runs leave the relevant gate `OPEN`. - No issue, pull request, comment, email, or other third-party write is made from corpus automation. Such contact requires explicit human authorization. - diff --git a/tests/corpus/corpus-candidates.test.ts b/tests/corpus/corpus-candidates.test.ts new file mode 100644 index 0000000..9288276 --- /dev/null +++ b/tests/corpus/corpus-candidates.test.ts @@ -0,0 +1,248 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { afterEach, expect, it } from 'vitest'; + +const temporaryDirectories: string[] = []; + +type Collector = (options: { + outputFile: string; + token: string; + fetchImpl: typeof fetch; +}) => Promise; + +function temporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), 'scriptspect-corpus-candidates-test-')); + temporaryDirectories.push(directory); + return directory; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +async function collector(): Promise { + const moduleUrl = pathToFileURL(join(process.cwd(), 'tools', 'corpus-candidates.ts')).href; + const module = (await import(moduleUrl).catch(() => ({}))) as { + collectCorpusCandidates?: Collector; + }; + expect(typeof module.collectCorpusCandidates).toBe('function'); + if (module.collectCorpusCandidates === undefined) { + throw new Error('corpus candidate collector was unavailable'); + } + return module.collectCorpusCandidates; +} + +function searchApi(): typeof fetch { + return (async (input: string | URL | Request) => { + const url = new URL(String(input)); + const query = url.searchParams.get('q'); + expect(query).toContain('is:public'); + const items = query?.includes('language:typescript') + ? [ + { full_name: 'alpha/shared', stargazers_count: 100 }, + { full_name: 'gamma/typescript', stargazers_count: 90 }, + ] + : [ + { full_name: 'beta/javascript', stargazers_count: 110 }, + { full_name: 'alpha/shared', stargazers_count: 100 }, + ]; + return Response.json( + { total_count: items.length, incomplete_results: false, items }, + { + headers: { + 'x-ratelimit-limit': '30', + 'x-ratelimit-remaining': '28', + 'x-ratelimit-reset': '1788220800', + 'x-ratelimit-used': '2', + 'x-ratelimit-resource': 'search', + 'x-github-request-id': 'SEARCH:TEST', + }, + }, + ); + }) as typeof fetch; +} + +function invalidSearchApi(body: unknown): typeof fetch { + return (async () => Response.json(body)) as typeof fetch; +} + +it('persists both ranked strata and a deterministic round-robin candidate universe', async () => { + const directory = temporaryDirectory(); + const outputFile = join(directory, 'repository-candidates.json'); + + await (await collector())({ + outputFile, + token: 'read-only-test-token', + fetchImpl: searchApi(), + }); + + const snapshot = JSON.parse(readFileSync(outputFile, 'utf8')) as Record; + expect(snapshot).toMatchObject({ + schemaVersion: 1, + status: 'complete', + method: 'popularity-strata-round-robin-v1', + strata: [ + { + id: 'typescript', + query: 'is:public language:typescript stars:>2000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + candidates: [ + { rank: 1, repository: 'alpha/shared', stars: 100 }, + { rank: 2, repository: 'gamma/typescript', stars: 90 }, + ], + }, + { + id: 'javascript', + query: 'is:public language:javascript stars:>5000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + candidates: [ + { rank: 1, repository: 'beta/javascript', stars: 110 }, + { rank: 2, repository: 'alpha/shared', stars: 100 }, + ], + }, + ], + orderedCandidates: [ + { position: 1, stratum: 'typescript', rank: 1, repository: 'alpha/shared' }, + { position: 2, stratum: 'javascript', rank: 1, repository: 'beta/javascript' }, + { position: 3, stratum: 'typescript', rank: 2, repository: 'gamma/typescript' }, + ], + }); +}); + +it('persists rate-limit headers without leaking the request token', async () => { + const directory = temporaryDirectory(); + const outputFile = join(directory, 'repository-candidates.json'); + const headers = { + 'x-ratelimit-limit': '30', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': '1788224400', + 'x-ratelimit-used': '30', + 'x-ratelimit-resource': 'search', + 'retry-after': '60', + 'x-github-request-id': 'SEARCH-REQ-1', + }; + + await expect( + (await collector())({ + outputFile, + token: 'github_pat_SHOULD_NOT_LEAK_12345678901234567890', + fetchImpl: (async () => + new Response('{"message":"rate limited"}', { status: 403, headers })) as typeof fetch, + }), + ).rejects.toThrow('GitHub API 403'); + + const text = readFileSync(outputFile, 'utf8'); + expect(text).not.toContain('SHOULD_NOT_LEAK'); + expect(JSON.parse(text)).toMatchObject({ + schemaVersion: 1, + status: 'failed', + failure: { + kind: 'primary-rate-limit-exhausted', + status: 403, + rateLimit: { + limit: '30', + remaining: '0', + reset: '1788224400', + used: '30', + resource: 'search', + }, + retryAfter: '60', + requestId: 'SEARCH-REQ-1', + }, + }); +}); + +it('distinguishes a permission denial from primary rate exhaustion', async () => { + const directory = temporaryDirectory(); + const outputFile = join(directory, 'repository-candidates.json'); + + await expect( + (await collector())({ + outputFile, + token: 'read-only-test-token', + fetchImpl: (async () => + new Response('{"message":"Resource not accessible by integration"}', { + status: 403, + headers: { + 'x-ratelimit-limit': '30', + 'x-ratelimit-remaining': '29', + 'x-ratelimit-reset': '1788224400', + 'x-ratelimit-used': '1', + 'x-ratelimit-resource': 'search', + 'x-github-request-id': 'SEARCH-PERMISSION-1', + }, + })) as typeof fetch, + }), + ).rejects.toThrow('GitHub API 403'); + + expect(JSON.parse(readFileSync(outputFile, 'utf8'))).toMatchObject({ + status: 'failed', + failure: { + kind: 'permission-denied', + status: 403, + rateLimit: { remaining: '29', resource: 'search' }, + retryAfter: null, + requestId: 'SEARCH-PERMISSION-1', + }, + }); +}); + +it('rejects Search responses without a valid total_count', async () => { + const directory = temporaryDirectory(); + + await expect( + (await collector())({ + outputFile: join(directory, 'repository-candidates.json'), + token: 'read-only-test-token', + fetchImpl: invalidSearchApi({ + incomplete_results: false, + items: [{ full_name: 'alpha/project', stargazers_count: 100 }], + }), + }), + ).rejects.toThrow(/search response was incomplete or invalid/); +}); + +it('rejects Search responses whose item count does not match the bounded total', async () => { + const directory = temporaryDirectory(); + + await expect( + (await collector())({ + outputFile: join(directory, 'repository-candidates.json'), + token: 'read-only-test-token', + fetchImpl: invalidSearchApi({ + total_count: 2, + incomplete_results: false, + items: [{ full_name: 'alpha/project', stargazers_count: 100 }], + }), + }), + ).rejects.toThrow(/search response was incomplete or invalid/); +}); + +it('rejects Search candidates that are not sorted by non-increasing stars', async () => { + const directory = temporaryDirectory(); + + await expect( + (await collector())({ + outputFile: join(directory, 'repository-candidates.json'), + token: 'read-only-test-token', + fetchImpl: invalidSearchApi({ + total_count: 2, + incomplete_results: false, + items: [ + { full_name: 'alpha/project', stargazers_count: 100 }, + { full_name: 'beta/project', stargazers_count: 101 }, + ], + }), + }), + ).rejects.toThrow(/ranked by stars/); +}); diff --git a/tests/corpus/corpus-resolve.test.ts b/tests/corpus/corpus-resolve.test.ts new file mode 100644 index 0000000..f61b020 --- /dev/null +++ b/tests/corpus/corpus-resolve.test.ts @@ -0,0 +1,475 @@ +import { createHash } from 'node:crypto'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { afterEach, expect, it } from 'vitest'; + +const ROOTLESS_COMMIT = '1111111111111111111111111111111111111111'; +const FIRST_ELIGIBLE_COMMIT = '2222222222222222222222222222222222222222'; +const SECOND_ELIGIBLE_COMMIT = '3333333333333333333333333333333333333333'; +const FIRST_BLOB = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +const SECOND_BLOB = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; +const temporaryDirectories: string[] = []; + +type Resolver = (options: { + candidateFile: string; + outputFile: string; + evidenceFile: string; + requested: number; + token: string; + fetchImpl: typeof fetch; +}) => Promise; + +function temporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), 'scriptspect-corpus-resolve-test-')); + temporaryDirectories.push(directory); + return directory; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +async function resolver(): Promise { + const resolverUrl = pathToFileURL(join(process.cwd(), 'tools', 'corpus-resolve.ts')).href; + const module = (await import(resolverUrl).catch(() => ({}))) as { + resolveCorpusSample?: Resolver; + }; + expect(typeof module.resolveCorpusSample).toBe('function'); + if (module.resolveCorpusSample === undefined) throw new Error('corpus resolver was unavailable'); + return module.resolveCorpusSample; +} + +function candidateSnapshot(): string { + return `${JSON.stringify( + { + schemaVersion: 1, + status: 'complete', + method: 'popularity-strata-round-robin-v1', + strata: [ + { + id: 'typescript', + query: 'is:public language:typescript stars:>2000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: 'a'.repeat(64), + candidates: [ + { rank: 1, repository: 'alpha/rootless', stars: 300 }, + { rank: 2, repository: 'gamma/eligible', stars: 200 }, + ], + }, + { + id: 'javascript', + query: 'is:public language:javascript stars:>5000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: 'b'.repeat(64), + candidates: [ + { rank: 1, repository: 'beta/eligible', stars: 400 }, + { rank: 2, repository: 'alpha/rootless', stars: 300 }, + ], + }, + ], + orderedCandidates: [ + { position: 1, stratum: 'typescript', rank: 1, repository: 'alpha/rootless' }, + { position: 2, stratum: 'javascript', rank: 1, repository: 'beta/eligible' }, + { position: 3, stratum: 'typescript', rank: 2, repository: 'gamma/eligible' }, + ], + }, + null, + 2, + )}\n`; +} + +function candidateApi(): typeof fetch { + return (async (input: string | URL | Request, init?: RequestInit) => { + expect(String(input)).toBe('https://api.github.com/graphql'); + expect(init?.method).toBe('POST'); + const body = JSON.parse(String(init?.body)) as { query: string }; + expect(body.query).toContain('file(path: "package.json")'); + expect(body.query).toContain('rateLimit'); + return Response.json({ + data: { + r0: { + nameWithOwner: 'alpha/rootless', + defaultBranchRef: { + name: 'main', + target: { __typename: 'Commit', oid: ROOTLESS_COMMIT, file: null }, + }, + }, + r1: { + nameWithOwner: 'beta/eligible', + defaultBranchRef: { + name: 'main', + target: { + __typename: 'Commit', + oid: FIRST_ELIGIBLE_COMMIT, + file: { + name: 'package.json', + mode: 33188, + type: 'blob', + oid: FIRST_BLOB, + object: { __typename: 'Blob', oid: FIRST_BLOB, byteSize: 42, isBinary: false }, + }, + }, + }, + }, + r2: { + nameWithOwner: 'gamma/eligible', + defaultBranchRef: { + name: 'main', + target: { + __typename: 'Commit', + oid: SECOND_ELIGIBLE_COMMIT, + file: { + name: 'package.json', + mode: 33188, + type: 'blob', + oid: SECOND_BLOB, + object: { __typename: 'Blob', oid: SECOND_BLOB, byteSize: 43, isBinary: false }, + }, + }, + }, + }, + rateLimit: { + cost: 1, + limit: 5000, + remaining: 4999, + used: 1, + resetAt: '2026-09-01T01:00:00Z', + }, + }, + errors: [ + { + type: 'NOT_FOUND', + path: ['r0', 'defaultBranchRef', 'target', 'file'], + message: "Could not resolve file for path 'package.json'.", + }, + ], + }); + }) as typeof fetch; +} + +it('interleaves popularity strata, replaces rootless candidates, and hashes the full snapshot', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const outputFile = join(directory, 'repos.txt'); + const evidenceFile = join(directory, 'repository-sample.json'); + const snapshotText = candidateSnapshot(); + writeFileSync(candidateFile, snapshotText, 'utf8'); + + await (await resolver())({ + candidateFile, + outputFile, + evidenceFile, + requested: 2, + token: 'read-only-test-token', + fetchImpl: candidateApi(), + }); + + expect(readFileSync(outputFile, 'utf8')).toBe( + `beta/eligible@${FIRST_ELIGIBLE_COMMIT}\ngamma/eligible@${SECOND_ELIGIBLE_COMMIT}\n`, + ); + expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toEqual({ + schemaVersion: 2, + method: 'popularity-strata-round-robin-v1', + candidateSnapshotSha256: createHash('sha256').update(snapshotText).digest('hex'), + requested: 2, + actual: 2, + candidatesConsidered: 3, + status: 'complete', + api: { + transport: 'github-graphql-batch-v1', + batchSize: 20, + requests: 1, + cost: 1, + rateLimit: { + limit: 5000, + remaining: 4999, + used: 1, + resetAt: '2026-09-01T01:00:00Z', + }, + }, + selected: [ + { + position: 2, + stratum: 'javascript', + rank: 1, + repository: 'beta/eligible', + commit: FIRST_ELIGIBLE_COMMIT, + rootManifestOid: FIRST_BLOB, + rootManifestBytes: 42, + }, + { + position: 3, + stratum: 'typescript', + rank: 2, + repository: 'gamma/eligible', + commit: SECOND_ELIGIBLE_COMMIT, + rootManifestOid: SECOND_BLOB, + rootManifestBytes: 43, + }, + ], + exclusions: [ + { + position: 1, + stratum: 'typescript', + rank: 1, + repository: 'alpha/rootless', + commit: ROOTLESS_COMMIT, + reason: 'root-package-json-unavailable', + }, + ], + }); +}); + +it('hard-fails a rate exhaustion and persists non-secret response metadata', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const outputFile = join(directory, 'repos.txt'); + const evidenceFile = join(directory, 'repository-sample.json'); + writeFileSync(candidateFile, candidateSnapshot(), 'utf8'); + const headers = { + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': '1788224400', + 'x-ratelimit-used': '5000', + 'x-ratelimit-resource': 'graphql', + 'retry-after': '60', + 'x-github-request-id': 'REQ-123', + }; + + await expect( + (await resolver())({ + candidateFile, + outputFile, + evidenceFile, + requested: 1, + token: 'read-only-test-token-SHOULD-NOT-LEAK', + fetchImpl: (async () => + new Response('{"message":"rate limited"}', { status: 403, headers })) as typeof fetch, + }), + ).rejects.toThrow('GitHub API 403 for https://api.github.com/graphql'); + + expect(readFileSync(outputFile, 'utf8')).toBe(''); + const evidenceText = readFileSync(evidenceFile, 'utf8'); + expect(evidenceText).not.toContain('SHOULD-NOT-LEAK'); + expect(JSON.parse(evidenceText)).toMatchObject({ + schemaVersion: 2, + status: 'failed', + api: { requests: 1 }, + failure: { + kind: 'primary-rate-limit-exhausted', + status: 403, + url: 'https://api.github.com/graphql', + rateLimit: { + limit: '5000', + remaining: '0', + reset: '1788224400', + used: '5000', + resource: 'graphql', + }, + retryAfter: '60', + requestId: 'REQ-123', + }, + }); +}); + +it('counts an emitted GraphQL request when GitHub responds with HTTP 429', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const outputFile = join(directory, 'repos.txt'); + const evidenceFile = join(directory, 'repository-sample.json'); + writeFileSync(candidateFile, candidateSnapshot(), 'utf8'); + + await expect( + (await resolver())({ + candidateFile, + outputFile, + evidenceFile, + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => + new Response('{"message":"slow down"}', { + status: 429, + headers: { 'retry-after': '30', 'x-github-request-id': 'REQ-429' }, + })) as typeof fetch, + }), + ).rejects.toThrow('GitHub API 429'); + + expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toMatchObject({ + status: 'failed', + api: { requests: 1 }, + failure: { + kind: 'secondary-rate-limit', + status: 429, + retryAfter: '30', + requestId: 'REQ-429', + }, + }); +}); + +it.each([ + { + graphQlType: 'RATE_LIMITED', + expectedKind: 'primary-rate-limit-exhausted', + remaining: '0', + }, + { + graphQlType: 'RATE_LIMITED', + expectedKind: 'secondary-rate-limit', + remaining: '4999', + }, + { graphQlType: 'FORBIDDEN', expectedKind: 'permission-denied', remaining: '4999' }, + { graphQlType: 'UNAUTHORIZED', expectedKind: 'authentication-failed', remaining: '4999' }, +])( + 'classifies a GraphQL 200 $graphQlType error', + async ({ graphQlType, expectedKind, remaining }) => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const evidenceFile = join(directory, 'repository-sample.json'); + writeFileSync(candidateFile, candidateSnapshot(), 'utf8'); + + await expect( + (await resolver())({ + candidateFile, + outputFile: join(directory, 'repos.txt'), + evidenceFile, + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => + Response.json( + { + data: null, + errors: [{ type: graphQlType, message: 'controlled GraphQL failure' }], + }, + { + headers: { + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': remaining, + 'x-ratelimit-reset': '1788224400', + 'x-ratelimit-used': remaining === '0' ? '5000' : '1', + 'x-ratelimit-resource': 'graphql', + 'x-github-request-id': `GRAPHQL-${graphQlType}`, + }, + }, + )) as typeof fetch, + }), + ).rejects.toThrow(/GitHub GraphQL/); + + expect(JSON.parse(readFileSync(evidenceFile, 'utf8'))).toMatchObject({ + status: 'failed', + api: { requests: 1 }, + failure: { + kind: expectedKind, + status: 200, + requestId: `GRAPHQL-${graphQlType}`, + }, + }); + }, +); + +it('rejects a snapshot whose ordered universe does not reproduce its ranked strata', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const snapshot = JSON.parse(candidateSnapshot()) as { + orderedCandidates: Array>; + }; + snapshot.orderedCandidates.reverse(); + writeFileSync(candidateFile, `${JSON.stringify(snapshot)}\n`, 'utf8'); + let called = false; + + await expect( + (await resolver())({ + candidateFile, + outputFile: join(directory, 'repos.txt'), + evidenceFile: join(directory, 'repository-sample.json'), + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => { + called = true; + return new Response('unexpected'); + }) as typeof fetch, + }), + ).rejects.toThrow('candidate snapshot ordering did not match its ranked strata'); + expect(called).toBe(false); +}); + +it('rejects a snapshot with more than 100 candidates in one Search stratum', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const snapshot = JSON.parse(candidateSnapshot()) as { + strata: Array<{ candidates: Array> }>; + }; + const [typescript, javascript] = snapshot.strata; + if (typescript === undefined || javascript === undefined) { + throw new Error('candidate snapshot test fixture was incomplete'); + } + typescript.candidates = Array.from({ length: 101 }, (_, index) => ({ + rank: index + 1, + repository: `typescript/project-${index}`, + stars: 1_000 - index, + })); + javascript.candidates = []; + writeFileSync(candidateFile, `${JSON.stringify(snapshot)}\n`, 'utf8'); + let called = false; + + await expect( + (await resolver())({ + candidateFile, + outputFile: join(directory, 'repos.txt'), + evidenceFile: join(directory, 'repository-sample.json'), + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => { + called = true; + return new Response('unexpected'); + }) as typeof fetch, + }), + ).rejects.toThrow(/stratum exceeded 100 candidates/); + expect(called).toBe(false); +}); + +it('rejects a snapshot whose raw Search candidate budget exceeds 200', async () => { + const directory = temporaryDirectory(); + const candidateFile = join(directory, 'repository-candidates.json'); + const snapshot = JSON.parse(candidateSnapshot()) as { + strata: Array<{ candidates: Array> }>; + }; + const [typescript, javascript] = snapshot.strata; + if (typescript === undefined || javascript === undefined) { + throw new Error('candidate snapshot test fixture was incomplete'); + } + typescript.candidates = Array.from({ length: 101 }, (_, index) => ({ + rank: index + 1, + repository: `typescript/project-${index}`, + stars: 2_000 - index, + })); + javascript.candidates = Array.from({ length: 100 }, (_, index) => ({ + rank: index + 1, + repository: `javascript/project-${index}`, + stars: 1_000 - index, + })); + writeFileSync(candidateFile, `${JSON.stringify(snapshot)}\n`, 'utf8'); + let called = false; + + await expect( + (await resolver())({ + candidateFile, + outputFile: join(directory, 'repos.txt'), + evidenceFile: join(directory, 'repository-sample.json'), + requested: 1, + token: 'read-only-test-token', + fetchImpl: (async () => { + called = true; + return new Response('unexpected'); + }) as typeof fetch, + }), + ).rejects.toThrow(/candidate budget exceeded 200/); + expect(called).toBe(false); +}); diff --git a/tests/corpus/corpus-run.test.ts b/tests/corpus/corpus-run.test.ts index f8ed140..45c1e5d 100644 --- a/tests/corpus/corpus-run.test.ts +++ b/tests/corpus/corpus-run.test.ts @@ -1,12 +1,24 @@ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; -import { DEFAULT_CORPUS_LIMITS, type TreeEntry } from '../../tools/corpus-lib'; -import { runCorpusScan } from '../../tools/corpus-scan'; +import { type CorpusLimits, DEFAULT_CORPUS_LIMITS, type TreeEntry } from '../../tools/corpus-lib'; +import { corpusScanOptionsFromCli, runCorpusScan } from '../../tools/corpus-scan'; const COMMIT = '0123456789abcdef0123456789abcdef01234567'; -const SOURCE_COMMIT = '89abcdef0123456789abcdef0123456789abcdef'; +const SOURCE_COMMIT = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); const temporaryDirectories: string[] = []; function temporaryDirectory(): string { @@ -21,20 +33,49 @@ afterEach(() => { } }); -function fakeGitHub(tree: TreeEntry[], blobs: Record): typeof fetch { - return (async (input: string | URL | Request) => { +interface GitHubObservation { + rawUrls: string[]; + rawAuthorization: Array; + rawRedirect: Array; +} + +type RawResponseFactory = (path: string, bytes: Buffer) => Response; + +function fixtureGitBlobOid(bytes: Buffer): string { + return createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); +} + +function authorizationHeader(input: string | URL | Request, init?: RequestInit): string | null { + const inputHeaders = input instanceof Request ? input.headers : undefined; + return new Headers(init?.headers ?? inputHeaders).get('authorization'); +} + +function fakeGitHub( + tree: TreeEntry[], + blobs: Record, + observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }, + rawResponse: RawResponseFactory = (_path, bytes) => new Response(bytes), +): typeof fetch { + return (async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); if (url.includes('/git/trees/')) { return Response.json({ tree, truncated: false }); } - const sha = url.split('/').at(-1) ?? ''; - const bytes = blobs[sha]; + const parsed = new URL(url); + if (parsed.hostname !== 'raw.githubusercontent.com') { + throw new Error(`unexpected GitHub request: ${url}`); + } + observation.rawUrls.push(url); + observation.rawAuthorization.push(authorizationHeader(input, init)); + observation.rawRedirect.push(init?.redirect); + const [, owner, repository, commit, ...encodedPath] = parsed.pathname.split('/'); + if (owner !== 'example' || repository !== 'project' || commit !== COMMIT) { + return new Response('missing', { status: 404 }); + } + const path = encodedPath.map(decodeURIComponent).join('/'); + const bytes = blobs[path]; if (bytes === undefined) return new Response('missing', { status: 404 }); - return Response.json({ - encoding: 'base64', - content: bytes.toString('base64'), - size: bytes.length, - }); + return rawResponse(path, bytes); }) as typeof fetch; } @@ -54,13 +95,19 @@ function fixture(): { tree: TreeEntry[]; blobs: Record; rawScrip return { rawScript, tree: [ - { path: 'package.json', type: 'blob', mode: '100644', size: root.length, sha: 'root' }, + { + path: 'package.json', + type: 'blob', + mode: '100644', + size: root.length, + sha: fixtureGitBlobOid(root), + }, { path: 'packages/child/package.json', type: 'blob', mode: '100644', size: child.length, - sha: 'child', + sha: fixtureGitBlobOid(child), }, { path: 'node_modules/leak/package.json', @@ -70,16 +117,310 @@ function fixture(): { tree: TreeEntry[]; blobs: Record; rawScrip sha: 'excluded', }, ], - blobs: { root, child }, + blobs: { 'package.json': root, 'packages/child/package.json': child }, }; } +function sensitiveDiagnosticFixture(): { + tree: TreeEntry[]; + blobs: Record; + sentinels: string[]; +} { + const sentinels = [ + 'CORPUS_ENV_SENTINEL_a71f', + 'CORPUS_SUBSTITUTION_SENTINEL_b82e', + 'CORPUS_CMD_SENTINEL_c93d', + ]; + const root = Buffer.from( + JSON.stringify({ + name: 'sensitive-message-fixture', + scripts: { + env: `echo $${sentinels[0]}`, + substitution: `echo $(printf ${sentinels[1]})`, + cmd: `echo %${sentinels[2]}%`, + }, + }), + ); + return { + sentinels, + tree: [ + { + path: 'package.json', + type: 'blob', + mode: '100644', + size: root.length, + sha: fixtureGitBlobOid(root), + }, + ], + blobs: { 'package.json': root }, + }; +} + +function completeCandidateSnapshot(): Record { + return { + schemaVersion: 1, + status: 'complete', + method: 'popularity-strata-round-robin-v1', + strata: [ + { + id: 'typescript', + query: 'is:public language:typescript stars:>2000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: 'a'.repeat(64), + candidates: [{ rank: 1, repository: 'example/project', stars: 10_000 }], + }, + { + id: 'javascript', + query: 'is:public language:javascript stars:>5000', + sort: 'stars', + order: 'desc', + perPage: 100, + responseSha256: 'b'.repeat(64), + candidates: [], + }, + ], + orderedCandidates: [ + { position: 1, stratum: 'typescript', rank: 1, repository: 'example/project' }, + ], + }; +} + +function completeSampleEvidence( + candidateSnapshot: Buffer, + rootManifestOid: string, + rootManifestBytes: number, +): Record { + return { + schemaVersion: 2, + method: 'popularity-strata-round-robin-v1', + candidateSnapshotSha256: createHash('sha256').update(candidateSnapshot).digest('hex'), + requested: 1, + actual: 1, + candidatesConsidered: 1, + status: 'complete', + api: { + transport: 'github-graphql-batch-v1', + batchSize: 20, + requests: 1, + cost: 1, + rateLimit: { + limit: 5000, + remaining: 4999, + used: 1, + resetAt: '2026-09-01T01:00:00Z', + }, + }, + selected: [ + { + position: 1, + stratum: 'typescript', + rank: 1, + repository: 'example/project', + commit: COMMIT, + rootManifestOid, + rootManifestBytes, + }, + ], + exclusions: [], + }; +} + +function completeProvenance(data: ReturnType): { + candidateSnapshot: Buffer; + sampleEvidence: Record; +} { + const candidateSnapshot = Buffer.from( + `${JSON.stringify(completeCandidateSnapshot(), null, 2)}\n`, + 'utf8', + ); + const root = data.tree.find((entry) => entry.path === 'package.json'); + if (root?.size === undefined) throw new Error('test root manifest fixture was incomplete'); + return { + candidateSnapshot, + sampleEvidence: completeSampleEvidence(candidateSnapshot, root.sha, root.size), + }; +} + +function git(directory: string, ...arguments_: string[]): string { + return execFileSync('git', ['-C', directory, ...arguments_], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +function posixShell(): string { + if (process.platform !== 'win32') return 'sh'; + const gitExecPath = execFileSync('git', ['--exec-path'], { encoding: 'utf8' }).trim(); + const gitRoot = dirname(dirname(dirname(gitExecPath))); + return join(gitRoot, 'bin', 'sh.exe'); +} + +async function replayFixture( + options: { gitlink?: boolean; limits?: CorpusLimits; runnerOs?: string } = {}, +): Promise<{ directory: string; reproduction: string }> { + const directory = temporaryDirectory(); + const outputRoot = temporaryDirectory(); + mkdirSync(join(directory, 'tools'), { recursive: true }); + writeFileSync(join(directory, 'tools', 'corpus-scan.ts'), 'export const committed = true;\n'); + copyFileSync( + join(process.cwd(), 'tools', 'corpus-replay-check.mjs'), + join(directory, 'tools', 'corpus-replay-check.mjs'), + ); + writeFileSync(join(directory, 'package.json'), '{"packageManager":"pnpm@11.24.0"}\n'); + writeFileSync(join(directory, 'README.md'), 'committed replay fixture\n'); + writeFileSync(join(directory, '.gitignore'), 'node_modules/\n'); + writeFileSync(join(directory, '.gitattributes'), 'filtered.txt filter=replay-clean\n'); + writeFileSync(join(directory, 'filtered.txt'), 'canonical\n'); + writeFileSync(join(directory, "special ' [x] ;.txt"), 'special filename\n'); + writeFileSync(join(directory, 'executable.sh'), '#!/bin/sh\nexit 0\n'); + if (process.platform !== 'win32') { + chmodSync(join(directory, 'executable.sh'), 0o755); + symlinkSync('README.md', join(directory, 'readme-link')); + } + git(directory, 'init', '--quiet'); + git(directory, 'config', 'user.name', 'Corpus Replay Test'); + git(directory, 'config', 'user.email', 'corpus-replay@example.invalid'); + git(directory, 'config', 'core.autocrlf', 'false'); + git(directory, 'config', 'filter.replay-clean.clean', "sed 's/.*/canonical/'"); + git(directory, 'config', 'filter.replay-clean.smudge', 'cat'); + git( + directory, + 'add', + '--', + 'tools/corpus-scan.ts', + 'tools/corpus-replay-check.mjs', + 'package.json', + 'README.md', + '.gitignore', + '.gitattributes', + 'filtered.txt', + "special ' [x] ;.txt", + 'executable.sh', + ...(process.platform === 'win32' ? [] : ['readme-link']), + ); + git(directory, 'commit', '--quiet', '-m', 'fixture'); + if (options.gitlink === true) { + const submodule = join(directory, 'vendor', 'submodule'); + execFileSync('git', ['clone', '--quiet', '--no-hardlinks', directory, submodule], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + const gitlinkCommit = git(submodule, 'rev-parse', 'HEAD'); + git( + directory, + 'update-index', + '--add', + '--cacheinfo', + `160000,${gitlinkCommit},vendor/submodule`, + ); + git(directory, 'commit', '--quiet', '-m', 'add gitlink fixture'); + } + const sourceCommit = git(directory, 'rev-parse', 'HEAD'); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync( + sampleEvidenceFile, + `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`, + 'utf8', + ); + + const previousRunnerOs = process.env.RUNNER_OS; + if (options.runnerOs === undefined) delete process.env.RUNNER_OS; + else process.env.RUNNER_OS = options.runnerOs; + try { + const manifest = await runCorpusScan({ + inputFile, + outputDir: join(outputRoot, 'initial-output'), + token: 'read-only-test-token-must-not-be-persisted', + sourceCommit, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, data.blobs), + limits: options.limits, + sampleMethod: 'popularity-strata-round-robin-v1', + sampleSeed: 'candidate-seed', + candidateSnapshotFile, + sampleEvidenceFile, + sourceCheckout: directory, + }); + return { directory, reproduction: manifest.reproduction }; + } finally { + if (previousRunnerOs === undefined) delete process.env.RUNNER_OS; + else process.env.RUNNER_OS = previousRunnerOs; + } +} + +function executeReplay( + directory: string, + reproduction: string, + options: { + shellSetup?: string; + shellAfter?: string; + runnerOs?: string; + environment?: NodeJS.ProcessEnv; + token?: string; + corepackBody?: string; + pnpmBody?: string; + } = {}, +): ReturnType { + const commandDirectory = temporaryDirectory(); + const corepackPath = join(commandDirectory, 'corepack'); + const pnpmPath = join(commandDirectory, 'pnpm'); + writeFileSync(corepackPath, `#!/bin/sh\nset -eu\n${options.corepackBody ?? 'exit 0'}\n`); + writeFileSync( + pnpmPath, + `#!/bin/sh\nset -eu\n${ + options.pnpmBody ?? + [ + `if [ "\${1-}" = "exec" ]; then`, + ` printf "RUNNER_OS=%s\\n" "\${RUNNER_OS-}" > replay-observation.txt`, + ` printf "CORPUS_LIMITS_JSON=%s\\n" "\${CORPUS_LIMITS_JSON-}" >> replay-observation.txt`, + ' printf "ARGS=" >> replay-observation.txt', + ' printf "<%s>" "$@" >> replay-observation.txt', + ' printf "\\n" >> replay-observation.txt', + 'fi', + 'exit 0', + ].join('\n') + }\n`, + ); + chmodSync(corepackPath, 0o755); + chmodSync(pnpmPath, 0o755); + const shellCommandDirectory = execFileSync(posixShell(), ['-c', 'pwd'], { + cwd: commandDirectory, + encoding: 'utf8', + }).trim(); + const quotedCommandDirectory = `'${shellCommandDirectory.replaceAll("'", `'"'"'`)}'`; + const script = [ + `PATH=${quotedCommandDirectory}:"$PATH"`, + 'export PATH', + options.shellSetup ?? '', + reproduction, + options.shellAfter ?? '', + ].join('\n'); + return spawnSync(posixShell(), ['-c', script], { + cwd: directory, + encoding: 'utf8', + env: { + ...process.env, + ...options.environment, + GITHUB_TOKEN: options.token ?? 'ephemeral-replay-test-token', + ...(options.runnerOs === undefined ? {} : { RUNNER_OS: options.runnerOs }), + }, + }); +} + describe('immutable corpus run evidence', () => { it('uses canonical workspace analysis while persisting hashes instead of script source', async () => { const directory = temporaryDirectory(); const inputFile = join(directory, 'repos.txt'); const outputDir = join(directory, 'out'); const data = fixture(); + const observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }; writeFileSync(inputFile, `example/project@${COMMIT}\n`); const manifest = await runCorpusScan({ @@ -88,7 +429,7 @@ describe('immutable corpus run evidence', () => { token: 'read-only-test-token', sourceCommit: SOURCE_COMMIT, generatedAt: '2026-09-01T00:00:00.000Z', - fetchImpl: fakeGitHub(data.tree, data.blobs), + fetchImpl: fakeGitHub(data.tree, data.blobs, observation), }); expect(manifest.repositories).toMatchObject([ @@ -116,6 +457,12 @@ describe('immutable corpus run evidence', () => { expect(evidence.every((finding) => String(finding.url).includes(COMMIT))).toBe(true); expect(findingsText).not.toContain(data.rawScript); expect(findingsText).not.toContain('CORPUS_PRIVATE_SENTINEL_7f86'); + expect(observation.rawUrls).toEqual([ + `https://raw.githubusercontent.com/example/project/${COMMIT}/package.json`, + `https://raw.githubusercontent.com/example/project/${COMMIT}/packages/child/package.json`, + ]); + expect(observation.rawAuthorization).toEqual([null, null]); + expect(observation.rawRedirect).toEqual(['error', 'error']); const persisted = JSON.parse( readFileSync(join(outputDir, 'corpus-run.json'), 'utf8'), @@ -130,11 +477,43 @@ describe('immutable corpus run evidence', () => { }); }); + it('uses source-free corpus messages for arbitrary environment and substitution findings', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = sensitiveDiagnosticFixture(); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, data.blobs), + }); + + const findingsText = readFileSync(join(outputDir, 'findings.jsonl'), 'utf8'); + const findings = findingsText + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { ruleId: string; message: string }); + expect(findings.length).toBeGreaterThan(0); + for (const sentinel of data.sentinels) expect(findingsText).not.toContain(sentinel); + expect( + findings.every( + (finding) => + finding.message === `${finding.ruleId} matched a portability rule at the recorded span.`, + ), + ).toBe(true); + }); + it('makes truncation explicit and excludes the repository from promoted totals', async () => { const directory = temporaryDirectory(); const inputFile = join(directory, 'repos.txt'); const outputDir = join(directory, 'out'); const data = fixture(); + const observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }; writeFileSync(inputFile, `example/project@${COMMIT}\n`); const manifest = await runCorpusScan({ @@ -143,7 +522,7 @@ describe('immutable corpus run evidence', () => { token: 'read-only-test-token', sourceCommit: SOURCE_COMMIT, generatedAt: '2026-09-01T00:00:00.000Z', - fetchImpl: fakeGitHub(data.tree, data.blobs), + fetchImpl: fakeGitHub(data.tree, data.blobs, observation), limits: { ...DEFAULT_CORPUS_LIMITS, maxManifests: 1 }, }); @@ -157,5 +536,1351 @@ describe('immutable corpus run evidence', () => { scripts: 0, findings: 0, }); + expect(readFileSync(join(outputDir, 'findings.jsonl'), 'utf8')).toBe(''); + expect(observation.rawUrls).toEqual([]); }); + + it.each([ + { name: 'missing', payload: (tree: TreeEntry[]) => ({ tree }) }, + { name: 'string', payload: (tree: TreeEntry[]) => ({ tree, truncated: 'false' }) }, + { name: 'null', payload: (tree: TreeEntry[]) => ({ tree, truncated: null }) }, + ])('fails closed when tree truncated is $name', async ({ payload }) => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + const observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }; + const rawApi = fakeGitHub(data.tree, data.blobs, observation); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async (input: string | URL | Request, init?: RequestInit) => + String(input).includes('/git/trees/') + ? Response.json(payload(data.tree)) + : rawApi(input, init)) as typeof fetch, + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; failure?: { kind?: string } }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + failure: { kind: 'response-invalid' }, + }); + expect(observation.rawUrls).toEqual([]); + }); + + it('classifies a truncated GitHub tree without a root manifest as truncated', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + let rawCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + const manifest = await runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async (input: string | URL | Request) => { + if (String(input).includes('/git/trees/')) { + return Response.json({ + tree: data.tree.filter((entry) => entry.path !== 'package.json'), + truncated: true, + }); + } + rawCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + }); + + expect(manifest.repositories[0]).toMatchObject({ + status: 'truncated', + truncations: ['github-tree-truncated'], + }); + expect(manifest.promotedTotals.workspaceFull.repositories).toBe(0); + expect(rawCalled).toBe(false); + }); + + it('fails closed when raw bytes do not match the immutable tree blob OID', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + const tamperedRoot = Buffer.from(data.blobs['package.json'] as Buffer); + const rootNameOffset = tamperedRoot.indexOf('root'); + tamperedRoot.write('soot', rootNameOffset, 'utf8'); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, { + ...data.blobs, + 'package.json': tamperedRoot, + }), + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: 'package.json: raw bytes did not match the immutable tree Git blob OID', + }); + }); + + it('fails closed when raw byte length does not match the immutable tree entry', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + const tree = data.tree.map((entry) => + entry.path === 'package.json' ? { ...entry, size: (entry.size as number) + 1 } : entry, + ); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(tree, data.blobs), + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: 'package.json: raw byte length did not match the immutable tree entry', + }); + }); + + it('rejects a non-SHA-1 tree blob OID before trusting raw content', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + const tree = data.tree.map((entry) => + entry.path === 'package.json' ? { ...entry, sha: 'a'.repeat(64) } : entry, + ); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(tree, data.blobs), + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: 'package.json: immutable tree Git blob OID was not 40 lowercase hex characters', + }); + }); + + it('cancels raw streaming as soon as bytes exceed the immutable tree size', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const data = fixture(); + let pulls = 0; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub( + data.tree, + data.blobs, + { rawUrls: [], rawAuthorization: [], rawRedirect: [] }, + (path, bytes) => { + if (path !== 'package.json') return new Response(bytes); + const body = new ReadableStream( + { + pull(controller) { + pulls += 1; + if (pulls === 1) controller.enqueue(bytes); + else if (pulls === 2) controller.enqueue(Uint8Array.of(0)); + else controller.error(new Error('unbounded raw read sentinel')); + }, + }, + { highWaterMark: 0 }, + ); + return new Response(body); + }, + ), + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: 'package.json: raw byte length did not match the immutable tree entry', + }); + expect(pulls).toBe(2); + }); + + it('persists rate-limit classification and response headers for tree API failures', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async () => + new Response(JSON.stringify({ message: 'API rate limit exceeded' }), { + status: 403, + headers: { + 'content-type': 'application/json', + 'retry-after': '60', + 'x-github-request-id': 'TEST:RATE:123', + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': '1788213600', + 'x-ratelimit-resource': 'core', + 'x-ratelimit-used': '5000', + }, + })) as typeof fetch, + }), + ).rejects.toThrow('one or more repositories failed'); + + const persistedText = readFileSync(join(outputDir, 'corpus-run.json'), 'utf8'); + const persisted = JSON.parse(persistedText) as { + repositories: Array<{ failure?: unknown }>; + }; + expect(persisted.repositories[0]?.failure).toEqual({ + kind: 'primary-rate-limit-exhausted', + status: 403, + url: `https://api.github.com/repos/example/project/git/trees/${COMMIT}?recursive=1`, + rateLimit: { + limit: '5000', + remaining: '0', + reset: '1788213600', + used: '5000', + resource: 'core', + }, + retryAfter: '60', + requestId: 'TEST:RATE:123', + }); + expect(persistedText).not.toContain('read-only-test-token'); + }); + + it('preserves response headers when tree JSON or shape is invalid', async () => { + for (const body of ['not-json', JSON.stringify({ truncated: false })]) { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async () => + new Response(body, { + status: 200, + headers: { + 'x-github-request-id': 'TEST:INVALID:123', + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': '4999', + 'x-ratelimit-reset': '1788213600', + 'x-ratelimit-resource': 'core', + 'x-ratelimit-used': '1', + }, + })) as typeof fetch, + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ failure?: unknown }>; + }; + expect(persisted.repositories[0]?.failure).toEqual({ + kind: 'response-invalid', + status: 200, + url: `https://api.github.com/repos/example/project/git/trees/${COMMIT}?recursive=1`, + rateLimit: { + limit: '5000', + remaining: '4999', + reset: '1788213600', + used: '1', + resource: 'core', + }, + retryAfter: null, + requestId: 'TEST:INVALID:123', + }); + } + }); + + it('hashes the complete candidate snapshot and sample evidence into the run manifest', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + const candidateSnapshot = provenance.candidateSnapshot; + const candidateSnapshotSha256 = createHash('sha256').update(candidateSnapshot).digest('hex'); + const sampleEvidence = Buffer.from( + `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`, + 'utf8', + ); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, candidateSnapshot); + writeFileSync(sampleEvidenceFile, sampleEvidence); + + const manifest = await runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, data.blobs), + sampleMethod: 'popularity-strata-round-robin-v1', + candidateSnapshotFile, + sampleEvidenceFile, + }); + + const sampleEvidenceSha256 = createHash('sha256').update(sampleEvidence).digest('hex'); + expect(manifest.sampling).toEqual({ + method: 'popularity-strata-round-robin-v1', + seed: 'none', + candidateSnapshotSha256, + sampleEvidenceSha256, + }); + expect(manifest.artifactSha256).toMatchObject({ + 'repository-candidates.json': candidateSnapshotSha256, + 'repository-sample.json': sampleEvidenceSha256, + }); + }); + + it('emits a safe path-independent replay command for the complete provenance contract', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos copy.txt'); + const outputDir = join(directory, 'initial output'); + const candidateSnapshotFile = join(directory, "repository candidate's.json"); + const sampleEvidenceFile = join(directory, 'repository sample;ignored.json'); + const data = fixture(); + const provenance = completeProvenance(data); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync( + sampleEvidenceFile, + `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`, + 'utf8', + ); + + const manifest = await runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token-must-not-be-persisted', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: fakeGitHub(data.tree, data.blobs), + sampleMethod: 'popularity-strata-round-robin-v1', + sampleSeed: 'candidate-seed', + candidateSnapshotFile, + sampleEvidenceFile, + }); + + expect(manifest.reproduction).toMatch(/^\(set \+a && SCRIPTSPECT_REPLAY_CHECK='/u); + expect(manifest.reproduction.split('node --input-type=module -e')).toHaveLength(3); + for (const digest of [ + manifest.inputSha256, + manifest.sampling.candidateSnapshotSha256, + manifest.sampling.sampleEvidenceSha256, + ]) { + expect(digest).toMatch(/^[a-f0-9]{64}$/u); + expect(manifest.reproduction.split(String(digest))).toHaveLength(3); + } + expect(manifest.reproduction).toContain('test ! -e node_modules'); + expect(manifest.reproduction).not.toMatch(/\bgit\b[^&]*\bcheckout\b/u); + expect(manifest.reproduction).not.toContain('tools/corpus-replay-check.mjs'); + expect(manifest.reproduction).toContain( + `CORPUS_LIMITS_JSON='{"maxTreeEntries":20000,"maxManifests":500,"maxDepth":12,"maxFileBytes":1048576,"maxTotalBytes":10485760}'`, + ); + expect(manifest.reproduction).toContain( + `pnpm exec tsx tools/corpus-scan.ts 'repos copy.txt' 'corpus-reproduction-${SOURCE_COMMIT}'`, + ); + expect(manifest.reproduction).not.toContain('read-only-test-token-must-not-be-persisted'); + expect(manifest.reproduction).not.toContain(directory); + }); + + it('does not execute a local clean filter while validating a replay checkout', async () => { + const replay = await replayFixture(); + const candidateEvidence = join(replay.directory, 'repository-candidates.json'); + const originalEvidence = readFileSync(candidateEvidence); + git( + replay.directory, + 'config', + 'filter.replay-clean.clean', + 'sh -c \'printf tampered > repository-candidates.json; cat >/dev/null; printf \\"canonical\\n\\"\'', + ); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).toBe(0); + expect(readFileSync(candidateEvidence)).toEqual(originalEvidence); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(true); + }); + + it('requires callers to prepare the exact commit without invoking checkout hooks', async () => { + const replay = await replayFixture(); + const hook = join(replay.directory, '.git', 'hooks', 'post-checkout'); + writeFileSync(hook, '#!/bin/sh\nprintf hook-ran > checkout-hook-ran\n'); + chmodSync(hook, 0o755); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).toBe(0); + expect(existsSync(join(replay.directory, 'checkout-hook-ran'))).toBe(false); + expect(replay.reproduction).not.toMatch(/\bgit\b[^&]*\bcheckout\b/u); + }); + + it('fails closed on a different clean HEAD without changing the caller checkout', async () => { + const replay = await replayFixture(); + writeFileSync(join(replay.directory, 'later-commit.txt'), 'later clean commit\n'); + git(replay.directory, 'add', '--', 'later-commit.txt'); + git(replay.directory, 'commit', '--quiet', '-m', 'later fixture commit'); + const laterHead = git(replay.directory, 'rev-parse', 'HEAD'); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(git(replay.directory, 'rev-parse', 'HEAD')).toBe(laterHead); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('keeps replay environment changes inside a subshell', async () => { + const replay = await replayFixture(); + + const result = executeReplay(replay.directory, replay.reproduction, { + shellAfter: `printf "GITHUB_TOKEN=%s\\nRUNNER_OS=%s\\n" "\${GITHUB_TOKEN-}" "\${RUNNER_OS-}" > caller-environment.txt`, + runnerOs: 'CallerRunner', + }); + + expect(result.status, String(result.stderr)).toBe(0); + expect(readFileSync(join(replay.directory, 'caller-environment.txt'), 'utf8')).toBe( + 'GITHUB_TOKEN=ephemeral-replay-test-token\nRUNNER_OS=CallerRunner\n', + ); + }); + + it('rejects a pre-existing dependency install before running package tooling', async () => { + const replay = await replayFixture(); + mkdirSync(join(replay.directory, 'node_modules', 'malicious'), { recursive: true }); + writeFileSync( + join(replay.directory, 'node_modules', 'malicious', 'index.js'), + 'throw new Error("pre-existing dependency executed");\n', + ); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('rejects a broken node_modules link before running package tooling', async () => { + const replay = await replayFixture(); + const nodeModules = join(replay.directory, 'node_modules'); + if (process.platform === 'win32') { + symlinkSync(join(replay.directory, 'missing-dependency-target'), nodeModules, 'junction'); + } else { + symlinkSync('missing-dependency-target', nodeModules); + } + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('allows the frozen install to create an ignored dependency directory', async () => { + const replay = await replayFixture(); + const pnpmBody = [ + `if [ "\${1-}" = "install" ]; then`, + ' mkdir -p node_modules/installed', + ' printf installed > node_modules/installed/package.json', + `elif [ "\${1-}" = "exec" ]; then`, + ' printf scanner-ran > replay-observation.txt', + 'fi', + 'exit 0', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { pnpmBody }); + + expect(result.status, String(result.stderr)).toBe(0); + expect(readFileSync(join(replay.directory, 'replay-observation.txt'), 'utf8')).toBe( + 'scanner-ran', + ); + }); + + it('withholds the token from package setup and injects it only into the scanner command', async () => { + const replay = await replayFixture(); + const pnpmBody = [ + `if [ "\${1-}" = "install" ]; then`, + ' if env | grep -q "^GITHUB_TOKEN=" || env | grep -q "^SCRIPTSPECT_REPLAY_TOKEN="; then', + ' printf leaked > package-setup-token-leak.txt', + ' exit 97', + ' fi', + `elif [ "\${1-}" = "exec" ]; then`, + ' env | grep "^GITHUB_TOKEN=" > scanner-token.txt', + 'fi', + 'exit 0', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { + shellSetup: 'set -a', + pnpmBody, + }); + + expect(result.status, String(result.stderr)).toBe(0); + expect(existsSync(join(replay.directory, 'package-setup-token-leak.txt'))).toBe(false); + expect(readFileSync(join(replay.directory, 'scanner-token.txt'), 'utf8')).toBe( + 'GITHUB_TOKEN=ephemeral-replay-test-token\n', + ); + }); + + it.each(['corepack', 'pnpm'])( + 'bypasses a caller-defined %s function that can read shell-local replay state', + async (command) => { + const replay = await replayFixture(); + const shellSetup = [ + `${command}() {`, + ` if [ -n "\${SCRIPTSPECT_REPLAY_TOKEN-}" ]; then`, + ' printf leaked > caller-function-token-leak.txt', + ' return 97', + ' fi', + ' return 96', + '}', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + + expect(result.status, String(result.stderr)).toBe(0); + expect(existsSync(join(replay.directory, 'caller-function-token-leak.txt'))).toBe(false); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(true); + }, + ); + + it('rejects an empty token before package setup or output creation and remains retryable', async () => { + const replay = await replayFixture(); + const markerDirectory = temporaryDirectory(); + const packageSetupMarker = join(markerDirectory, 'package-setup-called.txt'); + const outputDirectory = join( + replay.directory, + `corpus-reproduction-${git(replay.directory, 'rev-parse', 'HEAD')}`, + ); + const markerBody = 'printf called > "$PACKAGE_SETUP_MARKER"\nexit 0'; + + const missingToken = executeReplay(replay.directory, replay.reproduction, { + token: '', + environment: { PACKAGE_SETUP_MARKER: packageSetupMarker.replaceAll('\\', '/') }, + corepackBody: markerBody, + pnpmBody: markerBody, + }); + + expect(missingToken.status, String(missingToken.stderr)).not.toBe(0); + expect(existsSync(packageSetupMarker)).toBe(false); + expect(existsSync(outputDirectory)).toBe(false); + + const retry = executeReplay(replay.directory, replay.reproduction); + expect(retry.status, String(retry.stderr)).toBe(0); + }); + + it('ignores inherited Git repository redirection while validating the checkout', async () => { + const replay = await replayFixture(); + const redirectedWorktree = temporaryDirectory(); + for (const evidenceName of [ + 'repos.txt', + 'repository-candidates.json', + 'repository-sample.json', + ]) { + copyFileSync(join(replay.directory, evidenceName), join(redirectedWorktree, evidenceName)); + } + writeFileSync( + join(replay.directory, 'unexpected-untracked.ts'), + 'export const hidden = true;\n', + ); + + const result = executeReplay(replay.directory, replay.reproduction, { + environment: { GIT_WORK_TREE: redirectedWorktree }, + }); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it.each(['repos.txt', 'repository-candidates.json', 'repository-sample.json'])( + 'rejects changed %s evidence before the frozen install', + async (evidenceName) => { + const replay = await replayFixture(); + writeFileSync(join(replay.directory, evidenceName), `tampered ${evidenceName}\n`); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }, + ); + + it.each(['repos.txt', 'repository-candidates.json', 'repository-sample.json'])( + 'rejects lifecycle changes to %s evidence after the frozen install', + async (evidenceName) => { + const replay = await replayFixture(); + const pnpmBody = [ + `if [ "\${1-}" = "install" ]; then`, + ` printf tampered > '${evidenceName}'`, + `elif [ "\${1-}" = "exec" ]; then`, + ' printf scanner-ran > replay-observation.txt', + 'fi', + 'exit 0', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { pnpmBody }); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }, + ); + + it('rejects a non-regular evidence input before the frozen install', async () => { + const replay = await replayFixture(); + const evidencePath = join(replay.directory, 'repository-sample.json'); + rmSync(evidencePath); + mkdirSync(evidencePath); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('fails closed before scanning a checkout with dirty tracked, staged, or untracked files', async () => { + const cases = [ + { + name: 'tracked worktree', + dirty: (directory: string) => { + writeFileSync(join(directory, 'tools', 'corpus-scan.ts'), 'export const dirty = true;\n'); + }, + }, + { + name: 'staged index', + dirty: (directory: string) => { + writeFileSync( + join(directory, 'tools', 'corpus-scan.ts'), + 'export const staged = true;\n', + ); + git(directory, 'add', '--', 'tools/corpus-scan.ts'); + }, + }, + { + name: 'nonignored untracked code', + dirty: (directory: string) => { + writeFileSync(join(directory, 'rogue-config.ts'), 'export const rogue = true;\n'); + }, + }, + ]; + + for (const testCase of cases) { + const replay = await replayFixture(); + testCase.dirty(replay.directory); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, `${testCase.name}: ${result.stderr}`).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + } + }, 15_000); + + it.each([ + ['assume-unchanged', '--assume-unchanged'], + ['skip-worktree', '--skip-worktree'], + ])('fails closed when %s hides a dirty tracked scanner', async (_name, indexFlag) => { + const replay = await replayFixture(); + git(replay.directory, 'update-index', indexFlag, '--', 'tools/corpus-scan.ts'); + writeFileSync( + join(replay.directory, 'tools', 'corpus-scan.ts'), + 'export const hiddenDirtyScanner = true;\n', + ); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('fails closed when fsmonitor-valid hides a dirty tracked file', async () => { + const replay = await replayFixture(); + git(replay.directory, 'config', 'core.fsmonitor', 'true'); + writeFileSync(join(replay.directory, 'README.md'), 'dirty content hidden by fsmonitor\n'); + git(replay.directory, 'update-index', '--fsmonitor-valid', '--', 'README.md'); + expect(git(replay.directory, 'ls-files', '-v', '--', 'README.md')).toBe('H README.md'); + expect(git(replay.directory, 'ls-files', '-f', '--', 'README.md')).toBe('h README.md'); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('fails closed when a clean filter hides different worktree bytes', async () => { + const replay = await replayFixture(); + writeFileSync(join(replay.directory, 'filtered.txt'), 'malicious worktree bytes\n'); + git(replay.directory, 'add', '--', 'filtered.txt'); + git(replay.directory, 'diff', '--quiet', '--', 'filtered.txt'); + git(replay.directory, 'diff', '--cached', '--quiet', '--', 'filtered.txt'); + expect(git(replay.directory, 'status', '--porcelain=v1', '--', 'filtered.txt')).toBe(''); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('propagates a Git metadata failure instead of treating it as clean', async () => { + const replay = await replayFixture(); + git(replay.directory, 'config', 'core.repositoryFormatVersion', '999'); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it('rejects a gitlink even when its checked-out submodule is clean', async () => { + const replay = await replayFixture({ gitlink: true }); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }); + + it.skipIf(process.platform === 'win32')( + 'fails closed when core.filemode hides an executable-bit mismatch', + async () => { + const replay = await replayFixture(); + git(replay.directory, 'config', 'core.filemode', 'false'); + chmodSync(join(replay.directory, 'executable.sh'), 0o644); + git(replay.directory, 'diff', '--quiet', '--', 'executable.sh'); + + const result = executeReplay(replay.directory, replay.reproduction); + + expect(result.status, String(result.stderr)).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + }, + ); + + it('fails closed before scanning under a different Node runtime', async () => { + const cases = [ + { name: 'version', version: 'v0.0.0', platform: process.platform, arch: process.arch }, + { name: 'platform', version: process.version, platform: 'foreign-os', arch: process.arch }, + { + name: 'architecture', + version: process.version, + platform: process.platform, + arch: 'foreign-arch', + }, + ]; + + for (const testCase of cases) { + const replay = await replayFixture(); + const shellSetup = [ + 'node() {', + ` if [ "$1" = "--version" ]; then printf '%s\\n' '${testCase.version}'`, + ` elif [ "$1" = "-p" ] && [ "$2" = "process.platform" ]; then printf '%s\\n' '${testCase.platform}'`, + ` elif [ "$1" = "-p" ] && [ "$2" = "process.arch" ]; then printf '%s\\n' '${testCase.arch}'`, + ' else return 64', + ' fi', + '}', + ].join('\n'); + + const result = executeReplay(replay.directory, replay.reproduction, { shellSetup }); + + expect(result.status, `${testCase.name}: ${result.stderr}`).not.toBe(0); + expect(existsSync(join(replay.directory, 'replay-observation.txt'))).toBe(false); + } + }); + + it('restores defined and unset RUNNER_OS state plus exact default and custom limits', async () => { + const customLimits: CorpusLimits = { + maxTotalBytes: 50_000, + maxFileBytes: 4_000, + maxDepth: 3, + maxManifests: 2, + maxTreeEntries: 100, + }; + const cases = [ + { + name: 'unset runner and default limits', + replay: await replayFixture(), + outerRunnerOs: 'ConflictingRunner', + expectedRunnerOs: '', + expectedLimits: DEFAULT_CORPUS_LIMITS, + }, + { + name: 'defined runner and custom limits', + replay: await replayFixture({ limits: customLimits, runnerOs: 'RecordedRunner' }), + outerRunnerOs: 'ConflictingRunner', + expectedRunnerOs: 'RecordedRunner', + expectedLimits: customLimits, + }, + ]; + + for (const testCase of cases) { + const result = executeReplay(testCase.replay.directory, testCase.replay.reproduction, { + runnerOs: testCase.outerRunnerOs, + }); + + expect(result.status, `${testCase.name}: ${result.stderr}`).toBe(0); + const observation = readFileSync( + join(testCase.replay.directory, 'replay-observation.txt'), + 'utf8', + ); + expect(observation).toContain(`RUNNER_OS=${testCase.expectedRunnerOs}\n`); + expect(observation).toContain( + `CORPUS_LIMITS_JSON=${JSON.stringify({ + maxTreeEntries: testCase.expectedLimits.maxTreeEntries, + maxManifests: testCase.expectedLimits.maxManifests, + maxDepth: testCase.expectedLimits.maxDepth, + maxFileBytes: testCase.expectedLimits.maxFileBytes, + maxTotalBytes: testCase.expectedLimits.maxTotalBytes, + })}\n`, + ); + expect(observation).toContain( + ``, + ); + expect(testCase.replay.reproduction).not.toContain('ephemeral-replay-test-token'); + expect(testCase.replay.reproduction).not.toContain(testCase.replay.directory); + } + }); + + it('parses default and exact custom limits from the scanner CLI environment', () => { + const baseEnvironment = { + GITHUB_TOKEN: 'read-only-test-token', + SCRIPTSPECT_SOURCE_COMMIT: SOURCE_COMMIT, + }; + const defaults = corpusScanOptionsFromCli(['repos.txt', 'default-output'], baseEnvironment); + expect(defaults).toMatchObject({ + inputFile: 'repos.txt', + outputDir: 'default-output', + limits: DEFAULT_CORPUS_LIMITS, + }); + + const customJson = + '{"maxTotalBytes":50000,"maxDepth":3,"maxTreeEntries":100,"maxFileBytes":4000,"maxManifests":2}'; + const custom = corpusScanOptionsFromCli(['repos.txt', 'custom-output'], { + ...baseEnvironment, + CORPUS_LIMITS_JSON: customJson, + }); + expect(JSON.stringify(custom.limits)).toBe( + '{"maxTreeEntries":100,"maxManifests":2,"maxDepth":3,"maxFileBytes":4000,"maxTotalBytes":50000}', + ); + + for (const malformed of [ + '{}', + '[]', + '{"maxTreeEntries":1,"maxManifests":2,"maxDepth":3,"maxFileBytes":4,"maxTotalBytes":5,"extra":6}', + '{"maxTreeEntries":1,"maxManifests":2,"maxDepth":-1,"maxFileBytes":4,"maxTotalBytes":5}', + '{"maxTreeEntries":1,"maxManifests":2,"maxDepth":3.5,"maxFileBytes":4,"maxTotalBytes":5}', + 'not-json', + ]) { + expect(() => + corpusScanOptionsFromCli(['repos.txt'], { + ...baseEnvironment, + CORPUS_LIMITS_JSON: malformed, + }), + ).toThrow(/CORPUS_LIMITS_JSON/); + } + }); + + it('rejects mismatched sample provenance before making a network request', async () => { + const data = fixture(); + const provenance = completeProvenance(data); + const cases = [ + { + name: 'status', + mutate: (evidence: Record) => { + evidence.status = 'failed'; + }, + }, + { + name: 'method', + mutate: (evidence: Record) => { + evidence.method = 'wrong-method'; + }, + }, + { + name: 'candidate snapshot digest', + mutate: (evidence: Record) => { + evidence.candidateSnapshotSha256 = '0'.repeat(64); + }, + }, + { + name: 'selected locator sequence', + mutate: (evidence: Record) => { + const selected = evidence.selected as Array>; + const first = selected[0]; + if (first === undefined) throw new Error('sample evidence test fixture was incomplete'); + first.repository = 'another/project'; + }, + }, + ]; + + for (const testCase of cases) { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const evidence = structuredClone(provenance.sampleEvidence); + testCase.mutate(evidence); + let networkCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(evidence)}\n`, 'utf8'); + const upstream = fakeGitHub(data.tree, data.blobs); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + generatedAt: '2026-09-01T00:00:00.000Z', + fetchImpl: (async (input: string | URL | Request, init?: RequestInit) => { + networkCalled = true; + return upstream(input, init); + }) as typeof fetch, + sampleMethod: 'popularity-strata-round-robin-v1', + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/corpus sample evidence/); + expect(networkCalled, testCase.name).toBe(false); + } + }); + + it('rejects malformed candidate snapshot structure before making a network request', async () => { + const data = fixture(); + const cases = [ + { + name: 'schema', + mutate: (snapshot: Record) => { + snapshot.schemaVersion = 99; + }, + }, + { + name: 'stratum metadata', + mutate: (snapshot: Record) => { + const strata = snapshot.strata as Array>; + const first = strata[0]; + if (first === undefined) + throw new Error('candidate snapshot test fixture was incomplete'); + first.query = 'language:typescript'; + }, + }, + { + name: 'ordered universe', + mutate: (snapshot: Record) => { + snapshot.orderedCandidates = []; + }, + }, + ]; + + for (const testCase of cases) { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const snapshot = completeCandidateSnapshot(); + testCase.mutate(snapshot); + const candidateSnapshot = Buffer.from(`${JSON.stringify(snapshot)}\n`, 'utf8'); + const root = data.tree.find((entry) => entry.path === 'package.json'); + if (root?.size === undefined) throw new Error('test root manifest fixture was incomplete'); + const evidence = completeSampleEvidence(candidateSnapshot, root.sha, root.size); + let networkCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(evidence)}\n`, 'utf8'); + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/candidate snapshot/); + expect(networkCalled, testCase.name).toBe(false); + } + }); + + it('rejects incomplete sample evidence structure before making a network request', async () => { + const data = fixture(); + const provenance = completeProvenance(data); + const cases = [ + { + name: 'schemaVersion', + mutate: (evidence: Record) => { + delete evidence.schemaVersion; + }, + }, + { + name: 'requested/actual contract', + mutate: (evidence: Record) => { + evidence.requested = 2; + }, + }, + { + name: 'api', + mutate: (evidence: Record) => { + delete evidence.api; + }, + }, + { + name: 'exclusions', + mutate: (evidence: Record) => { + evidence.exclusions = [{}]; + }, + }, + { + name: 'root manifest identity', + mutate: (evidence: Record) => { + const selected = evidence.selected as Array>; + const first = selected[0]; + if (first === undefined) throw new Error('sample evidence test fixture was incomplete'); + delete first.rootManifestOid; + }, + }, + ]; + + for (const testCase of cases) { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const evidence = structuredClone(provenance.sampleEvidence); + testCase.mutate(evidence); + let networkCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(evidence)}\n`, 'utf8'); + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/corpus sample evidence/); + expect(networkCalled, testCase.name).toBe(false); + } + }); + + it.each([ + { + name: 'OID', + mutate: (selected: Record) => { + selected.rootManifestOid = 'f'.repeat(40); + }, + }, + { + name: 'byte size', + mutate: (selected: Record) => { + selected.rootManifestBytes = Number(selected.rootManifestBytes) + 1; + }, + }, + ])('cross-checks the sample root manifest $name against the REST tree', async ({ mutate }) => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const outputDir = join(directory, 'out'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + const selected = (provenance.sampleEvidence.selected as Array>)[0]; + if (selected === undefined) throw new Error('sample evidence test fixture was incomplete'); + mutate(selected); + const observation: GitHubObservation = { rawUrls: [], rawAuthorization: [], rawRedirect: [] }; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(provenance.sampleEvidence)}\n`, 'utf8'); + + await expect( + runCorpusScan({ + inputFile, + outputDir, + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: fakeGitHub(data.tree, data.blobs, observation), + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow('one or more repositories failed'); + + const persisted = JSON.parse(readFileSync(join(outputDir, 'corpus-run.json'), 'utf8')) as { + repositories: Array<{ status: string; error?: string }>; + }; + expect(persisted.repositories[0]).toMatchObject({ + status: 'failed', + error: expect.stringMatching(/root package\.json.*sample evidence/), + }); + expect(observation.rawUrls).toEqual([]); + }); + + it('requires candidate and sample evidence files to be provided together', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const data = fixture(); + const provenance = completeProvenance(data); + let networkCalled = false; + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + }), + ).rejects.toThrow(/candidate snapshot and sample evidence must be provided together/); + expect(networkCalled).toBe(false); + }); + + it('rejects evidence paths that collapse to duplicate replay basenames before network access', async () => { + const directory = temporaryDirectory(); + const inputDirectory = join(directory, 'input'); + const candidateDirectory = join(directory, 'candidate'); + const sampleDirectory = join(directory, 'sample'); + mkdirSync(inputDirectory); + mkdirSync(candidateDirectory); + mkdirSync(sampleDirectory); + const inputFile = join(inputDirectory, 'evidence.json'); + const candidateSnapshotFile = join(candidateDirectory, 'evidence.json'); + const sampleEvidenceFile = join(sampleDirectory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`); + let networkCalled = false; + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/unique.*basename|basename.*unique/iu); + expect(networkCalled).toBe(false); + }); + + it('rejects an evidence basename tracked at the recorded source commit before network access', async () => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'package.json'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + const sourceCommit = git(process.cwd(), 'rev-parse', 'HEAD'); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`); + let networkCalled = false; + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/basename.*tracked.*source commit|tracked.*source commit.*basename/iu); + expect(networkCalled).toBe(false); + }); + + it.each([ + ['repository list', 'findings.jsonl', 'repository-candidates.json', 'repository-sample.json'], + ['candidate snapshot', 'repos.txt', 'summary.md', 'repository-sample.json'], + ['sample evidence', 'repos.txt', 'repository-candidates.json', 'corpus-run.json'], + [ + 'repository list', + `corpus-reproduction-${SOURCE_COMMIT}`, + 'repository-candidates.json', + 'repository-sample.json', + ], + ['repository list', 'node_modules', 'repository-candidates.json', 'repository-sample.json'], + ])( + 'rejects a %s basename reserved for corpus output before network access', + async (_role, inputName, candidateName, sampleName) => { + const directory = temporaryDirectory(); + const inputFile = join(directory, inputName); + const candidateSnapshotFile = join(directory, candidateName); + const sampleEvidenceFile = join(directory, sampleName); + const data = fixture(); + const provenance = completeProvenance(data); + writeFileSync(inputFile, `example/project@${COMMIT}\n`); + writeFileSync(candidateSnapshotFile, provenance.candidateSnapshot); + writeFileSync(sampleEvidenceFile, `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`); + let networkCalled = false; + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(/reserved.*corpus output|corpus output.*reserved/iu); + expect(networkCalled).toBe(false); + }, + ); + + it('applies replay basename collision checks to CLI-derived options', () => { + const environment = { + GITHUB_TOKEN: 'read-only-test-token', + SCRIPTSPECT_SOURCE_COMMIT: SOURCE_COMMIT, + CORPUS_CANDIDATE_SNAPSHOT: 'candidate/evidence.json', + CORPUS_SAMPLE_EVIDENCE: 'sample/evidence.json', + }; + expect(() => corpusScanOptionsFromCli(['input/repos.txt'], environment)).toThrow( + /unique.*basename|basename.*unique/iu, + ); + expect(() => + corpusScanOptionsFromCli(['input/findings.jsonl'], { + ...environment, + CORPUS_CANDIDATE_SNAPSHOT: 'repository-candidates.json', + CORPUS_SAMPLE_EVIDENCE: 'repository-sample.json', + }), + ).toThrow(/reserved.*corpus output|corpus output.*reserved/iu); + }); + + it.each(['repository list', 'candidate snapshot', 'sample evidence'])( + 'requires the %s input to be a regular file before network access', + async (role) => { + const directory = temporaryDirectory(); + const inputFile = join(directory, 'repos.txt'); + const candidateSnapshotFile = join(directory, 'repository-candidates.json'); + const sampleEvidenceFile = join(directory, 'repository-sample.json'); + const data = fixture(); + const provenance = completeProvenance(data); + const paths = new Map([ + ['repository list', inputFile], + ['candidate snapshot', candidateSnapshotFile], + ['sample evidence', sampleEvidenceFile], + ]); + for (const [currentRole, path] of paths) { + if (currentRole === role) mkdirSync(path); + else if (currentRole === 'repository list') { + writeFileSync(path, `example/project@${COMMIT}\n`); + } else if (currentRole === 'candidate snapshot') { + writeFileSync(path, provenance.candidateSnapshot); + } else { + writeFileSync(path, `${JSON.stringify(provenance.sampleEvidence, null, 2)}\n`); + } + } + let networkCalled = false; + + await expect( + runCorpusScan({ + inputFile, + outputDir: join(directory, 'out'), + token: 'read-only-test-token', + sourceCommit: SOURCE_COMMIT, + fetchImpl: (async () => { + networkCalled = true; + return new Response('unexpected'); + }) as typeof fetch, + candidateSnapshotFile, + sampleEvidenceFile, + }), + ).rejects.toThrow(new RegExp(`${role}.*regular file`, 'iu')); + expect(networkCalled).toBe(false); + }, + ); }); diff --git a/tests/corpus/corpus-scan.test.ts b/tests/corpus/corpus-scan.test.ts index da04e9f..2192152 100644 --- a/tests/corpus/corpus-scan.test.ts +++ b/tests/corpus/corpus-scan.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { DEFAULT_CORPUS_LIMITS, + gitBlobOid, parseRepoLocator, redactCorpusText, selectCorpusFiles, @@ -9,6 +10,12 @@ import { const SHA = '0123456789abcdef0123456789abcdef01234567'; +describe('Git blob integrity', () => { + it('derives the canonical SHA-1 object ID from the exact blob bytes', () => { + expect(gitBlobOid(Buffer.from('hello\n'))).toBe('ce013625030ba8dba906f756967f9e9ca394464a'); + }); +}); + describe('immutable corpus locators', () => { it('accepts only owner/repo plus an exact 40-character commit', () => { expect(parseRepoLocator(`open-source/project@${SHA}`)).toEqual({ @@ -59,6 +66,22 @@ describe('bounded workspace manifest selection', () => { expect.arrayContaining(['manifest-limit:2', 'depth-limit:3', 'byte-limit:160']), ); }); + + it('keeps the root manifest inside the tree-entry budget even when GitHub lists it late', () => { + const tree: TreeEntry[] = [ + { path: 'a/readme.md', type: 'blob', mode: '100644', size: 20, sha: 'a' }, + { path: 'b/readme.md', type: 'blob', mode: '100644', size: 20, sha: 'b' }, + { path: 'package.json', type: 'blob', mode: '100644', size: 80, sha: 'root' }, + ]; + + const selected = selectCorpusFiles(tree, { + ...DEFAULT_CORPUS_LIMITS, + maxTreeEntries: 2, + }); + + expect(selected.files.map((entry) => entry.path)).toEqual(['package.json']); + expect(selected.truncations).toEqual(['tree-entry-limit:2']); + }); }); describe('corpus evidence redaction', () => { diff --git a/tests/workflows/ci-policy.test.ts b/tests/workflows/ci-policy.test.ts index 6e5d988..6d841c9 100644 --- a/tests/workflows/ci-policy.test.ts +++ b/tests/workflows/ci-policy.test.ts @@ -1,6 +1,5 @@ -import { execFileSync, spawnSync } from 'node:child_process'; -import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { execFileSync } from 'node:child_process'; +import { readdirSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; @@ -49,49 +48,6 @@ function workflowNames(): string[] { .sort(); } -function runCorpusSelection(requested: number, candidates: string[]) { - const selectionStep = allSteps(workflow('corpus.yml')).find( - (step) => step.name === 'Select the exact deterministic repository sample', - ); - const match = selectionStep?.run?.match( - /node --input-type=module <<'NODE'\n(?[\s\S]+?)\nNODE/u, - ); - if (!match?.groups?.program) { - throw new Error('corpus selection step must expose an executable inline Node program'); - } - - const directory = mkdtempSync(join(tmpdir(), 'scriptspect-corpus-selection-')); - const candidateFile = join(directory, 'candidates.txt'); - const selectedFile = join(directory, 'selected.txt'); - const selectionFile = join(directory, 'selection.json'); - try { - writeFileSync(candidateFile, `${candidates.join('\n')}\n`, 'utf8'); - const result = spawnSync( - process.execPath, - ['--input-type=module', '--eval', match.groups.program], - { - encoding: 'utf8', - env: { - ...process.env, - REPO_COUNT: String(requested), - CANDIDATE_FILE: candidateFile, - SELECTED_FILE: selectedFile, - SELECTION_FILE: selectionFile, - }, - }, - ); - if (result.status !== 0) { - throw new Error(`corpus selection failed:\n${result.stderr}`); - } - return { - repositories: readFileSync(selectedFile, 'utf8').trimEnd().split('\n'), - selection: JSON.parse(readFileSync(selectionFile, 'utf8')) as unknown, - }; - } finally { - rmSync(directory, { recursive: true, force: true }); - } -} - describe('pull-request trust boundary', () => { it('keeps contributor-controlled workflows read-only and free of secrets or pushes', () => { const pullRequestWorkflows = workflowNames().filter((name) => @@ -336,14 +292,15 @@ describe('reproducible CI', () => { ); }); - it('bounds every hosted job and caps GitHub Search pages at 100', () => { + it('bounds every hosted job and delegates the 100-repository cap to the typed resolver', () => { for (const name of workflowNames()) { for (const [jobName, job] of Object.entries(workflow(name).jobs ?? {})) { expect(job['timeout-minutes'], `${name}:${jobName}`).toBeGreaterThan(0); } } const corpus = workflowSource('corpus.yml'); - expect(corpus).toContain('REPO_COUNT" -le 100'); + expect(corpus).toContain("default: '100'"); + expect(corpus).toContain('tools/corpus-resolve.ts'); expect(corpus).not.toContain('{0,2}'); }); @@ -383,37 +340,53 @@ describe('reproducible CI', () => { }); describe('corpus repository selection', () => { - it('selects exactly one repository and records requested and actual counts', () => { - const result = runCorpusSelection(1, ['zeta/project', 'alpha/project']); + it('captures the complete ranked candidate universe before resolving the sample', () => { + const steps = allSteps(workflow('corpus.yml')); + const collector = steps.find( + (step) => step.name === 'Collect the ranked repository candidate snapshot', + ); + const resolver = steps.find( + (step) => step.name === 'Resolve the exact root-eligible repository sample', + ); - expect(result.repositories).toEqual(['alpha/project']); - expect(result.selection).toEqual({ schemaVersion: 1, requested: 1, actual: 1 }); + expect(collector?.run).toBe( + 'pnpm exec tsx tools/corpus-candidates.ts repository-candidates.json', + ); + expect(collector?.env).toEqual({ GITHUB_TOKEN: `\${{ github.token }}` }); + expect(steps.indexOf(collector as Step)).toBeLessThan(steps.indexOf(resolver as Step)); + expect(workflowSource('corpus.yml')).not.toContain('uniqueCandidates.sort'); + expect(workflowSource('corpus.yml')).not.toContain('gh api'); }); - it('selects exactly 100 repositories at the supported upper bound', () => { - const candidates = Array.from( - { length: 120 }, - (_, index) => `owner/project-${String(119 - index).padStart(3, '0')}`, + it('resolves the exact root-eligible sample before the scanner runs', () => { + const steps = allSteps(workflow('corpus.yml')); + const resolver = steps.find( + (step) => step.name === 'Resolve the exact root-eligible repository sample', + ); + const scannerIndex = steps.findIndex( + (step) => step.name === 'Scan scripts without writing to sampled repositories', ); - const result = runCorpusSelection(100, candidates); + const resolverIndex = steps.indexOf(resolver as Step); - expect(result.repositories).toHaveLength(100); - expect(result.repositories[0]).toBe('owner/project-000'); - expect(result.repositories[99]).toBe('owner/project-099'); - expect(result.selection).toEqual({ schemaVersion: 1, requested: 100, actual: 100 }); - }); + expect(resolverIndex).toBeGreaterThanOrEqual(0); + expect(resolverIndex).toBeLessThan(scannerIndex); + expect(resolver?.run).toBe( + 'pnpm exec tsx tools/corpus-resolve.ts repository-candidates.json repos.txt repository-sample.json "$REPO_COUNT"', + ); + expect(resolver?.env).toMatchObject({ + GITHUB_TOKEN: `\${{ github.token }}`, + REPO_COUNT: `\${{ inputs.repo-count || '100' }}`, + }); - it('deduplicates overlapping search results before applying the exact limit', () => { - const result = runCorpusSelection(3, [ - 'owner/project-c', - 'owner/project-a', - 'owner/project-b', - 'owner/project-a', - 'owner/project-c', - 'owner/project-d', - ]); - - expect(result.repositories).toEqual(['owner/project-a', 'owner/project-b', 'owner/project-c']); - expect(result.selection).toEqual({ schemaVersion: 1, requested: 3, actual: 3 }); + const scanner = steps[scannerIndex]; + expect(scanner?.env).toMatchObject({ + CORPUS_SAMPLE_METHOD: 'popularity-strata-round-robin-v1', + CORPUS_CANDIDATE_SNAPSHOT: 'repository-candidates.json', + CORPUS_SAMPLE_EVIDENCE: 'repository-sample.json', + }); + const upload = steps.find( + (step) => step.name === 'Upload draft evidence for maintainer review', + ); + expect(String(upload?.with?.path)).toContain('repository-candidates.json'); }); }); diff --git a/tools/corpus-candidates.ts b/tools/corpus-candidates.ts new file mode 100644 index 0000000..ca34ee6 --- /dev/null +++ b/tools/corpus-candidates.ts @@ -0,0 +1,425 @@ +/** Collect and persist the complete ranked candidate universe for corpus selection. */ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { redactCorpusText, sha256 } from './corpus-lib'; +import { + type GitHubFailureEvidence, + githubApiResponse, + githubFailureEvidence, + invalidGitHubResponse, +} from './github-api'; + +const GITHUB_API = 'https://api.github.com'; +export const CORPUS_SAMPLE_METHOD = 'popularity-strata-round-robin-v1' as const; + +interface SearchItem { + full_name?: string; + stargazers_count?: number; +} + +interface SearchResponse { + total_count?: number; + incomplete_results?: boolean; + items?: SearchItem[]; +} + +export interface RankedCandidate { + rank: number; + repository: string; + stars: number; +} + +export interface CandidateStratum { + id: 'typescript' | 'javascript'; + query: string; + sort: 'stars'; + order: 'desc'; + perPage: 100; + responseSha256: string; + candidates: RankedCandidate[]; +} + +export interface OrderedCandidate { + position: number; + stratum: CandidateStratum['id']; + rank: number; + repository: string; +} + +export interface CorpusCandidateSnapshot { + schemaVersion: 1; + status: 'complete' | 'failed'; + method: typeof CORPUS_SAMPLE_METHOD; + strata: CandidateStratum[]; + orderedCandidates: OrderedCandidate[]; + error?: string; + failure?: GitHubFailureEvidence; +} + +export interface CollectCorpusCandidatesOptions { + outputFile: string; + token: string; + fetchImpl?: typeof fetch; +} + +export const CORPUS_CANDIDATE_STRATA: ReadonlyArray< + Pick +> = [ + { + id: 'typescript', + query: 'is:public language:typescript stars:>2000', + sort: 'stars', + order: 'desc', + perPage: 100, + }, + { + id: 'javascript', + query: 'is:public language:javascript stars:>5000', + sort: 'stars', + order: 'desc', + perPage: 100, + }, +]; + +function validRepositoryName(value: string): boolean { + return ( + /^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99})$/.test(value) && + !value.includes('..') && + !value.endsWith('.') + ); +} + +function exactKeys(value: Record, keys: readonly string[]): boolean { + const actual = Object.keys(value).toSorted(); + const expected = [...keys].toSorted(); + return actual.length === expected.length && actual.every((key, index) => key === expected[index]); +} + +function validateRankedCandidate( + value: unknown, + index: number, + stratumId: CandidateStratum['id'], +): RankedCandidate { + if ( + typeof value !== 'object' || + value === null || + Array.isArray(value) || + !exactKeys(value as Record, ['rank', 'repository', 'stars']) + ) { + throw new Error(`candidate snapshot ${stratumId}: ranked candidate ${index + 1} was invalid`); + } + const candidate = value as Record; + if ( + candidate.rank !== index + 1 || + typeof candidate.repository !== 'string' || + !validRepositoryName(candidate.repository) || + typeof candidate.stars !== 'number' || + !Number.isSafeInteger(candidate.stars) || + candidate.stars < 0 + ) { + throw new Error(`candidate snapshot ${stratumId}: ranked candidate ${index + 1} was invalid`); + } + return { + rank: candidate.rank, + repository: candidate.repository, + stars: candidate.stars, + }; +} + +function validateCandidateStratum(value: unknown, index: number): CandidateStratum { + if ( + typeof value !== 'object' || + value === null || + Array.isArray(value) || + !exactKeys(value as Record, [ + 'id', + 'query', + 'sort', + 'order', + 'perPage', + 'responseSha256', + 'candidates', + ]) + ) { + throw new Error('candidate snapshot stratum metadata was invalid'); + } + const stratum = value as Record; + const expected = CORPUS_CANDIDATE_STRATA[index]; + if ( + expected === undefined || + stratum.id !== expected.id || + stratum.query !== expected.query || + stratum.sort !== expected.sort || + stratum.order !== expected.order || + stratum.perPage !== expected.perPage || + typeof stratum.responseSha256 !== 'string' || + !/^[a-f0-9]{64}$/.test(stratum.responseSha256) || + !Array.isArray(stratum.candidates) + ) { + throw new Error('candidate snapshot stratum metadata was invalid'); + } + if (stratum.candidates.length > 100) { + throw new Error(`candidate snapshot ${expected.id}: candidate stratum exceeded 100 candidates`); + } + const candidates = stratum.candidates.map((candidate, candidateIndex) => + validateRankedCandidate(candidate, candidateIndex, expected.id), + ); + if ( + candidates.some((candidate, candidateIndex) => { + const previous = candidates[candidateIndex - 1]; + return previous !== undefined && candidate.stars > previous.stars; + }) + ) { + throw new Error( + `candidate snapshot ${expected.id}: ranked candidates were not non-increasing by stars`, + ); + } + if (new Set(candidates.map((candidate) => candidate.repository)).size !== candidates.length) { + throw new Error(`candidate snapshot ${expected.id}: ranked candidates contained duplicates`); + } + return { ...expected, responseSha256: stratum.responseSha256, candidates }; +} + +/** Parse and fully validate the durable Search snapshot before any downstream API use. */ +export function parseCorpusCandidateSnapshot(bytes: Buffer): { + snapshot: CorpusCandidateSnapshot; + digest: string; +} { + let parsed: unknown; + try { + parsed = JSON.parse(bytes.toString('utf8')) as unknown; + } catch { + throw new Error('candidate snapshot was not valid JSON'); + } + if ( + typeof parsed !== 'object' || + parsed === null || + Array.isArray(parsed) || + !exactKeys(parsed as Record, [ + 'schemaVersion', + 'status', + 'method', + 'strata', + 'orderedCandidates', + ]) + ) { + throw new Error('candidate snapshot was incomplete or incompatible'); + } + const candidate = parsed as Record; + if ( + candidate.schemaVersion !== 1 || + candidate.status !== 'complete' || + candidate.method !== CORPUS_SAMPLE_METHOD || + !Array.isArray(candidate.strata) || + !Array.isArray(candidate.orderedCandidates) + ) { + throw new Error('candidate snapshot was incomplete or incompatible'); + } + if (candidate.strata.length !== CORPUS_CANDIDATE_STRATA.length) { + throw new Error('candidate snapshot did not contain the required popularity strata'); + } + const rawCandidateCount = candidate.strata.reduce((total, value) => { + if (typeof value !== 'object' || value === null || !Array.isArray(value.candidates)) { + return total; + } + return total + value.candidates.length; + }, 0); + if (rawCandidateCount > 200) { + throw new Error('candidate snapshot Search candidate budget exceeded 200'); + } + const strata = candidate.strata.map(validateCandidateStratum); + const expectedOrder = interleaveCandidateStrata(strata); + const orderedCandidates = candidate.orderedCandidates.map((value, index): OrderedCandidate => { + if ( + typeof value !== 'object' || + value === null || + Array.isArray(value) || + !exactKeys(value as Record, ['position', 'stratum', 'rank', 'repository']) + ) { + throw new Error('candidate snapshot ordering did not match its ranked strata'); + } + const ordered = value as Record; + const expected = expectedOrder[index]; + if ( + expected === undefined || + ordered.position !== expected.position || + ordered.stratum !== expected.stratum || + ordered.rank !== expected.rank || + ordered.repository !== expected.repository + ) { + throw new Error('candidate snapshot ordering did not match its ranked strata'); + } + return expected; + }); + if (orderedCandidates.length !== expectedOrder.length) { + throw new Error('candidate snapshot ordering did not match its ranked strata'); + } + return { + snapshot: { + schemaVersion: 1, + status: 'complete', + method: CORPUS_SAMPLE_METHOD, + strata, + orderedCandidates, + }, + digest: sha256(bytes), + }; +} + +export function interleaveCandidateStrata(strata: readonly CandidateStratum[]): OrderedCandidate[] { + const seen = new Set(); + const ordered: OrderedCandidate[] = []; + const maxRank = Math.max(0, ...strata.map((stratum) => stratum.candidates.length)); + for (let offset = 0; offset < maxRank; offset += 1) { + for (const stratum of strata) { + const candidate = stratum.candidates[offset]; + if (candidate === undefined || seen.has(candidate.repository)) continue; + seen.add(candidate.repository); + ordered.push({ + position: ordered.length + 1, + stratum: stratum.id, + rank: candidate.rank, + repository: candidate.repository, + }); + } + } + return ordered; +} + +function writeSnapshot(outputFile: string, snapshot: CorpusCandidateSnapshot): void { + mkdirSync(dirname(resolve(outputFile)), { recursive: true }); + writeFileSync(outputFile, `${JSON.stringify(snapshot, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + }); +} + +function searchUrl(stratum: (typeof CORPUS_CANDIDATE_STRATA)[number]): string { + const url = new URL(`${GITHUB_API}/search/repositories`); + url.searchParams.set('q', stratum.query); + url.searchParams.set('sort', stratum.sort); + url.searchParams.set('order', stratum.order); + url.searchParams.set('per_page', String(stratum.perPage)); + return url.href; +} + +export async function collectCorpusCandidates( + options: CollectCorpusCandidatesOptions, +): Promise { + if (options.token === '') throw new Error('GITHUB_TOKEN is required (read-only public access)'); + const fetchImpl = options.fetchImpl ?? fetch; + const strata: CandidateStratum[] = []; + try { + for (const definition of CORPUS_CANDIDATE_STRATA) { + const url = searchUrl(definition); + const response = await githubApiResponse( + fetchImpl, + url, + options.token, + 'scriptspect-corpus-candidates', + ); + const responseText = await response.text(); + let parsed: SearchResponse; + try { + parsed = JSON.parse(responseText) as SearchResponse; + } catch { + throw invalidGitHubResponse( + url, + `${definition.id}: GitHub search response was not JSON`, + response, + ); + } + if ( + parsed.incomplete_results !== false || + typeof parsed.total_count !== 'number' || + !Number.isSafeInteger(parsed.total_count) || + parsed.total_count < 0 || + !Array.isArray(parsed.items) || + parsed.items.length !== Math.min(parsed.total_count, definition.perPage) + ) { + throw invalidGitHubResponse( + url, + `${definition.id}: GitHub search response was incomplete or invalid`, + response, + ); + } + const candidates = parsed.items.map((item, index): RankedCandidate => { + if ( + typeof item.full_name !== 'string' || + !validRepositoryName(item.full_name) || + typeof item.stargazers_count !== 'number' || + !Number.isSafeInteger(item.stargazers_count) || + item.stargazers_count < 0 + ) { + throw invalidGitHubResponse( + url, + `${definition.id}: GitHub search candidate was invalid`, + response, + ); + } + return { rank: index + 1, repository: item.full_name, stars: item.stargazers_count }; + }); + if ( + candidates.some((candidate, index) => { + const previous = candidates[index - 1]; + return previous !== undefined && candidate.stars > previous.stars; + }) + ) { + throw invalidGitHubResponse( + url, + `${definition.id}: GitHub search candidates were not ranked by stars`, + response, + ); + } + strata.push({ ...definition, responseSha256: sha256(responseText), candidates }); + } + const snapshot: CorpusCandidateSnapshot = { + schemaVersion: 1, + status: 'complete', + method: CORPUS_SAMPLE_METHOD, + strata, + orderedCandidates: interleaveCandidateStrata(strata), + }; + writeSnapshot(options.outputFile, snapshot); + return snapshot; + } catch (error) { + const message = redactCorpusText(error instanceof Error ? error.message : String(error)); + const snapshot: CorpusCandidateSnapshot = { + schemaVersion: 1, + status: 'failed', + method: CORPUS_SAMPLE_METHOD, + strata, + orderedCandidates: interleaveCandidateStrata(strata), + error: message, + ...(githubFailureEvidence(error) === undefined + ? {} + : { failure: githubFailureEvidence(error) }), + }; + writeSnapshot(options.outputFile, snapshot); + throw new Error(message); + } +} + +async function main(): Promise { + const outputFile = process.argv[2]; + if (outputFile === undefined) { + throw new Error('usage: tsx tools/corpus-candidates.ts repository-candidates.json'); + } + await collectCorpusCandidates({ + outputFile, + token: process.env.GITHUB_TOKEN ?? '', + }); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)) +) { + main().catch((error: unknown) => { + console.error( + `scriptspect corpus candidates: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + }); +} diff --git a/tools/corpus-lib.ts b/tools/corpus-lib.ts index 586aef1..0d44c32 100644 --- a/tools/corpus-lib.ts +++ b/tools/corpus-lib.ts @@ -63,6 +63,12 @@ export function sha256(value: string | Buffer): string { return createHash('sha256').update(value).digest('hex'); } +/** Derive the canonical Git object ID for exact blob bytes. */ +export function gitBlobOid(value: string | Buffer): string { + const bytes = Buffer.isBuffer(value) ? value : Buffer.from(value); + return createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); +} + export function parseRepoLocator(value: string): RepoLocator { const match = /^([A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99}))@([a-f0-9]{40})$/.exec( @@ -96,6 +102,23 @@ function addTruncation(truncations: string[], reason: string): void { if (!truncations.includes(reason)) truncations.push(reason); } +function boundedTreeEntries(tree: readonly TreeEntry[], maxTreeEntries: number): TreeEntry[] { + const rootControlFiles = tree + .filter( + (entry) => + (entry.path === 'package.json' || entry.path === 'pnpm-workspace.yaml') && + isCandidate(entry), + ) + .sort((left, right) => { + if (left.path === 'package.json') return -1; + if (right.path === 'package.json') return 1; + return left.path.localeCompare(right.path); + }); + const rootPaths = new Set(rootControlFiles.map((entry) => entry.path)); + const boundedPrefix = tree.slice(0, maxTreeEntries).filter((entry) => !rootPaths.has(entry.path)); + return [...rootControlFiles, ...boundedPrefix].slice(0, maxTreeEntries); +} + /** Select only bounded manifest inputs; every discarded limit is surfaced. */ export function selectCorpusFiles( tree: readonly TreeEntry[], @@ -105,8 +128,7 @@ export function selectCorpusFiles( if (tree.length > limits.maxTreeEntries) { addTruncation(truncations, `tree-entry-limit:${limits.maxTreeEntries}`); } - const candidates = tree - .slice(0, limits.maxTreeEntries) + const candidates = boundedTreeEntries(tree, limits.maxTreeEntries) .filter(isCandidate) .sort((left, right) => { if (left.path === 'package.json') return -1; diff --git a/tools/corpus-replay-check.mjs b/tools/corpus-replay-check.mjs new file mode 100644 index 0000000..1609e72 --- /dev/null +++ b/tools/corpus-replay-check.mjs @@ -0,0 +1,250 @@ +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { lstatSync, readFileSync, readlinkSync } from 'node:fs'; +import { basename, dirname, isAbsolute } from 'node:path'; + +const MAX_GIT_OUTPUT_BYTES = 64 * 1024 * 1024; +const GIT_SAFE_CONFIG = ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null']; +const GIT_REDIRECT_ENVIRONMENT = new Set([ + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_CEILING_DIRECTORIES', + 'GIT_COMMON_DIR', + 'GIT_CONFIG', + 'GIT_CONFIG_COUNT', + 'GIT_CONFIG_GLOBAL', + 'GIT_CONFIG_NOSYSTEM', + 'GIT_CONFIG_PARAMETERS', + 'GIT_CONFIG_SYSTEM', + 'GIT_DIR', + 'GIT_DISCOVERY_ACROSS_FILESYSTEM', + 'GIT_INDEX_FILE', + 'GIT_NAMESPACE', + 'GIT_NO_REPLACE_OBJECTS', + 'GIT_OBJECT_DIRECTORY', + 'GIT_PREFIX', + 'GIT_QUARANTINE_PATH', + 'GIT_REPLACE_REF_BASE', + 'GIT_SHALLOW_FILE', + 'GIT_WORK_TREE', +]); + +function fail(message) { + throw new Error(`corpus replay preflight failed: ${message}`); +} + +function gitEnvironment(worktree) { + const environment = { ...process.env }; + for (const key of Object.keys(environment)) { + const normalized = key.toUpperCase(); + if ( + GIT_REDIRECT_ENVIRONMENT.has(normalized) || + /^GIT_CONFIG_(?:KEY|VALUE)_\d+$/u.test(normalized) + ) { + delete environment[key]; + } + } + environment.GIT_NO_REPLACE_OBJECTS = '1'; + environment.GIT_WORK_TREE = worktree; + return environment; +} + +function git(arguments_) { + try { + return execFileSync('git', [...GIT_SAFE_CONFIG, ...arguments_], { + encoding: null, + env: gitEnvironment(process.cwd()), + maxBuffer: MAX_GIT_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch { + fail(`git ${arguments_[0] ?? 'command'} did not complete successfully`); + } +} + +function nulRecords(output, description) { + const records = []; + let recordStart = 0; + for (let index = 0; index < output.length; index += 1) { + if (output[index] !== 0) continue; + records.push(output.subarray(recordStart, index)); + recordStart = index + 1; + } + if (recordStart !== output.length) fail(`${description} was not NUL terminated`); + return records; +} + +function validateEvidenceArguments(values) { + if (values.length === 0 || values.length % 2 !== 0) { + fail('evidence basenames and SHA-256 digests must be paired'); + } + const evidence = []; + const seen = new Set(); + for (let index = 0; index < values.length; index += 2) { + const value = values[index]; + const digest = values[index + 1]; + if ( + value === undefined || + value === '' || + value === '.' || + value === '..' || + value.includes('\0') || + isAbsolute(value) || + dirname(value) !== '.' || + basename(value) !== value + ) { + fail('evidence inputs must be root-level basenames'); + } + if (seen.has(value)) fail('evidence basenames must be unique'); + if (digest === undefined || !/^[a-f0-9]{64}$/u.test(digest)) { + fail('evidence SHA-256 digest was invalid'); + } + seen.add(value); + evidence.push({ name: value, sha256: digest }); + } + return evidence; +} + +function validateEvidenceFiles(evidence) { + for (const expected of evidence) { + let stat; + try { + stat = lstatSync(expected.name); + } catch { + fail('an evidence input could not be inspected'); + } + if (!stat.isFile()) fail('evidence inputs must be regular files'); + let bytes; + try { + bytes = readFileSync(expected.name); + } catch { + fail('an evidence input could not be read'); + } + const actual = createHash('sha256').update(bytes).digest('hex'); + if (actual !== expected.sha256) fail('evidence input bytes differ from the recorded run'); + } +} + +function worktreeBytes(mode, path) { + let stat; + try { + stat = lstatSync(path); + } catch { + fail('a tracked worktree path could not be inspected'); + } + + if (mode === '120000') { + if (!stat.isSymbolicLink()) fail('a tracked symlink is not a worktree symlink'); + try { + return readlinkSync(path, { encoding: 'buffer' }); + } catch { + fail('a tracked symlink target could not be read'); + } + } + + if (mode !== '100644' && mode !== '100755') fail('unsupported tracked blob mode'); + if (!stat.isFile()) fail('a tracked blob is not a regular worktree file'); + if (process.platform !== 'win32') { + const executable = (stat.mode & 0o111) !== 0; + if (executable !== (mode === '100755')) fail('tracked executable mode differs from HEAD'); + } + try { + return readFileSync(path); + } catch { + fail('a tracked worktree file could not be read'); + } +} + +function validateRawHeadTree() { + const entries = new Map(); + const tree = git(['ls-tree', '-r', '-z', '--full-tree', 'HEAD']); + for (const record of nulRecords(tree, 'HEAD tree')) { + const tab = record.indexOf(0x09); + if (tab <= 0 || tab === record.length - 1) fail('HEAD tree record is malformed'); + const metadata = record.subarray(0, tab).toString('ascii').split(' '); + if (metadata.length !== 3) fail('HEAD tree metadata is malformed'); + const [mode, type, oid] = metadata; + if (mode === '160000' || type !== 'blob') fail('gitlinks and non-blob entries are unsupported'); + if (oid === undefined || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(oid)) { + fail('HEAD tree object ID is malformed'); + } + const path = record.subarray(tab + 1); + const pathKey = path.toString('hex'); + if (entries.has(pathKey)) fail('HEAD tree contains a duplicate path'); + const bytes = worktreeBytes(mode, path); + const algorithm = oid.length === 40 ? 'sha1' : 'sha256'; + const actual = createHash(algorithm) + .update(Buffer.from(`blob ${bytes.length}\0`, 'ascii')) + .update(bytes) + .digest('hex'); + if (actual !== oid) fail('tracked worktree bytes differ from HEAD'); + entries.set(pathKey, { mode, oid }); + } + return entries; +} + +function validateIndexAgainstHead(headEntries) { + const indexEntries = new Map(); + const index = git(['ls-files', '--stage', '-z', '--full-name', '--']); + for (const record of nulRecords(index, 'index')) { + const tab = record.indexOf(0x09); + if (tab <= 0 || tab === record.length - 1) fail('index record is malformed'); + const metadata = record.subarray(0, tab).toString('ascii').split(' '); + if (metadata.length !== 3) fail('index metadata is malformed'); + const [mode, oid, stage] = metadata; + if ( + stage !== '0' || + (mode !== '100644' && mode !== '100755' && mode !== '120000') || + oid === undefined || + !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(oid) + ) { + fail('index entry is unsupported or malformed'); + } + const pathKey = record.subarray(tab + 1).toString('hex'); + if (indexEntries.has(pathKey)) fail('index contains a duplicate path'); + indexEntries.set(pathKey, { mode, oid }); + } + if (indexEntries.size !== headEntries.size) fail('index tree differs from HEAD'); + for (const [pathKey, expected] of headEntries) { + const actual = indexEntries.get(pathKey); + if (actual?.mode !== expected.mode || actual.oid !== expected.oid) { + fail('index tree differs from HEAD'); + } + } +} + +function validateUntrackedEvidence(evidence) { + const expectedPaths = new Set(evidence.map((value) => Buffer.from(value.name).toString('hex'))); + const untracked = git(['ls-files', '--others', '--exclude-standard', '-z', '--full-name', '--']); + for (const path of nulRecords(untracked, 'untracked file list')) { + if (!expectedPaths.delete(path.toString('hex'))) { + fail('checkout contains a non-evidence untracked file'); + } + } + if (expectedPaths.size !== 0) { + fail('evidence inputs must be nonignored untracked root files'); + } +} + +function main() { + const [sourceCommit, ...evidenceArguments] = process.argv.slice(1); + if (sourceCommit === undefined || !/^[a-f0-9]{40}$/u.test(sourceCommit)) { + fail('source commit must be an exact 40-character lowercase SHA'); + } + const evidence = validateEvidenceArguments(evidenceArguments); + validateEvidenceFiles(evidence); + + const head = git(['rev-parse', '--verify', 'HEAD']).toString('ascii').trim(); + if (head !== sourceCommit) fail('HEAD does not match the recorded source commit'); + + const headEntries = validateRawHeadTree(); + validateIndexAgainstHead(headEntries); + validateUntrackedEvidence(evidence); +} + +try { + main(); +} catch (error) { + const message = error instanceof Error ? error.message : 'unknown preflight failure'; + console.error(message); + process.exitCode = 1; +} diff --git a/tools/corpus-resolve.ts b/tools/corpus-resolve.ts new file mode 100644 index 0000000..30d0cf9 --- /dev/null +++ b/tools/corpus-resolve.ts @@ -0,0 +1,516 @@ +/** Resolve the ranked corpus candidate snapshot to immutable, root-eligible commits. */ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + CORPUS_SAMPLE_METHOD, + type OrderedCandidate, + parseCorpusCandidateSnapshot, +} from './corpus-candidates'; +import { DEFAULT_CORPUS_LIMITS, redactCorpusText } from './corpus-lib'; +import { + classifiedGitHubError, + type GitHubFailureEvidence, + githubApiResponse, + githubFailureEvidence, + invalidGitHubResponse, +} from './github-api'; + +const GRAPHQL_URL = 'https://api.github.com/graphql'; +const GRAPHQL_BATCH_SIZE = 20; + +interface GraphQlError { + type?: unknown; + path?: unknown; + message?: unknown; +} + +interface GraphQlRateLimit { + cost?: unknown; + limit?: unknown; + remaining?: unknown; + used?: unknown; + resetAt?: unknown; +} + +interface GraphQlTreeEntry { + name?: unknown; + mode?: unknown; + type?: unknown; + oid?: unknown; + object?: { + __typename?: unknown; + oid?: unknown; + byteSize?: unknown; + isBinary?: unknown; + } | null; +} + +interface GraphQlRepository { + nameWithOwner?: unknown; + defaultBranchRef?: { + name?: unknown; + target?: { + __typename?: unknown; + oid?: unknown; + file?: GraphQlTreeEntry | null; + } | null; + } | null; +} + +interface GraphQlResponse { + data?: Record & { rateLimit?: GraphQlRateLimit }; + errors?: GraphQlError[]; +} + +function classifiedGraphQlFailure( + errors: readonly GraphQlError[], + response: Response, +): + | { + kind: + | 'primary-rate-limit-exhausted' + | 'secondary-rate-limit' + | 'authentication-failed' + | 'permission-denied'; + type: 'RATE_LIMITED' | 'UNAUTHORIZED' | 'FORBIDDEN'; + } + | undefined { + for (const error of errors) { + if (error.type === 'RATE_LIMITED') { + return { + kind: + response.headers.get('x-ratelimit-remaining') === '0' + ? 'primary-rate-limit-exhausted' + : 'secondary-rate-limit', + type: error.type, + }; + } + if (error.type === 'UNAUTHORIZED') { + return { kind: 'authentication-failed', type: error.type }; + } + if (error.type === 'FORBIDDEN') { + return { kind: 'permission-denied', type: error.type }; + } + } + return undefined; +} + +interface ApiEvidence { + transport: 'github-graphql-batch-v1'; + batchSize: 20; + requests: number; + cost: number; + rateLimit?: { + limit: number; + remaining: number; + used: number; + resetAt: string; + }; +} + +interface ParsedRateLimit extends NonNullable { + cost: number; +} + +interface SelectedCandidate extends OrderedCandidate { + commit: string; + rootManifestOid: string; + rootManifestBytes: number; +} + +interface CorpusSampleExclusion extends OrderedCandidate { + commit: string; + reason: 'root-package-json-unavailable'; +} + +export interface CorpusSampleEvidence { + schemaVersion: 2; + method: typeof CORPUS_SAMPLE_METHOD; + candidateSnapshotSha256: string; + requested: number; + actual: number; + candidatesConsidered: number; + status: 'complete' | 'failed'; + api: ApiEvidence; + selected: SelectedCandidate[]; + exclusions: CorpusSampleExclusion[]; + error?: string; + failure?: GitHubFailureEvidence; +} + +export interface CorpusResolveOptions { + candidateFile: string; + outputFile: string; + evidenceFile: string; + requested: number; + token: string; + fetchImpl?: typeof fetch; +} + +function exactOid(value: unknown, description: string): string { + if (typeof value !== 'string' || !/^[a-f0-9]{40}$/.test(value)) { + throw invalidGitHubResponse(GRAPHQL_URL, `${description} was not an exact 40-character oid`); + } + return value; +} + +function safeInteger(value: unknown, description: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw invalidGitHubResponse(GRAPHQL_URL, `${description} was invalid`); + } + return value; +} + +function graphQlQuery(candidates: readonly OrderedCandidate[]): string { + const repositories = candidates.map((candidate, index) => { + const [owner, name] = candidate.repository.split('/'); + return `r${index}: repository(owner: ${JSON.stringify(owner)}, name: ${JSON.stringify(name)}) { + nameWithOwner + defaultBranchRef { + name + target { + __typename + ... on Commit { + oid + file(path: "package.json") { + name + mode + type + oid + object { + __typename + ... on Blob { oid byteSize isBinary } + } + } + } + } + } + }`; + }); + return `query CorpusEligibility {\n${repositories.join('\n')}\nrateLimit { cost limit remaining used resetAt }\n}`; +} + +function expectedMissingRoot(error: GraphQlError, alias: string): boolean { + return ( + error.type === 'NOT_FOUND' && + Array.isArray(error.path) && + JSON.stringify(error.path) === JSON.stringify([alias, 'defaultBranchRef', 'target', 'file']) + ); +} + +function graphQlRateLimit(value: unknown): ParsedRateLimit { + if (typeof value !== 'object' || value === null) { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response had no rateLimit'); + } + const rate = value as GraphQlRateLimit; + return { + cost: safeInteger(rate.cost, 'GitHub GraphQL rateLimit.cost'), + limit: safeInteger(rate.limit, 'GitHub GraphQL rateLimit.limit'), + remaining: safeInteger(rate.remaining, 'GitHub GraphQL rateLimit.remaining'), + used: safeInteger(rate.used, 'GitHub GraphQL rateLimit.used'), + resetAt: + typeof rate.resetAt === 'string' && !Number.isNaN(Date.parse(rate.resetAt)) + ? rate.resetAt + : (() => { + throw invalidGitHubResponse( + GRAPHQL_URL, + 'GitHub GraphQL rateLimit.resetAt was invalid', + ); + })(), + }; +} + +function resolveRepository( + candidate: OrderedCandidate, + alias: string, + repository: unknown, + errors: readonly GraphQlError[], +): SelectedCandidate | CorpusSampleExclusion { + if (typeof repository !== 'object' || repository === null) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}: GitHub GraphQL repository was unavailable`, + ); + } + const result = repository as GraphQlRepository; + if (result.nameWithOwner !== candidate.repository) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}: canonical repository name did not match`, + ); + } + const target = result.defaultBranchRef?.target; + if (target?.__typename !== 'Commit') { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}: default branch did not resolve to a commit`, + ); + } + const commit = exactOid(target.oid, `${candidate.repository}: default branch commit`); + const aliasErrors = errors.filter( + (error) => Array.isArray(error.path) && error.path[0] === alias, + ); + if (target.file === null || target.file === undefined) { + if (aliasErrors.length !== 1 || !expectedMissingRoot(aliasErrors[0] as GraphQlError, alias)) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}@${commit}: root package.json was unresolved without the expected NOT_FOUND evidence`, + ); + } + return { ...candidate, commit, reason: 'root-package-json-unavailable' }; + } + if (aliasErrors.length !== 0) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}@${commit}: GitHub GraphQL returned a partial error`, + ); + } + const file = target.file; + const oid = exactOid(file.oid, `${candidate.repository}@${commit}: root manifest tree entry`); + const objectOid = exactOid( + file.object?.oid, + `${candidate.repository}@${commit}: root manifest blob`, + ); + const bytes = safeInteger( + file.object?.byteSize, + `${candidate.repository}@${commit}: root manifest byte size`, + ); + if ( + file.name !== 'package.json' || + file.type !== 'blob' || + (file.mode !== 33188 && file.mode !== 33261) || + file.object?.__typename !== 'Blob' || + file.object.isBinary !== false || + oid !== objectOid || + bytes > DEFAULT_CORPUS_LIMITS.maxFileBytes + ) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `${candidate.repository}@${commit}: root package.json did not satisfy immutable blob invariants`, + ); + } + return { ...candidate, commit, rootManifestOid: oid, rootManifestBytes: bytes }; +} + +function writeEvidence( + outputFile: string, + evidenceFile: string, + locators: readonly string[], + evidence: CorpusSampleEvidence, +): void { + mkdirSync(dirname(resolve(outputFile)), { recursive: true }); + mkdirSync(dirname(resolve(evidenceFile)), { recursive: true }); + writeFileSync(outputFile, locators.length === 0 ? '' : `${locators.join('\n')}\n`, { + encoding: 'utf8', + flag: 'wx', + }); + writeFileSync(evidenceFile, `${JSON.stringify(evidence, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + }); +} + +/** Select exactly `requested` immutable repositories from the audited ranked snapshot. */ +export async function resolveCorpusSample( + options: CorpusResolveOptions, +): Promise { + if (options.token === '') throw new Error('GITHUB_TOKEN is required (read-only public access)'); + if ( + !Number.isSafeInteger(options.requested) || + options.requested < 1 || + options.requested > 100 + ) { + throw new Error('requested repository count must be an integer from 1 through 100'); + } + const { snapshot, digest } = parseCorpusCandidateSnapshot(readFileSync(options.candidateFile)); + if (snapshot.orderedCandidates.length < options.requested) { + throw new Error( + `requested ${options.requested} repositories but only ${snapshot.orderedCandidates.length} ordered candidates were captured`, + ); + } + + const fetchImpl = options.fetchImpl ?? fetch; + const selected: SelectedCandidate[] = []; + const exclusions: CorpusSampleExclusion[] = []; + const api: ApiEvidence = { + transport: 'github-graphql-batch-v1', + batchSize: GRAPHQL_BATCH_SIZE, + requests: 0, + cost: 0, + }; + let candidatesConsidered = 0; + let lastApiResponse: Response | undefined; + + try { + for (let offset = 0; offset < snapshot.orderedCandidates.length; offset += GRAPHQL_BATCH_SIZE) { + const batch = snapshot.orderedCandidates.slice(offset, offset + GRAPHQL_BATCH_SIZE); + api.requests += 1; + const response = await githubApiResponse( + fetchImpl, + GRAPHQL_URL, + options.token, + 'scriptspect-corpus-resolve', + { method: 'POST', body: JSON.stringify({ query: graphQlQuery(batch) }) }, + ); + lastApiResponse = response; + let payload: GraphQlResponse; + try { + payload = (await response.json()) as GraphQlResponse; + } catch { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response was not valid JSON'); + } + const errors = payload.errors ?? []; + if (!Array.isArray(errors)) { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL errors were invalid'); + } + const classifiedFailure = classifiedGraphQlFailure(errors, response); + if (classifiedFailure !== undefined) { + throw classifiedGitHubError( + classifiedFailure.kind, + GRAPHQL_URL, + `GitHub GraphQL returned ${classifiedFailure.type}`, + response, + ); + } + if (typeof payload.data !== 'object' || payload.data === null) { + throw invalidGitHubResponse(GRAPHQL_URL, 'GitHub GraphQL response had no data'); + } + const unrelatedError = errors.find( + (error) => + typeof error !== 'object' || + error === null || + !batch.some((_, index) => expectedMissingRoot(error, `r${index}`)), + ); + if (unrelatedError !== undefined) { + throw invalidGitHubResponse( + GRAPHQL_URL, + 'GitHub GraphQL returned an unexpected partial error', + ); + } + const rate = graphQlRateLimit(payload.data.rateLimit); + api.cost += rate.cost; + api.rateLimit = { + limit: rate.limit, + remaining: rate.remaining, + used: rate.used, + resetAt: rate.resetAt, + }; + + for (const [index, candidate] of batch.entries()) { + candidatesConsidered += 1; + const resolved = resolveRepository( + candidate, + `r${index}`, + payload.data[`r${index}`], + errors, + ); + if ('reason' in resolved) exclusions.push(resolved); + else selected.push(resolved); + if (selected.length === options.requested) break; + } + if (selected.length === options.requested) break; + if (rate.remaining === 0) { + throw invalidGitHubResponse( + GRAPHQL_URL, + `GitHub GraphQL primary budget was exhausted before ${options.requested} repositories resolved`, + ); + } + } + if (selected.length !== options.requested) { + throw new Error( + `requested ${options.requested} root-eligible repositories but only ${selected.length} were resolved`, + ); + } + } catch (error) { + const originalFailure = githubFailureEvidence(error); + const recordedError = + originalFailure?.kind === 'response-invalid' && + originalFailure.status === null && + lastApiResponse !== undefined + ? invalidGitHubResponse( + GRAPHQL_URL, + error instanceof Error ? error.message : String(error), + lastApiResponse, + ) + : error; + const message = redactCorpusText( + recordedError instanceof Error ? recordedError.message : String(recordedError), + ); + const failure = githubFailureEvidence(recordedError); + const failed: CorpusSampleEvidence = { + schemaVersion: 2, + method: CORPUS_SAMPLE_METHOD, + candidateSnapshotSha256: digest, + requested: options.requested, + actual: selected.length, + candidatesConsidered, + status: 'failed', + api, + selected, + exclusions, + error: message, + ...(failure === undefined ? {} : { failure }), + }; + writeEvidence( + options.outputFile, + options.evidenceFile, + selected.map((candidate) => `${candidate.repository}@${candidate.commit}`), + failed, + ); + throw new Error(message); + } + + const complete: CorpusSampleEvidence = { + schemaVersion: 2, + method: CORPUS_SAMPLE_METHOD, + candidateSnapshotSha256: digest, + requested: options.requested, + actual: selected.length, + candidatesConsidered, + status: 'complete', + api, + selected, + exclusions, + }; + writeEvidence( + options.outputFile, + options.evidenceFile, + selected.map((candidate) => `${candidate.repository}@${candidate.commit}`), + complete, + ); + return complete; +} + +async function main(): Promise { + const [candidateFile, outputFile, evidenceFile, requestedText] = process.argv.slice(2); + if ( + candidateFile === undefined || + outputFile === undefined || + evidenceFile === undefined || + requestedText === undefined + ) { + throw new Error( + 'usage: tsx tools/corpus-resolve.ts repository-candidates.json repos.txt repository-sample.json count', + ); + } + await resolveCorpusSample({ + candidateFile, + outputFile, + evidenceFile, + requested: Number(requestedText), + token: process.env.GITHUB_TOKEN ?? '', + }); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)) +) { + main().catch((error: unknown) => { + console.error( + `scriptspect corpus resolver: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + }); +} diff --git a/tools/corpus-scan.ts b/tools/corpus-scan.ts index 6359655..7fea8d9 100644 --- a/tools/corpus-scan.ts +++ b/tools/corpus-scan.ts @@ -5,35 +5,107 @@ * package manifests are downloaded, scripts are never executed, and raw * script source is never written to evidence artifacts. */ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; import { type AnalysisResult, analyze } from '../src/core/analyze'; import { DEFAULT_TARGETS } from '../src/core/targets'; import { RULES } from '../src/rules'; import type { Finding } from '../src/rules/types'; +import { + CORPUS_SAMPLE_METHOD, + type CorpusCandidateSnapshot, + type OrderedCandidate, + parseCorpusCandidateSnapshot, +} from './corpus-candidates'; import { type CorpusLimits, DEFAULT_CORPUS_LIMITS, + gitBlobOid, parseRepoLocator, redactCorpusText, selectCorpusFiles, sha256, type TreeEntry, } from './corpus-lib'; +import { + checkedResponse, + type GitHubFailureEvidence, + githubApiResponse, + githubFailureEvidence, + invalidGitHubResponse, +} from './github-api'; const GITHUB_API = 'https://api.github.com'; +const GITHUB_RAW = 'https://raw.githubusercontent.com'; +const CORPUS_SOURCE_CHECKOUT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const MAX_GIT_OUTPUT_BYTES = 16 * 1024 * 1024; +const GIT_SAFE_CONFIG = ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null']; +const GIT_REDIRECT_ENVIRONMENT = new Set([ + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_CEILING_DIRECTORIES', + 'GIT_COMMON_DIR', + 'GIT_CONFIG', + 'GIT_CONFIG_COUNT', + 'GIT_CONFIG_GLOBAL', + 'GIT_CONFIG_NOSYSTEM', + 'GIT_CONFIG_PARAMETERS', + 'GIT_CONFIG_SYSTEM', + 'GIT_DIR', + 'GIT_DISCOVERY_ACROSS_FILESYSTEM', + 'GIT_INDEX_FILE', + 'GIT_NAMESPACE', + 'GIT_NO_REPLACE_OBJECTS', + 'GIT_OBJECT_DIRECTORY', + 'GIT_PREFIX', + 'GIT_QUARANTINE_PATH', + 'GIT_REPLACE_REF_BASE', + 'GIT_SHALLOW_FILE', + 'GIT_WORK_TREE', +]); +const CORPUS_REPLAY_CHECK_SOURCE = readFileSync( + fileURLToPath(new URL('./corpus-replay-check.mjs', import.meta.url)), + 'utf8', +); +const CORPUS_REPLAY_CHECK_GZIP_BASE64 = gzipSync(Buffer.from(CORPUS_REPLAY_CHECK_SOURCE), { + level: 9, +}).toString('base64'); +const CORPUS_REPLAY_BOOTSTRAP = + 'const encoded = process.argv.splice(1, 1)[0]; if (encoded === undefined) throw new Error("missing replay check"); const { gunzipSync } = await import("node:zlib"); await import("data:text/javascript;base64," + gunzipSync(Buffer.from(encoded, "base64")).toString("base64"));'; +const CORPUS_LIMIT_KEYS = [ + 'maxTreeEntries', + 'maxManifests', + 'maxDepth', + 'maxFileBytes', + 'maxTotalBytes', +] as const satisfies readonly (keyof CorpusLimits)[]; +const CORPUS_RESERVED_BASENAMES = new Set([ + '.git', + 'node_modules', + 'findings.jsonl', + 'summary.md', + 'corpus-run.json', +]); -interface GitHubTreeResponse { - tree?: TreeEntry[]; - truncated?: boolean; +type CorpusEvidenceRole = 'repository list' | 'candidate snapshot' | 'sample evidence'; + +interface CorpusEvidencePath { + role: CorpusEvidenceRole; + path: string; + name: string; } -interface GitHubBlobResponse { - content?: string; - encoding?: string; - size?: number; +interface CorpusEvidenceInput extends CorpusEvidencePath { + bytes: Buffer; + sha256: string; +} + +interface GitHubTreeResponse { + tree?: TreeEntry[]; + truncated?: unknown; } interface CountSummary { @@ -52,6 +124,7 @@ interface RepositoryEvidence { manifestPaths: string[]; truncations: string[]; error?: string; + failure?: GitHubFailureEvidence; rootOnly: Omit; workspaceFull: Omit; } @@ -73,6 +146,12 @@ interface FindingEvidence { message: string; } +interface ValidatedSampleSelection extends OrderedCandidate { + commit: string; + rootManifestOid: string; + rootManifestBytes: number; +} + interface CorpusRunManifest { schemaVersion: 1; generatedAt: string; @@ -83,7 +162,12 @@ interface CorpusRunManifest { mode: 'root-and-workspace'; targets: typeof DEFAULT_TARGETS; limits: CorpusLimits; - sampling: { method: string; seed: string }; + sampling: { + method: string; + seed: string; + candidateSnapshotSha256?: string; + sampleEvidenceSha256?: string; + }; environment: { node: string; platform: NodeJS.Platform; arch: string; runnerOs?: string }; repositories: RepositoryEvidence[]; promotedTotals: { rootOnly: CountSummary; workspaceFull: CountSummary }; @@ -100,6 +184,9 @@ export interface CorpusScanOptions { limits?: CorpusLimits; sampleMethod?: string; sampleSeed?: string; + candidateSnapshotFile?: string; + sampleEvidenceFile?: string; + sourceCheckout?: string; fetchImpl?: typeof fetch; } @@ -112,54 +199,573 @@ function exactSourceCommit(value: string): string { return value; } -function readLocators(inputFile: string): ReturnType[] { - const locators = readFileSync(inputFile, 'utf8') +function posixShellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function corpusFilenameKey(value: string | Buffer): string { + const bytes = typeof value === 'string' ? Buffer.from(value) : value; + if (process.platform !== 'win32' && process.platform !== 'darwin') { + return bytes.toString('hex'); + } + let decoded: string; + try { + decoded = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error('recorded source commit contains a filename unsupported on this platform'); + } + return decoded.normalize('NFC').toLocaleLowerCase('en-US'); +} + +function sourceGit(sourceCheckout: string, arguments_: string[], description: string): Buffer { + const environment = { ...process.env }; + for (const key of Object.keys(environment)) { + const normalized = key.toUpperCase(); + if ( + GIT_REDIRECT_ENVIRONMENT.has(normalized) || + /^GIT_CONFIG_(?:KEY|VALUE)_\d+$/u.test(normalized) + ) { + delete environment[key]; + } + } + environment.GIT_NO_REPLACE_OBJECTS = '1'; + environment.GIT_WORK_TREE = sourceCheckout; + try { + return execFileSync('git', [...GIT_SAFE_CONFIG, '-C', sourceCheckout, ...arguments_], { + encoding: null, + env: environment, + maxBuffer: MAX_GIT_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch { + throw new Error(`recorded source checkout ${description} could not be verified`); + } +} + +function nulRecords(output: Buffer, description: string): Buffer[] { + if (output.length === 0) return []; + if (output.at(-1) !== 0) throw new Error(`${description} was not NUL terminated`); + const records: Buffer[] = []; + let start = 0; + for (let index = 0; index < output.length; index += 1) { + if (output[index] !== 0) continue; + if (index === start) throw new Error(`${description} contained an empty filename`); + records.push(output.subarray(start, index)); + start = index + 1; + } + return records; +} + +function validateEvidenceAgainstSourceTree( + evidence: CorpusEvidencePath[], + sourceCommit: string, + sourceCheckout = CORPUS_SOURCE_CHECKOUT, +): void { + const checkout = resolve(sourceCheckout); + const head = sourceGit(checkout, ['rev-parse', '--verify', 'HEAD^{commit}'], 'HEAD') + .toString('ascii') + .trim(); + if (head !== sourceCommit) { + throw new Error('recorded source checkout HEAD does not match the source commit'); + } + const trackedRootKeys = new Set( + nulRecords( + sourceGit(checkout, ['ls-tree', '-z', '--name-only', sourceCommit], 'root tree'), + 'recorded source root tree', + ).map(corpusFilenameKey), + ); + for (const input of evidence) { + if (trackedRootKeys.has(corpusFilenameKey(input.name))) { + throw new Error(`${input.role} basename is tracked at the recorded source commit root`); + } + } +} + +function corpusEvidencePaths( + options: Pick< + CorpusScanOptions, + 'inputFile' | 'candidateSnapshotFile' | 'sampleEvidenceFile' | 'sourceCommit' + >, +): CorpusEvidencePath[] { + if ( + (options.candidateSnapshotFile === undefined) !== + (options.sampleEvidenceFile === undefined) + ) { + throw new Error('candidate snapshot and sample evidence must be provided together'); + } + const paths: Array<{ role: CorpusEvidenceRole; path: string | undefined }> = [ + { role: 'repository list', path: options.inputFile }, + { role: 'candidate snapshot', path: options.candidateSnapshotFile }, + { role: 'sample evidence', path: options.sampleEvidenceFile }, + ]; + const reserved = new Set(CORPUS_RESERVED_BASENAMES); + if (/^[a-f0-9]{40}$/u.test(options.sourceCommit)) { + reserved.add(`corpus-reproduction-${options.sourceCommit}`); + } + const reservedKeys = new Set([...reserved].map(corpusFilenameKey)); + const seen = new Set(); + const evidence: CorpusEvidencePath[] = []; + for (const entry of paths) { + if (entry.path === undefined) continue; + const name = entry.path.includes('\0') ? '' : basename(entry.path); + if (name === '' || name === '.' || name === '..') { + throw new Error(`${entry.role} must have a safe replay basename`); + } + const key = corpusFilenameKey(name); + if (reservedKeys.has(key)) { + throw new Error(`${entry.role} basename is reserved for corpus output or replay state`); + } + if (seen.has(key)) { + throw new Error('corpus evidence inputs must have unique replay basenames'); + } + seen.add(key); + evidence.push({ role: entry.role, path: entry.path, name }); + } + return evidence; +} + +function readCorpusEvidence(input: CorpusEvidencePath): CorpusEvidenceInput { + let stat: ReturnType; + try { + stat = lstatSync(input.path); + } catch { + throw new Error(`${input.role} could not be inspected`); + } + if (!stat.isFile()) throw new Error(`${input.role} must be a regular file`); + let bytes: Buffer; + try { + bytes = readFileSync(input.path); + } catch { + throw new Error(`${input.role} could not be read`); + } + return { ...input, bytes, sha256: sha256(bytes) }; +} + +function normalizeCorpusLimits(value: unknown, source: string): CorpusLimits { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`${source} must be an object with the complete corpus limit contract`); + } + const record = value as Record; + const actualKeys = Object.keys(record).sort(); + const expectedKeys = [...CORPUS_LIMIT_KEYS].sort(); + if (JSON.stringify(actualKeys) !== JSON.stringify(expectedKeys)) { + throw new Error(`${source} must contain exactly ${CORPUS_LIMIT_KEYS.join(', ')}`); + } + for (const key of CORPUS_LIMIT_KEYS) { + if (!Number.isSafeInteger(record[key]) || (record[key] as number) < 0) { + throw new Error(`${source}.${key} must be a non-negative safe integer`); + } + } + return { + maxTreeEntries: record.maxTreeEntries as number, + maxManifests: record.maxManifests as number, + maxDepth: record.maxDepth as number, + maxFileBytes: record.maxFileBytes as number, + maxTotalBytes: record.maxTotalBytes as number, + }; +} + +function corpusLimitsFromEnvironment(value: string | undefined): CorpusLimits { + if (value === undefined) return normalizeCorpusLimits(DEFAULT_CORPUS_LIMITS, 'default limits'); + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error('CORPUS_LIMITS_JSON must be valid JSON'); + } + return normalizeCorpusLimits(parsed, 'CORPUS_LIMITS_JSON'); +} + +function reproductionCommand(options: { + sourceCommit: string; + generatedAt: string; + sampleMethod: string; + sampleSeed: string; + environment: CorpusRunManifest['environment']; + limits: CorpusLimits; + input: CorpusEvidenceInput; + candidateSnapshot?: CorpusEvidenceInput; + sampleEvidence?: CorpusEvidenceInput; +}): string { + const outputDirectory = `corpus-reproduction-${options.sourceCommit}`; + const evidence = [ + options.input, + ...(options.candidateSnapshot === undefined ? [] : [options.candidateSnapshot]), + ...(options.sampleEvidence === undefined ? [] : [options.sampleEvidence]), + ]; + const replayCheckArguments = [ + options.sourceCommit, + ...evidence.flatMap((value) => [value.name, value.sha256]), + ] + .map(posixShellQuote) + .join(' '); + const cleanCheckout = [ + `node --input-type=module -e ${posixShellQuote(CORPUS_REPLAY_BOOTSTRAP)} -- "$SCRIPTSPECT_REPLAY_CHECK" ${replayCheckArguments}`, + ]; + const environment = [ + `SCRIPTSPECT_SOURCE_COMMIT=${posixShellQuote(options.sourceCommit)}`, + `CORPUS_GENERATED_AT=${posixShellQuote(options.generatedAt)}`, + `CORPUS_SAMPLE_METHOD=${posixShellQuote(options.sampleMethod)}`, + `CORPUS_SAMPLE_SEED=${posixShellQuote(options.sampleSeed)}`, + `CORPUS_LIMITS_JSON=${posixShellQuote(JSON.stringify(options.limits))}`, + ...(options.candidateSnapshot === undefined + ? [] + : [`CORPUS_CANDIDATE_SNAPSHOT=${posixShellQuote(options.candidateSnapshot.name)}`]), + ...(options.sampleEvidence === undefined + ? [] + : [`CORPUS_SAMPLE_EVIDENCE=${posixShellQuote(options.sampleEvidence.name)}`]), + ]; + const commands = [ + 'set +a', + `SCRIPTSPECT_REPLAY_CHECK=${posixShellQuote(CORPUS_REPLAY_CHECK_GZIP_BASE64)}`, + 'unset SCRIPTSPECT_REPLAY_TOKEN', + `SCRIPTSPECT_REPLAY_TOKEN="\${GITHUB_TOKEN-}"`, + 'unset GITHUB_TOKEN', + 'test -n "$SCRIPTSPECT_REPLAY_TOKEN"', + `test "$(node --version)" = ${posixShellQuote(options.environment.node)}`, + `test "$(node -p 'process.platform')" = ${posixShellQuote(options.environment.platform)}`, + `test "$(node -p 'process.arch')" = ${posixShellQuote(options.environment.arch)}`, + ...cleanCheckout, + 'test ! -e node_modules && test ! -L node_modules', + 'command env corepack enable', + "command env corepack prepare 'pnpm@11.24.0' --activate", + 'command env pnpm install --frozen-lockfile', + ...cleanCheckout, + `test ! -e ${posixShellQuote(outputDirectory)}`, + `mkdir -- ${posixShellQuote(outputDirectory)}`, + options.environment.runnerOs === undefined + ? 'unset RUNNER_OS' + : `export RUNNER_OS=${posixShellQuote(options.environment.runnerOs)}`, + `command env GITHUB_TOKEN="$SCRIPTSPECT_REPLAY_TOKEN" ${environment.join(' ')} pnpm exec tsx tools/corpus-scan.ts ${posixShellQuote(options.input.name)} ${posixShellQuote(outputDirectory)}`, + ]; + return `(${commands.join(' && ')})`; +} + +function readLocatorSequence(input: Buffer): ReturnType[] { + return input + .toString('utf8') .split(/\r?\n/u) .map((line) => line.trim()) .filter((line) => line !== '' && !line.startsWith('#')) .map(parseRepoLocator); +} + +function sortedUniqueLocators( + locators: readonly ReturnType[], +): ReturnType[] { const unique = new Map(locators.map((locator) => [`${locator.repo}@${locator.commit}`, locator])); return [...unique.values()].sort((left, right) => `${left.repo}@${left.commit}`.localeCompare(`${right.repo}@${right.commit}`), ); } -function headers(token: string): Record { - return { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': '2022-11-28', - 'User-Agent': 'scriptspect-corpus-scan', - }; +function exactEvidenceRecord( + value: unknown, + keys: readonly string[], + description: string, +): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`corpus sample evidence ${description} was invalid`); + } + const record = value as Record; + const actualKeys = Object.keys(record).toSorted(); + const expectedKeys = [...keys].toSorted(); + if ( + actualKeys.length !== expectedKeys.length || + !actualKeys.every((key, index) => key === expectedKeys[index]) + ) { + throw new Error(`corpus sample evidence ${description} was invalid`); + } + return record; } -async function fetchJson(fetchImpl: typeof fetch, url: string, token: string): Promise { - const response = await fetchImpl(url, { headers: headers(token) }); - if (!response.ok) throw new Error(`GitHub API ${response.status} for ${url}`); - return (await response.json()) as T; +function evidenceInteger(value: unknown, description: string, minimum = 0): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) { + throw new Error(`corpus sample evidence ${description} was invalid`); + } + return value; +} + +function evidenceOid(value: unknown, description: string): string { + if (typeof value !== 'string' || !/^[a-f0-9]{40}$/.test(value)) { + throw new Error(`corpus sample evidence ${description} was invalid`); + } + return value; +} + +function evidenceCandidateIdentity( + record: Record, + snapshot: CorpusCandidateSnapshot, + description: string, +): OrderedCandidate { + const position = evidenceInteger(record.position, `${description} position`, 1); + const expected = snapshot.orderedCandidates[position - 1]; + if ( + expected === undefined || + record.stratum !== expected.stratum || + record.rank !== expected.rank || + record.repository !== expected.repository + ) { + throw new Error(`corpus sample evidence ${description} did not match the candidate snapshot`); + } + return expected; +} + +function validateSampleEvidence( + bytes: Buffer, + candidateSnapshotSha256: string, + sampleMethod: string, + inputLocators: readonly ReturnType[], + candidateSnapshot: CorpusCandidateSnapshot, +): Map { + let parsed: unknown; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + throw new Error('corpus sample evidence was not valid JSON'); + } + const evidence = exactEvidenceRecord( + parsed, + [ + 'schemaVersion', + 'method', + 'candidateSnapshotSha256', + 'requested', + 'actual', + 'candidatesConsidered', + 'status', + 'api', + 'selected', + 'exclusions', + ], + 'root object', + ); + if ( + evidence.schemaVersion !== 2 || + evidence.status !== 'complete' || + evidence.method !== sampleMethod || + evidence.method !== CORPUS_SAMPLE_METHOD || + evidence.candidateSnapshotSha256 !== candidateSnapshotSha256 + ) { + throw new Error('corpus sample evidence header did not match the scanner provenance'); + } + const requested = evidenceInteger(evidence.requested, 'requested count', 1); + const actual = evidenceInteger(evidence.actual, 'actual count', 1); + const candidatesConsidered = evidenceInteger( + evidence.candidatesConsidered, + 'candidatesConsidered', + 1, + ); + if (requested > 100 || actual !== requested || candidatesConsidered < actual) { + throw new Error('corpus sample evidence requested/actual contract was invalid'); + } + + const api = exactEvidenceRecord( + evidence.api, + ['transport', 'batchSize', 'requests', 'cost', 'rateLimit'], + 'api object', + ); + const requests = evidenceInteger(api.requests, 'api.requests', 1); + const cost = evidenceInteger(api.cost, 'api.cost', 1); + const rateLimit = exactEvidenceRecord( + api.rateLimit, + ['limit', 'remaining', 'used', 'resetAt'], + 'api.rateLimit object', + ); + const limit = evidenceInteger(rateLimit.limit, 'api.rateLimit.limit', 1); + const remaining = evidenceInteger(rateLimit.remaining, 'api.rateLimit.remaining'); + const used = evidenceInteger(rateLimit.used, 'api.rateLimit.used'); + if ( + api.transport !== 'github-graphql-batch-v1' || + api.batchSize !== 20 || + requests !== Math.ceil(candidatesConsidered / 20) || + cost < requests || + remaining > limit || + used > limit || + typeof rateLimit.resetAt !== 'string' || + Number.isNaN(Date.parse(rateLimit.resetAt)) + ) { + throw new Error('corpus sample evidence api contract was invalid'); + } + if (!Array.isArray(evidence.selected) || !Array.isArray(evidence.exclusions)) { + throw new Error('corpus sample evidence selected/exclusions were invalid'); + } + + const selected = evidence.selected.map((value, index): ValidatedSampleSelection => { + const record = exactEvidenceRecord( + value, + [ + 'position', + 'stratum', + 'rank', + 'repository', + 'commit', + 'rootManifestOid', + 'rootManifestBytes', + ], + `selected item ${index + 1}`, + ); + const identity = evidenceCandidateIdentity( + record, + candidateSnapshot, + `selected item ${index + 1}`, + ); + const commit = evidenceOid(record.commit, `selected item ${index + 1} commit`); + const rootManifestOid = evidenceOid( + record.rootManifestOid, + `selected item ${index + 1} rootManifestOid`, + ); + const rootManifestBytes = evidenceInteger( + record.rootManifestBytes, + `selected item ${index + 1} rootManifestBytes`, + ); + if (rootManifestBytes > DEFAULT_CORPUS_LIMITS.maxFileBytes) { + throw new Error( + `corpus sample evidence selected item ${index + 1} root manifest was oversized`, + ); + } + return { ...identity, commit, rootManifestOid, rootManifestBytes }; + }); + const exclusions = evidence.exclusions.map((value, index) => { + const record = exactEvidenceRecord( + value, + ['position', 'stratum', 'rank', 'repository', 'commit', 'reason'], + `exclusion item ${index + 1}`, + ); + const identity = evidenceCandidateIdentity( + record, + candidateSnapshot, + `exclusion item ${index + 1}`, + ); + const commit = evidenceOid(record.commit, `exclusion item ${index + 1} commit`); + if (record.reason !== 'root-package-json-unavailable') { + throw new Error(`corpus sample evidence exclusion item ${index + 1} reason was invalid`); + } + return { ...identity, commit }; + }); + if ( + selected.length !== actual || + selected.length + exclusions.length !== candidatesConsidered || + candidatesConsidered > candidateSnapshot.orderedCandidates.length + ) { + throw new Error('corpus sample evidence candidate accounting was invalid'); + } + const considered = [...selected, ...exclusions].toSorted( + (left, right) => left.position - right.position, + ); + if ( + considered.some((candidate, index) => candidate.position !== index + 1) || + selected.some((candidate, index) => { + const previous = selected[index - 1]; + return previous !== undefined && candidate.position <= previous.position; + }) || + exclusions.some((candidate, index) => { + const previous = exclusions[index - 1]; + return previous !== undefined && candidate.position <= previous.position; + }) + ) { + throw new Error('corpus sample evidence candidate sequence was invalid'); + } + const selectedSequence = selected.map((value) => `${value.repository}@${value.commit}`); + const inputSequence = inputLocators.map((value) => `${value.repo}@${value.commit}`); + if (JSON.stringify(selectedSequence) !== JSON.stringify(inputSequence)) { + throw new Error('corpus sample evidence selected locators did not match repos.txt'); + } + return new Map(selected.map((value) => [`${value.repository}@${value.commit}`, value])); +} + +async function fetchJson( + fetchImpl: typeof fetch, + url: string, + token: string, +): Promise<{ data: T; response: Response }> { + const response = await githubApiResponse(fetchImpl, url, token, 'scriptspect-corpus-scan'); + try { + return { data: (await response.json()) as T, response }; + } catch { + throw invalidGitHubResponse(url, `GitHub API returned invalid JSON for ${url}`, response); + } +} + +function rawManifestUrl(repo: string, commit: string, path: string): string { + const [owner, repository] = repo.split('/'); + if (owner === undefined || repository === undefined) { + throw new Error(`invalid repository name: ${repo}`); + } + const encodedPath = path.split('/').map(encodeURIComponent).join('/'); + return `${GITHUB_RAW}/${encodeURIComponent(owner)}/${encodeURIComponent(repository)}/${commit}/${encodedPath}`; +} + +async function readBoundedBody( + response: Response, + maxBytes: number, + overflowMessage: string, +): Promise { + if (response.body === null) return Buffer.alloc(0); + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let totalBytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > maxBytes) { + await reader.cancel().catch(() => undefined); + throw new Error(overflowMessage); + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock(); + } + return Buffer.concat(chunks, totalBytes); } async function downloadSelectedFiles( repo: string, + commit: string, entries: readonly TreeEntry[], targetRoot: string, - token: string, fetchImpl: typeof fetch, limits: CorpusLimits, ): Promise { let actualTotal = 0; for (const entry of entries) { - const blob = await fetchJson( + if (!/^[a-f0-9]{40}$/.test(entry.sha)) { + throw new Error( + `${entry.path}: immutable tree Git blob OID was not 40 lowercase hex characters`, + ); + } + if (!Number.isSafeInteger(entry.size) || (entry.size as number) < 0) { + throw new Error(`${entry.path}: immutable tree entry had no valid byte size`); + } + const expectedBytes = entry.size as number; + const url = rawManifestUrl(repo, commit, entry.path); + const response = await checkedResponse( fetchImpl, - `${GITHUB_API}/repos/${repo}/git/blobs/${entry.sha}`, - token, + url, + { + headers: { + Accept: 'application/octet-stream', + 'User-Agent': 'scriptspect-corpus-scan', + }, + redirect: 'error', + }, + 'GitHub raw', ); - if (blob.encoding !== 'base64' || typeof blob.content !== 'string') { - throw new Error(`${entry.path}: GitHub blob response was not base64`); + const hardCap = Math.min( + expectedBytes, + limits.maxFileBytes, + limits.maxTotalBytes - actualTotal, + ); + const sizeMismatchMessage = `${entry.path}: raw byte length did not match the immutable tree entry`; + const bytes = await readBoundedBody(response, hardCap, sizeMismatchMessage); + if (bytes.length !== expectedBytes) { + throw new Error(`${entry.path}: raw byte length did not match the immutable tree entry`); } - const bytes = Buffer.from(blob.content.replace(/\s/gu, ''), 'base64'); - if (bytes.length !== entry.size || blob.size !== entry.size) { - throw new Error(`${entry.path}: blob size did not match the immutable tree entry`); + if (gitBlobOid(bytes) !== entry.sha) { + throw new Error(`${entry.path}: raw bytes did not match the immutable tree Git blob OID`); } actualTotal += bytes.length; if (bytes.length > limits.maxFileBytes || actualTotal > limits.maxTotalBytes) { @@ -230,7 +836,7 @@ function findingEvidence( confidence: finding.confidence, affectedTargets: finding.affectedTargets, span: finding.span, - message: redactCorpusText(finding.message), + message: `${finding.ruleId} matched a portability rule at the recorded span.`, }; } @@ -282,10 +888,50 @@ function renderSummary(manifest: CorpusRunManifest): string { export async function runCorpusScan(options: CorpusScanOptions): Promise { if (options.token === '') throw new Error('GITHUB_TOKEN is required (read-only public access)'); const sourceCommit = exactSourceCommit(options.sourceCommit); - const limits = options.limits ?? DEFAULT_CORPUS_LIMITS; + const evidencePaths = corpusEvidencePaths({ ...options, sourceCommit }); + validateEvidenceAgainstSourceTree(evidencePaths, sourceCommit, options.sourceCheckout); + const evidenceByRole = new Map( + evidencePaths.map(readCorpusEvidence).map((input) => [input.role, input] as const), + ); + const inputEvidence = evidenceByRole.get('repository list'); + if (inputEvidence === undefined) throw new Error('repository list evidence was unavailable'); + const candidateSnapshotEvidence = evidenceByRole.get('candidate snapshot'); + const sampleEvidenceInput = evidenceByRole.get('sample evidence'); + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const environment: CorpusRunManifest['environment'] = { + node: process.version, + platform: process.platform, + arch: process.arch, + ...(process.env.RUNNER_OS === undefined ? {} : { runnerOs: process.env.RUNNER_OS }), + }; + const limits = normalizeCorpusLimits(options.limits ?? DEFAULT_CORPUS_LIMITS, 'corpus limits'); const fetchImpl = options.fetchImpl ?? fetch; - const locators = readLocators(options.inputFile); - if (locators.length === 0) throw new Error('repository list is empty'); + const sampleMethod = options.sampleMethod ?? CORPUS_SAMPLE_METHOD; + const candidateSnapshotBytes = candidateSnapshotEvidence?.bytes; + const sampleEvidenceBytes = sampleEvidenceInput?.bytes; + const parsedCandidateSnapshot = + candidateSnapshotBytes === undefined + ? undefined + : parseCorpusCandidateSnapshot(candidateSnapshotBytes); + const candidateSnapshotSha256 = parsedCandidateSnapshot?.digest; + const sampleEvidenceSha256 = sampleEvidenceInput?.sha256; + const locatorSequence = readLocatorSequence(inputEvidence.bytes); + if (locatorSequence.length === 0) throw new Error('repository list is empty'); + let sampleSelections = new Map(); + if ( + parsedCandidateSnapshot !== undefined && + candidateSnapshotSha256 !== undefined && + sampleEvidenceBytes !== undefined + ) { + sampleSelections = validateSampleEvidence( + sampleEvidenceBytes, + candidateSnapshotSha256, + sampleMethod, + locatorSequence, + parsedCandidateSnapshot.snapshot, + ); + } + const locators = sortedUniqueLocators(locatorSequence); const outputDir = resolve(options.outputDir); mkdirSync(outputDir, { recursive: true }); @@ -296,23 +942,56 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise( - fetchImpl, - `${GITHUB_API}/repos/${locator.repo}/git/trees/${locator.commit}?recursive=1`, - options.token, - ); - if (!Array.isArray(treeResponse.tree)) throw new Error('GitHub tree response had no tree'); + const treeUrl = `${GITHUB_API}/repos/${locator.repo}/git/trees/${locator.commit}?recursive=1`; + const { data: treeResponse, response: treeHttpResponse } = + await fetchJson(fetchImpl, treeUrl, options.token); + if (!Array.isArray(treeResponse.tree)) { + throw invalidGitHubResponse(treeUrl, 'GitHub tree response had no tree', treeHttpResponse); + } + if (typeof treeResponse.truncated !== 'boolean') { + throw invalidGitHubResponse( + treeUrl, + 'GitHub tree response truncated flag was not boolean', + treeHttpResponse, + ); + } const selected = selectCorpusFiles(treeResponse.tree, limits); truncations = [...selected.truncations]; - if (treeResponse.truncated === true) truncations.unshift('github-tree-truncated'); + if (treeResponse.truncated) truncations.unshift('github-tree-truncated'); manifestPaths = selected.files.map((entry) => entry.path); + if (truncations.length !== 0) { + repositories.push({ + repository: locator.repo, + commit: locator.commit, + status: 'truncated', + manifestPaths, + truncations, + rootOnly: emptyCounts(), + workspaceFull: emptyCounts(), + }); + continue; + } + const sampleSelection = sampleSelections.get(`${locator.repo}@${locator.commit}`); + if (sampleSelection !== undefined) { + const rootEntries = treeResponse.tree.filter((entry) => entry.path === 'package.json'); + const rootEntry = rootEntries[0]; + if ( + rootEntries.length !== 1 || + rootEntry?.type !== 'blob' || + (rootEntry.mode !== '100644' && rootEntry.mode !== '100755') || + rootEntry.sha !== sampleSelection.rootManifestOid || + rootEntry.size !== sampleSelection.rootManifestBytes + ) { + throw new Error('root package.json tree entry did not match corpus sample evidence'); + } + } if (!manifestPaths.includes('package.json')) throw new Error('root package.json was unavailable'); await downloadSelectedFiles( locator.repo, + locator.commit, selected.files, tempRoot, - options.token, fetchImpl, limits, ); @@ -320,7 +999,7 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise = { schemaVersion: 1, - generatedAt: options.generatedAt ?? new Date().toISOString(), + generatedAt, sourceCommit, scannerSha256: sha256(readFileSync(scannerPath)), registrySha256: sha256(JSON.stringify(registryPayload)), - inputSha256: sha256(readFileSync(options.inputFile)), + inputSha256: inputEvidence.sha256, mode: 'root-and-workspace', targets: DEFAULT_TARGETS, limits, sampling: { - method: options.sampleMethod ?? 'workflow-curated-popularity-strata', + method: sampleMethod, seed: options.sampleSeed ?? 'none', + ...(candidateSnapshotSha256 === undefined ? {} : { candidateSnapshotSha256 }), + ...(sampleEvidenceSha256 === undefined ? {} : { sampleEvidenceSha256 }), }, - environment: { - node: process.version, - platform: process.platform, - arch: process.arch, - ...(process.env.RUNNER_OS === undefined ? {} : { runnerOs: process.env.RUNNER_OS }), - }, + environment, repositories, promotedTotals: { rootOnly: sumComplete(repositories, 'rootOnly'), workspaceFull: sumComplete(repositories, 'workspaceFull'), }, - reproduction: `SCRIPTSPECT_SOURCE_COMMIT=${sourceCommit} pnpm exec tsx tools/corpus-scan.ts ${basename(options.inputFile)}`, + reproduction: reproductionCommand({ + sourceCommit, + generatedAt, + sampleMethod, + sampleSeed: options.sampleSeed ?? 'none', + environment, + limits, + input: inputEvidence, + candidateSnapshot: candidateSnapshotEvidence, + sampleEvidence: sampleEvidenceInput, + }), }; const provisional = { ...partialManifest, artifactSha256: {} } satisfies CorpusRunManifest; const summaryText = renderSummary(provisional); @@ -393,6 +1083,12 @@ export async function runCorpusScan(options: CorpusScanOptions): Promise { - const inputFile = process.argv[2]; +export function corpusScanOptionsFromCli( + arguments_: readonly string[], + environment: NodeJS.ProcessEnv, +): CorpusScanOptions { + const inputFile = arguments_[0]; if (inputFile === undefined) { throw new Error('usage: tsx tools/corpus-scan.ts repos.txt [output-directory]'); } - await runCorpusScan({ + const options: CorpusScanOptions = { inputFile, - outputDir: process.argv[3] ?? process.cwd(), - token: process.env.GITHUB_TOKEN ?? '', - sourceCommit: process.env.SCRIPTSPECT_SOURCE_COMMIT ?? process.env.GITHUB_SHA ?? '', - sampleMethod: process.env.CORPUS_SAMPLE_METHOD, - sampleSeed: process.env.CORPUS_SAMPLE_SEED, - }); + outputDir: arguments_[1] ?? process.cwd(), + token: environment.GITHUB_TOKEN ?? '', + sourceCommit: environment.SCRIPTSPECT_SOURCE_COMMIT ?? environment.GITHUB_SHA ?? '', + generatedAt: environment.CORPUS_GENERATED_AT, + limits: corpusLimitsFromEnvironment(environment.CORPUS_LIMITS_JSON), + sampleMethod: environment.CORPUS_SAMPLE_METHOD, + sampleSeed: environment.CORPUS_SAMPLE_SEED, + candidateSnapshotFile: environment.CORPUS_CANDIDATE_SNAPSHOT, + sampleEvidenceFile: environment.CORPUS_SAMPLE_EVIDENCE, + }; + corpusEvidencePaths(options); + return options; +} + +async function main(): Promise { + await runCorpusScan(corpusScanOptionsFromCli(process.argv.slice(2), process.env)); } if ( diff --git a/tools/github-api.ts b/tools/github-api.ts new file mode 100644 index 0000000..773d4eb --- /dev/null +++ b/tools/github-api.ts @@ -0,0 +1,166 @@ +/** Shared GitHub HTTP failure evidence. Tokens stay in request headers only. */ + +export type GitHubFailureKind = + | 'primary-rate-limit-exhausted' + | 'secondary-rate-limit' + | 'authentication-failed' + | 'permission-denied' + | 'not-found' + | 'http-error' + | 'response-invalid'; + +export interface GitHubFailureEvidence { + kind: GitHubFailureKind; + status: number | null; + url: string; + rateLimit: { + limit: string | null; + remaining: string | null; + reset: string | null; + used: string | null; + resource: string | null; + }; + retryAfter: string | null; + requestId: string | null; +} + +export class GitHubRequestError extends Error { + readonly evidence: GitHubFailureEvidence; + + constructor(message: string, evidence: GitHubFailureEvidence) { + super(message); + this.name = 'GitHubRequestError'; + this.evidence = evidence; + } +} + +function responseHeaders(response: Response): Omit { + return { + status: response.status, + rateLimit: { + limit: response.headers.get('x-ratelimit-limit'), + remaining: response.headers.get('x-ratelimit-remaining'), + reset: response.headers.get('x-ratelimit-reset'), + used: response.headers.get('x-ratelimit-used'), + resource: response.headers.get('x-ratelimit-resource'), + }, + retryAfter: response.headers.get('retry-after'), + requestId: response.headers.get('x-github-request-id'), + }; +} + +function failureKind(response: Response, responseText: string): GitHubFailureKind { + const metadata = responseHeaders(response); + const lower = responseText.toLowerCase(); + if (metadata.rateLimit.remaining === '0') return 'primary-rate-limit-exhausted'; + if ( + response.status === 429 || + metadata.retryAfter !== null || + lower.includes('secondary rate limit') || + lower.includes('abuse detection') + ) { + return 'secondary-rate-limit'; + } + if (response.status === 401) return 'authentication-failed'; + if (response.status === 403) return 'permission-denied'; + if (response.status === 404) return 'not-found'; + return 'http-error'; +} + +export function githubErrorFromResponse( + response: Response, + url: string, + label: string, + responseText: string, + message = `${label} ${response.status} for ${url}`, +): GitHubRequestError { + return new GitHubRequestError(message, { + kind: failureKind(response, responseText), + url, + ...responseHeaders(response), + }); +} + +export function invalidGitHubResponse( + url: string, + message: string, + response?: Response, +): GitHubRequestError { + return new GitHubRequestError(message, { + kind: 'response-invalid', + url, + ...(response === undefined + ? { + status: null, + rateLimit: { limit: null, remaining: null, reset: null, used: null, resource: null }, + retryAfter: null, + requestId: null, + } + : responseHeaders(response)), + }); +} + +/** Persist a semantic GitHub failure even when GraphQL returns HTTP 200. */ +export function classifiedGitHubError( + kind: GitHubFailureKind, + url: string, + message: string, + response?: Response, +): GitHubRequestError { + return new GitHubRequestError(message, { + kind, + url, + ...(response === undefined + ? { + status: null, + rateLimit: { limit: null, remaining: null, reset: null, used: null, resource: null }, + retryAfter: null, + requestId: null, + } + : responseHeaders(response)), + }); +} + +export function githubFailureEvidence(error: unknown): GitHubFailureEvidence | undefined { + return error instanceof GitHubRequestError ? error.evidence : undefined; +} + +export function githubApiHeaders(token: string, userAgent: string): Record { + return { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': userAgent, + }; +} + +export async function checkedResponse( + fetchImpl: typeof fetch, + url: string, + init: RequestInit, + label: string, +): Promise { + const response = await fetchImpl(url, init); + if (response.ok) return response; + const responseText = await response.text(); + throw githubErrorFromResponse(response, url, label, responseText); +} + +export async function githubApiResponse( + fetchImpl: typeof fetch, + url: string, + token: string, + userAgent: string, + init: RequestInit = {}, +): Promise { + return checkedResponse( + fetchImpl, + url, + { + ...init, + headers: { ...githubApiHeaders(token, userAgent), ...(init.headers ?? {}) }, + }, + 'GitHub API', + ); +}