From 12b2b8de1d6552c28d62b6662a33bf45a08504a0 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:42:25 -0700 Subject: [PATCH] =?UTF-8?q?feat(benchmark):=20anti-overfit=20controls=20?= =?UTF-8?q?=E2=80=94=20repo-level=20seeded=20split,=20caps,=20rotation=20(?= =?UTF-8?q?#9263)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A public leaderboard creates a direct incentive to overfit, and an overfit agent is worse than useless: it looks best exactly when it generalizes worst. Four decisions, each recorded with its reasoning in the module header, because each is a knob an adversary optimizes against. 1. SPLIT GRANULARITY IS THE REPO, NOT THE WORK UNIT. The split reuses splitBacktestCorpus verbatim -- asserted, not assumed: a test recomputes the assignment through the primitive directly and requires the same held-out repos -- but keys on the repo, so a repo's units never straddle the boundary. Work-unit splitting leaks: two PRs in one repo share a maintainer, a review culture and often the same files, so an agent that saw the visible ones has effectively seen the answer pattern for the held-out ones. Per-repo is what makes "held out" mean "generalizes to a repo it has never seen". 2. HELD-OUT SCORES DO NOT PUBLISH PER SUBMISSION. Each published score is an oracle query, and differencing successive ones recovers membership. Publication happens at evaluation close, or on a fixed public cadence counted from opensAt -- never on demand. gateHeldOutPublication DROPS the held-out value rather than flagging it, so a caller cannot forget to check a flag and serialize a field that is present in memory. 3. SUBMISSION CAP PER (AGENT, WINDOW). Unbounded resubmission against a fixed corpus is gradient descent on the test set by brute force. Scoped per window rather than per day (which just spreads the same brute force over more days) or globally (which would punish an agent that keeps competing across rotations). Refusals are NAMED, so a submitter can tell a cap they can wait out from a window that will never reopen. 4. ROTATION RETIRES A WINDOW, IT DOES NOT EXTEND IT. A benchmark public for a year is training data with extra steps. A retired window stays published for historical comparison but stops being a scoring basis, and refuses submissions even with quota left. Closes #9263 --- .../src/calibration/benchmark-anti-overfit.ts | 195 +++++++++++++++++ packages/loopover-engine/src/index.ts | 1 + .../test/benchmark-anti-overfit.test.ts | 199 ++++++++++++++++++ 3 files changed, 395 insertions(+) create mode 100644 packages/loopover-engine/src/calibration/benchmark-anti-overfit.ts create mode 100644 packages/loopover-engine/test/benchmark-anti-overfit.test.ts diff --git a/packages/loopover-engine/src/calibration/benchmark-anti-overfit.ts b/packages/loopover-engine/src/calibration/benchmark-anti-overfit.ts new file mode 100644 index 0000000000..f60e2f6ea1 --- /dev/null +++ b/packages/loopover-engine/src/calibration/benchmark-anti-overfit.ts @@ -0,0 +1,195 @@ +// Anti-overfit controls for the public benchmark (#9263, harness #9216, epic #8534). +// +// A public leaderboard creates a direct incentive to overfit, and an overfit agent is worse than useless: it +// looks best exactly when it generalizes worst. The internal backtest gate already enforces this discipline; +// this module applies the same controls to EXTERNAL submissions, plus the ones only an adversarial public +// benchmark needs. Every knob below is one an adversary optimizes against, so each carries its reasoning +// rather than a bare value. +// +// ── DECISION 1: SPLIT GRANULARITY IS THE REPO, NOT THE WORK UNIT ────────────────────────────────────── +// The split reuses `splitBacktestCorpus` verbatim (requirement 1 — no second split mechanism), but the key +// it splits on is the REPO, so every work unit belonging to a repo lands in the same slice. Work-unit-level +// splitting leaks badly: two PRs in the same repo share a maintainer, a review culture, a CI setup and often +// the same files, so an agent that saw repo A's visible PRs has effectively seen the answer pattern for repo +// A's held-out PRs. Splitting per repo is what makes "held out" mean "generalizes to a repo it has never +// seen" — which is the property actually worth measuring, and the one #9216 requirement 3 names. +// +// ── DECISION 2: HELD-OUT SCORES ARE NOT PUBLISHED PER SUBMISSION ───────────────────────────────────── +// A leaderboard that reports a held-out score on every attempt converts the held-out set into a visible set +// within a few dozen submissions: each score is an oracle query, and differencing successive scores recovers +// per-unit membership. Held-out results publish on a fixed cadence or at evaluation close, never on demand; +// `heldOutPublicationDecision` is the single place that rule is decided, and #9265's emitter is where it is +// enforced, so a leaderboard cannot leak membership by accident. +// +// ── DECISION 3: SUBMISSION CAP PER (AGENT, WINDOW) ─────────────────────────────────────────────────── +// Unbounded resubmission against a fixed corpus is gradient descent on the test set by brute force — with +// enough attempts a random-search agent tops the board having learned nothing. The cap is per (agent, +// benchmark window) rather than global or per-day: a per-day cap just spreads the same brute force over more +// days, and a global cap would punish an agent that keeps competing across rotations. +// +// ── DECISION 4: ROTATION RETIRES A WINDOW, IT DOES NOT EXTEND IT ───────────────────────────────────── +// A benchmark public for a year is not a test set any more — it is training data with extra steps. A window +// has a fixed evaluation close; after it, the snapshot is RETIRED and a fresh one takes over. Retired +// windows stay published for historical comparison but stop being the scoring basis, which is exactly the +// distinction that keeps an old leaderboard honest instead of quietly authoritative. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads — the +// caller supplies `now` wherever a time comparison is needed, so every decision here is reproducible. + +import type { BacktestCase } from "./backtest-corpus.js"; +import { splitBacktestCorpus } from "./backtest-split.js"; + +/** Default share of REPOS (not work units — see decision 1) withheld from the visible slice. */ +export const DEFAULT_HELD_OUT_FRACTION = 0.25; + +/** Default submissions permitted per (agent, benchmark window). See decision 3 for why the cap is scoped + * this way; the value itself is deliberately generous enough for honest iteration and far too small for + * brute-force search over a fixed corpus. */ +export const DEFAULT_SUBMISSION_CAP = 20; + +export type BenchmarkSplitPolicy = { + /** Unguessable without knowing it; identical across runs. Never published while a window is open. */ + splitSeed: string; + heldOutFraction: number; +}; + +export type BenchmarkWorkUnitRef = { + workUnitId: string; + /** `owner/repo` — the split key (decision 1). */ + repoFullName: string; +}; + +/** + * Partition work units into visible and held-out slices at REPO granularity, through the shared + * `splitBacktestCorpus` primitive. + * + * The reuse is literal: one synthetic case per distinct repo, with the repo in the `targetKey` slot, so the + * assignment digest is byte-identically the one the internal backtest split already computes. A repo's + * assignment therefore depends only on (seed, benchmarkId, repo) — never on corpus size or ordering — so a + * benchmark that grows over time never reshuffles which repos were already held out. + */ +export function splitBenchmarkWorkUnits( + benchmarkId: string, + workUnits: readonly BenchmarkWorkUnitRef[], + policy: BenchmarkSplitPolicy, +): { visible: BenchmarkWorkUnitRef[]; heldOut: BenchmarkWorkUnitRef[]; heldOutRepoCount: number; visibleRepoCount: number } { + const repos = [...new Set(workUnits.map((unit) => unit.repoFullName))]; + const repoCases: BacktestCase[] = repos.map((repoFullName) => ({ + ruleId: benchmarkId, + targetKey: repoFullName, + outcome: "repo", + label: "confirmed", + firedAt: "", + decidedAt: "", + })); + const split = splitBacktestCorpus(repoCases, policy.heldOutFraction, policy.splitSeed); + const heldOutRepos = new Set(split.heldOut.map((repoCase) => repoCase.targetKey)); + const visible: BenchmarkWorkUnitRef[] = []; + const heldOut: BenchmarkWorkUnitRef[] = []; + for (const unit of workUnits) { + if (heldOutRepos.has(unit.repoFullName)) heldOut.push(unit); + else visible.push(unit); + } + return { visible, heldOut, heldOutRepoCount: heldOutRepos.size, visibleRepoCount: repos.length - heldOutRepos.size }; +} + +export type BenchmarkWindow = { + benchmarkId: string; + snapshotRef: string; + opensAt: string; + /** Inclusive evaluation close. After this the window is RETIRED (decision 4), not extended. */ + closesAt: string; + /** Held-out scores may also publish periodically before close; 0/absent means "at close only". */ + heldOutPublishEveryDays?: number | undefined; +}; + +export type BenchmarkWindowState = "pending" | "open" | "retired"; + +/** Where `now` sits relative to a window. A retired window stays readable for historical comparison but is + * no longer a scoring basis — the caller enforces that by refusing submissions (below). */ +export function benchmarkWindowState(window: BenchmarkWindow, now: string): BenchmarkWindowState { + const at = Date.parse(now); + if (at < Date.parse(window.opensAt)) return "pending"; + return at > Date.parse(window.closesAt) ? "retired" : "open"; +} + +export type SubmissionDecision = + | { accepted: true; submissionIndex: number; remaining: number } + | { accepted: false; reason: "window_not_open" | "window_retired" | "cap_reached"; remaining: number }; + +/** + * Decide whether one more submission from this agent is admissible. + * + * PURE and total: it never throws for ordinarily-invalid input, and the refusal reason is NAMED so a + * submitter learns which control stopped them (a cap they can wait out, versus a window that will never + * reopen) rather than seeing an opaque rejection. + */ +export function decideSubmission(input: { + window: BenchmarkWindow; + now: string; + /** Submissions this agent has already made against THIS window. */ + priorSubmissions: number; + cap?: number | undefined; +}): SubmissionDecision { + const cap = input.cap ?? DEFAULT_SUBMISSION_CAP; + const remaining = Math.max(0, cap - input.priorSubmissions); + const state = benchmarkWindowState(input.window, input.now); + if (state === "pending") return { accepted: false, reason: "window_not_open", remaining }; + if (state === "retired") return { accepted: false, reason: "window_retired", remaining }; + if (remaining <= 0) return { accepted: false, reason: "cap_reached", remaining: 0 }; + return { accepted: true, submissionIndex: input.priorSubmissions + 1, remaining: remaining - 1 }; +} + +export type HeldOutPublicationDecision = + | { publish: true; reason: "evaluation_closed" | "scheduled_cadence" } + | { publish: false; reason: "window_open_between_cadences" | "window_not_open" }; + +/** + * Decide whether held-out scores may be published right now (decision 2). + * + * At or after close, always — the window is over and there is nothing left to leak into. Before close, only + * on the configured cadence boundary, counted in whole periods since `opensAt`, so the publication instants + * are a fixed public schedule rather than something a submitter can trigger by submitting. + */ +export function heldOutPublicationDecision(window: BenchmarkWindow, now: string): HeldOutPublicationDecision { + const state = benchmarkWindowState(window, now); + if (state === "pending") return { publish: false, reason: "window_not_open" }; + if (state === "retired") return { publish: true, reason: "evaluation_closed" }; + const everyDays = window.heldOutPublishEveryDays ?? 0; + if (everyDays <= 0) return { publish: false, reason: "window_open_between_cadences" }; + const elapsedMs = Date.parse(now) - Date.parse(window.opensAt); + const periodMs = everyDays * 24 * 60 * 60 * 1000; + // A boundary instant publishes; anything strictly between boundaries does not. + return elapsedMs > 0 && elapsedMs % periodMs === 0 + ? { publish: true, reason: "scheduled_cadence" } + : { publish: false, reason: "window_open_between_cadences" }; +} + +/** What a leaderboard is allowed to show for one submission while its window is open. */ +export type PublishableScores = { + visible: T; + /** Present ONLY when {@link heldOutPublicationDecision} allowed it. `null` is the honest "not published + * yet", and carries no count, no fraction and no per-unit membership — anything derived from the + * held-out slice is an oracle query, so the shape itself refuses to answer one. */ + heldOut: T | null; + heldOutPublication: HeldOutPublicationDecision; +}; + +/** + * Gate a scored pair through the publication policy. + * + * The held-out score is DROPPED, not merely hidden behind a flag: a caller cannot forget to check the flag + * and serialize a field that is present in memory. That is the difference between a policy and a habit. + */ +export function gateHeldOutPublication( + window: BenchmarkWindow, + now: string, + scores: { visible: T; heldOut: T }, +): PublishableScores { + const decision = heldOutPublicationDecision(window, now); + return { + visible: scores.visible, + heldOut: decision.publish ? scores.heldOut : null, + heldOutPublication: decision, + }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 5989875d62..d9fb9903d7 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -186,6 +186,7 @@ export * from "./calibration/attester.js"; export * from "./calibration/benchmark-proposal.js"; export * from "./calibration/benchmark-ground-truth.js"; export * from "./calibration/benchmark-score.js"; +export * from "./calibration/benchmark-anti-overfit.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/benchmark-anti-overfit.test.ts b/packages/loopover-engine/test/benchmark-anti-overfit.test.ts new file mode 100644 index 0000000000..9a582af87a --- /dev/null +++ b/packages/loopover-engine/test/benchmark-anti-overfit.test.ts @@ -0,0 +1,199 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + benchmarkWindowState, + decideSubmission, + DEFAULT_HELD_OUT_FRACTION, + DEFAULT_SUBMISSION_CAP, + gateHeldOutPublication, + heldOutPublicationDecision, + splitBacktestCorpus, + splitBenchmarkWorkUnits, + type BenchmarkWindow, + type BenchmarkWorkUnitRef, +} from "../dist/index.js"; + +// #9263 (harness #9216, epic #8534): the controls that make a leaderboard position mean "generalizes" +// rather than "has submitted the most times". Each of the four recorded decisions in the module header has +// its enforcing test here, and the load-bearing one is that held-out MEMBERSHIP is not inferable from +// anything the published shape can carry. + +const POLICY = { splitSeed: "seed-2026q3", heldOutFraction: DEFAULT_HELD_OUT_FRACTION }; + +/** Work units across many repos, several units per repo — the shape decision 1 is about. */ +function unitsAcrossRepos(repoCount: number, perRepo = 3): BenchmarkWorkUnitRef[] { + const units: BenchmarkWorkUnitRef[] = []; + for (let repo = 0; repo < repoCount; repo += 1) { + for (let index = 0; index < perRepo; index += 1) { + units.push({ workUnitId: `owner/repo${repo}#${index}`, repoFullName: `owner/repo${repo}` }); + } + } + return units; +} + +const WINDOW: BenchmarkWindow = { + benchmarkId: "bench-2026q3", + snapshotRef: "a1".repeat(32), + opensAt: "2026-07-01T00:00:00.000Z", + closesAt: "2026-09-30T00:00:00.000Z", +}; + +test("DECISION 1: the split is at REPO granularity — a repo's units never straddle the boundary", () => { + const units = unitsAcrossRepos(24); + const split = splitBenchmarkWorkUnits("bench-2026q3", units, POLICY); + const heldOutRepos = new Set(split.heldOut.map((unit) => unit.repoFullName)); + const visibleRepos = new Set(split.visible.map((unit) => unit.repoFullName)); + // The two repo sets are disjoint: no repo contributes to both slices, which is the whole property. + for (const repo of heldOutRepos) assert.equal(visibleRepos.has(repo), false, `${repo} straddles the split`); + assert.equal(heldOutRepos.size + visibleRepos.size, 24); + assert.equal(split.heldOutRepoCount, heldOutRepos.size); + assert.equal(split.visibleRepoCount, visibleRepos.size); + // Every unit is accounted for exactly once. + assert.equal(split.visible.length + split.heldOut.length, units.length); +}); + +test("DECISION 1: the assignment IS the shared primitive's, not a lookalike", () => { + // Recomputing through splitBacktestCorpus directly must name the same held-out repos — a second split + // mechanism is exactly what requirement 1 forbids, so the equality is asserted rather than assumed. + const units = unitsAcrossRepos(16); + const repos = [...new Set(units.map((unit) => unit.repoFullName))]; + const direct = splitBacktestCorpus( + repos.map((repoFullName) => ({ + ruleId: "bench-2026q3", + targetKey: repoFullName, + outcome: "repo", + label: "confirmed" as const, + firedAt: "", + decidedAt: "", + })), + POLICY.heldOutFraction, + POLICY.splitSeed, + ); + const split = splitBenchmarkWorkUnits("bench-2026q3", units, POLICY); + assert.deepEqual( + [...new Set(split.heldOut.map((unit) => unit.repoFullName))].sort(), + direct.heldOut.map((repoCase) => repoCase.targetKey).sort(), + ); +}); + +test("PROPERTY: assignment is deterministic, seed-dependent, and stable as the corpus grows", () => { + const small = unitsAcrossRepos(12); + const grown = [...small, ...unitsAcrossRepos(20).slice(36)]; + const a = splitBenchmarkWorkUnits("b", small, POLICY); + const b = splitBenchmarkWorkUnits("b", small, POLICY); + assert.deepEqual(a, b, "identical inputs must give identical output"); + // A different seed reassigns; without the seed, membership is unguessable. + const other = splitBenchmarkWorkUnits("b", small, { ...POLICY, splitSeed: "different" }); + assert.notDeepEqual( + [...new Set(other.heldOut.map((unit) => unit.repoFullName))].sort(), + [...new Set(a.heldOut.map((unit) => unit.repoFullName))].sort(), + ); + // Growth never reshuffles an already-assigned repo — the property backtest-split.ts exists to guarantee. + const grownSplit = splitBenchmarkWorkUnits("b", grown, POLICY); + const grownHeldOutRepos = new Set(grownSplit.heldOut.map((unit) => unit.repoFullName)); + for (const unit of a.heldOut) assert.ok(grownHeldOutRepos.has(unit.repoFullName), `${unit.repoFullName} moved`); +}); + +test("the split degenerates cleanly at both extremes and on an empty corpus", () => { + const units = unitsAcrossRepos(8); + assert.equal(splitBenchmarkWorkUnits("b", units, { ...POLICY, heldOutFraction: 0 }).heldOut.length, 0); + assert.equal(splitBenchmarkWorkUnits("b", units, { ...POLICY, heldOutFraction: 1 }).visible.length, 0); + const empty = splitBenchmarkWorkUnits("b", [], POLICY); + assert.deepEqual(empty, { visible: [], heldOut: [], heldOutRepoCount: 0, visibleRepoCount: 0 }); + // An out-of-range fraction fails closed through the primitive's own guard, rather than splitting nothing. + assert.throws(() => splitBenchmarkWorkUnits("b", units, { ...POLICY, heldOutFraction: 1.5 }), /invalid_held_out_fraction/); +}); + +test("benchmarkWindowState: pending before opening, open through the inclusive close, retired after", () => { + assert.equal(benchmarkWindowState(WINDOW, "2026-06-30T23:59:59.000Z"), "pending"); + assert.equal(benchmarkWindowState(WINDOW, WINDOW.opensAt), "open"); + assert.equal(benchmarkWindowState(WINDOW, "2026-08-01T00:00:00.000Z"), "open"); + assert.equal(benchmarkWindowState(WINDOW, WINDOW.closesAt), "open"); + assert.equal(benchmarkWindowState(WINDOW, "2026-10-01T00:00:00.000Z"), "retired"); +}); + +test("DECISION 3: the submission cap is enforced per (agent, window) and names its refusal", () => { + const now = "2026-08-01T00:00:00.000Z"; + const first = decideSubmission({ window: WINDOW, now, priorSubmissions: 0 }); + assert.deepEqual(first, { accepted: true, submissionIndex: 1, remaining: DEFAULT_SUBMISSION_CAP - 1 }); + const last = decideSubmission({ window: WINDOW, now, priorSubmissions: DEFAULT_SUBMISSION_CAP - 1 }); + assert.deepEqual(last, { accepted: true, submissionIndex: DEFAULT_SUBMISSION_CAP, remaining: 0 }); + // Brute force stops here — the refusal says WHICH control fired. + assert.deepEqual(decideSubmission({ window: WINDOW, now, priorSubmissions: DEFAULT_SUBMISSION_CAP }), { + accepted: false, + reason: "cap_reached", + remaining: 0, + }); + // An over-count (a replayed or double-recorded submission) still refuses, and never reports negative room. + assert.deepEqual(decideSubmission({ window: WINDOW, now, priorSubmissions: 999 }), { accepted: false, reason: "cap_reached", remaining: 0 }); + // An explicit cap overrides the default in both directions. + assert.equal(decideSubmission({ window: WINDOW, now, priorSubmissions: 2, cap: 3 }).accepted, true); + assert.equal(decideSubmission({ window: WINDOW, now, priorSubmissions: 3, cap: 3 }).accepted, false); +}); + +test("DECISION 4: a retired window refuses submissions; so does one that has not opened", () => { + assert.deepEqual(decideSubmission({ window: WINDOW, now: "2026-10-02T00:00:00.000Z", priorSubmissions: 0 }), { + accepted: false, + reason: "window_retired", + remaining: DEFAULT_SUBMISSION_CAP, + }); + assert.deepEqual(decideSubmission({ window: WINDOW, now: "2026-06-01T00:00:00.000Z", priorSubmissions: 0 }), { + accepted: false, + reason: "window_not_open", + remaining: DEFAULT_SUBMISSION_CAP, + }); + // A retired window refuses even with room left — rotation is not extended by an unused quota. + assert.equal(decideSubmission({ window: WINDOW, now: "2026-12-01T00:00:00.000Z", priorSubmissions: 1 }).accepted, false); +}); + +test("DECISION 2: held-out publishes at close, never on demand while the window is open", () => { + assert.deepEqual(heldOutPublicationDecision(WINDOW, "2026-08-15T00:00:00.000Z"), { + publish: false, + reason: "window_open_between_cadences", + }); + assert.deepEqual(heldOutPublicationDecision(WINDOW, "2026-10-05T00:00:00.000Z"), { publish: true, reason: "evaluation_closed" }); + assert.deepEqual(heldOutPublicationDecision(WINDOW, "2026-06-01T00:00:00.000Z"), { publish: false, reason: "window_not_open" }); +}); + +test("DECISION 2: a configured cadence publishes on the fixed schedule only, not on submission", () => { + const cadenced: BenchmarkWindow = { ...WINDOW, heldOutPublishEveryDays: 30 }; + // Exactly 30 and 60 days after opening: scheduled boundaries. + assert.deepEqual(heldOutPublicationDecision(cadenced, "2026-07-31T00:00:00.000Z"), { publish: true, reason: "scheduled_cadence" }); + assert.deepEqual(heldOutPublicationDecision(cadenced, "2026-08-30T00:00:00.000Z"), { publish: true, reason: "scheduled_cadence" }); + // A day either side of a boundary — and the opening instant itself — publish nothing. + for (const between of ["2026-07-30T00:00:00.000Z", "2026-08-01T00:00:00.000Z", WINDOW.opensAt]) { + assert.equal(heldOutPublicationDecision(cadenced, between).publish, false, between); + } +}); + +test("REGRESSION: a gated payload DROPS the held-out score rather than flagging it, so it cannot be serialized by mistake", () => { + const scores = { visible: { macro: 0.7 }, heldOut: { macro: 0.4 } }; + const open = gateHeldOutPublication(WINDOW, "2026-08-15T00:00:00.000Z", scores); + assert.equal(open.heldOut, null); + // The held-out value is absent from the payload ENTIRELY -- no count, no fraction, nothing an observer + // could difference across submissions to recover per-unit membership. + assert.equal(JSON.stringify(open).includes("0.4"), false); + assert.deepEqual(open.visible, { macro: 0.7 }); + // After close there is nothing left to leak into, so the real score is published. + const closed = gateHeldOutPublication(WINDOW, "2026-10-05T00:00:00.000Z", scores); + assert.deepEqual(closed.heldOut, { macro: 0.4 }); + assert.deepEqual(closed.heldOutPublication, { publish: true, reason: "evaluation_closed" }); +}); + +test("REGRESSION: nothing in a published open-window payload reveals held-out MEMBERSHIP", () => { + // The adversary's question is "is repo X held out?". The published shape must not answer it, directly or + // by differencing: it carries the visible slice and a policy decision, and no held-out identifiers. + const units = unitsAcrossRepos(12); + const split = splitBenchmarkWorkUnits("bench-2026q3", units, POLICY); + const published = gateHeldOutPublication(WINDOW, "2026-08-15T00:00:00.000Z", { + visible: { units: split.visible.map((unit) => unit.workUnitId) }, + heldOut: { units: split.heldOut.map((unit) => unit.workUnitId) }, + }); + const serialized = JSON.stringify(published); + for (const unit of split.heldOut) { + assert.equal(serialized.includes(unit.workUnitId), false, `${unit.workUnitId} leaked`); + assert.equal(serialized.includes(unit.repoFullName), false, `${unit.repoFullName} leaked`); + } + assert.ok(split.heldOut.length > 0, "fixture must actually hold something out"); +});