From 7efacf772d94e8e35c343b480b25bc6aa5309d34 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 11:34:53 +0000 Subject: [PATCH 1/4] feat(ci): the shard-timings generator writes per-package suite-duration ceilings Each package's ceiling is its slowest executed run in the refresh window times 1.25, written as a `ceilings` field beside the median weights, with `uncapped` naming the packages that get none (provisional, or carried with no executed run in the window). A refresh holds every ceiling it already set: it never raises one (a raise is a ruling) and never lowers one on a quieter window. The spread table the refresh PR carries gains the ceiling column. Claude-Session: https://claude.ai/code/session_0115N1oNnQS5WqofZ2DzaT3q Co-authored-by: Claude --- scripts/measure-test-shard-timings.mjs | 286 +++++++++++++++++++++++-- 1 file changed, 274 insertions(+), 12 deletions(-) diff --git a/scripts/measure-test-shard-timings.mjs b/scripts/measure-test-shard-timings.mjs index 43b32dd6bb1..4c08f3d8463 100644 --- a/scripts/measure-test-shard-timings.mjs +++ b/scripts/measure-test-shard-timings.mjs @@ -155,6 +155,92 @@ export function median(values) { // lane exists to end. export const MINIMUM_EXECUTED_RUNS = 3; +// The headroom of the per-package suite-duration ceiling (#16468). A ruled +// number, not a tunable: the maintainer's ruling put it at 25% over "the +// refreshed measurement", and the ruling that followed read that measurement +// as the package's SLOWEST executed run in the refresh window -- not its +// median, which is this file's weight. Measured on the 21-run refresh that +// landed as 040184752c: 31 of the 71 measured packages had a window maximum +// above median x 1.25, so a ceiling built on the median would have been red on +// the dataset's own runs, the standing-red shape a brake is learned around. +// +// The ceiling travels with the dataset as its own field (`ceilings`), beside +// the weights rather than in place of them: the partitioner and the drift gate +// keep reading `packages` as name -> median, unchanged. +export const CEILING_HEADROOM = 1.25; + +// One package's ceiling from its executed per-run readings in this refresh: +// the slowest of them times the headroom, rounded UP to the hundredth, so the +// rounding can never put the window's own slowest run over its own line. +// A file-sharded package's readings are already per-run SUMS of its slices +// (collectSamples), so this is the slowest whole-suite run, never one slice. +export function ceilingOf(values, headroom = CEILING_HEADROOM) { + if (!Array.isArray(values) || values.length === 0) throw new Error('ceilingOf: no executed readings'); + return Math.ceil(Math.max(...values) * headroom * 100) / 100; +} + +// The per-package ceilings a refresh writes (#16468), and the packages that +// get none, with the reason. Every package in `packages` lands in exactly one +// of the two. +// +// HELD a package the prior dataset already gave a ceiling keeps it, +// byte for byte. A refresh NEVER raises a ceiling -- the ruling +// is "a ceiling rises only by a ruling", and a refresh that +// re-derived it from a slower window would raise it by the very +// growth the ratchet exists to stop (the line ratchets' rule, +// which the card cites). A raise is a ruled record in +// scripts/check-test-suite-ceilings.mjs, never an edit here. +// Nor does a refresh LOWER one: a quieter window is variance, not +// a faster suite, and a ceiling ratcheted down on variance is red +// on the next ordinary run. +// NEW a package with no prior ceiling, not provisional, and executed +// in this refresh: its slowest executed run x the headroom. +// uncapped `provisional` -- fewer than `minimumRuns` executed runs is not a +// baseline (#22014), so no ceiling and, per the ruling, no red; +// `carried` -- re-confirmed by a cache HIT but executed in no run +// of this window, so there is no slowest run to read. The next +// refresh that executes it gives it one. +// +// A prior ceiling for a package no longer in `packages` is dropped with it. +export function suiteCeilings({ bySample, packages, provisional, prior = null, headroom = CEILING_HEADROOM }) { + if (prior !== null) { + if (typeof prior !== 'object' || Array.isArray(prior)) { + throw new Error( + `the prior dataset's \`ceilings\` is ${JSON.stringify(prior)}, not a map of package -> seconds. ` + + 'Refusing to guess which ceilings to hold.' + ); + } + for (const [name, seconds] of Object.entries(prior)) { + if (typeof seconds !== 'number' || !Number.isFinite(seconds) || !(seconds > 0)) { + throw new Error( + `the prior dataset's ceiling for ${name} is ${JSON.stringify(seconds)}, not a positive duration. ` + + 'Refusing to hold or replace it by guessing.' + ); + } + } + } + const short = new Set(provisional); + const ceilings = {}; + const uncapped = {}; + for (const name of Object.keys(packages).sort((a, b) => a.localeCompare(b, 'en'))) { + if (prior !== null && Object.hasOwn(prior, name)) { + ceilings[name] = prior[name]; + continue; + } + if (short.has(name)) { + uncapped[name] = 'provisional'; + continue; + } + const values = bySample.get(name); + if (!values || values.length === 0) { + uncapped[name] = 'carried'; + continue; + } + ceilings[name] = ceilingOf(values, headroom); + } + return { ceilings, uncapped }; +} + // Which file-level slice a summary's tasks were run as, or null for a whole // package (#16173). // @@ -569,6 +655,9 @@ export function buildDataset({ provenance, carryFrom = null, carryProvisional = null, + // The prior dataset's `ceilings`, or null when it has none -- a dataset + // written before the ceilings existed, read as "nothing to hold" (#16468). + carryCeilings = null, minimumRuns = MINIMUM_EXECUTED_RUNS, }) { const { bySample, cachedNames, incompleteSlices } = collectSamples(perSummary); @@ -628,6 +717,8 @@ export function buildDataset({ if (!Number.isInteger(minimumRuns) || minimumRuns < 1) { throw new Error(`minimumRuns must be a positive integer, got ${JSON.stringify(minimumRuns)}`); } + const provisional = provisionalPackages({ bySample, carriedOver, carryProvisional, minimumRuns }); + const { ceilings, uncapped } = suiteCeilings({ bySample, packages, provisional, prior: carryCeilings }); return { note: 'GENERATED by scripts/measure-test-shard-timings.mjs -- do not hand-edit. Per-package ' + @@ -636,10 +727,15 @@ export function buildDataset({ 'Every weight is a real measurement: the ones in `carriedOver` were measured by an earlier ' + 'refresh and re-confirmed unchanged by a turbo cache HIT in this one. A weight is the median ' + 'of the executed runs that measured it; the ones in `provisional` rest on fewer than ' + - '`provenance.minimumRuns` of them -- fit to balance on, NOT a baseline to set a ceiling from.', + '`provenance.minimumRuns` of them -- fit to balance on, NOT a baseline to set a ceiling from. ' + + '`ceilings` is each package\'s suite-duration ceiling, graded by scripts/check-test-suite-ceilings.mjs: ' + + 'its slowest executed run in the refresh window that first set it, times `provenance.ceilingHeadroom`, ' + + 'then held by every later refresh -- raised only by a ruling, never by a refresh. `uncapped` names ' + + 'the packages with none, and why.', // The bar `provisional` was judged against travels with the list, so a // reader never has to know which version of this script wrote the file. - provenance: { ...provenance, minimumRuns }, + // The ceiling headroom travels the same way (#16468). + provenance: { ...provenance, minimumRuns, ceilingHeadroom: CEILING_HEADROOM }, secondsPerTestFileFallback: fallbackRate(measured, fileCounts), packages, skippedAsCached: [...cachedNames].sort((a, b) => a.localeCompare(b, 'en')), @@ -657,14 +753,22 @@ export function buildDataset({ // the drift gate read `packages` as name -> number. Always present -- an // empty list and an absent key must not read the same way, since an absent // key is exactly how a dataset that predates this rule is recognised. - provisional: provisionalPackages({ bySample, carriedOver, carryProvisional, minimumRuns }), + provisional, + // #16468. The per-package suite-duration ceilings and the packages with + // none (see suiteCeilings). Beside the weights, never in place of them -- + // the same reason as `carriedOver` and `provisional`. Always present, for + // the same reason as `provisional`: an ABSENT `ceilings` is how the check + // recognises a dataset written before the ceilings existed, which it reads + // as NOT MEASURED rather than as a table with no rows. + ceilings, + uncapped, }; } // The run-to-run spread of every package, as the Markdown table the refresh // PR's body carries (#22014). One row per package in the dataset: // -// package | runs | min | median | max | max/min +// package | runs | min | median | max | max/min | ceiling // // where `runs` counts the executed runs behind the weight and the three // seconds columns are those runs' readings. A carried package has no reading in @@ -672,6 +776,13 @@ export function buildDataset({ // one is marked in its own column. Rows sort by max/min, widest first, so the // noisiest suites are the ones a reviewer meets first. // +// `ceiling` (#16468) is the dataset's own `ceilings` entry, `—` for an +// uncapped package with the reason in the last column. A HELD ceiling that this +// window would have set differently says so -- `held; this window: N` -- so the +// reviewer sees the growth a refresh deliberately did not absorb (a raise is a +// ruling), and a window reading already OVER its held ceiling is called out +// rather than left to be inferred from two columns. +// // ⛔ Kept OUT of the dataset file. Nothing machine-reads a spread -- the // partitioner and the drift gate read the weights, and the one machine fact a // baseline needs (did this weight meet the minimum?) is `provisional` -- so the @@ -679,12 +790,30 @@ export function buildDataset({ export function renderSpread({ bySample, dataset }) { const fmt = (n) => n.toFixed(2); const provisional = new Set(dataset.provisional); + const ceilings = dataset.ceilings ?? {}; + const uncapped = dataset.uncapped ?? {}; + const headroom = dataset.provenance?.ceilingHeadroom ?? CEILING_HEADROOM; + const ceilingCells = (name, values) => { + const flags = []; + if (provisional.has(name)) flags.push('provisional'); + if (!Object.hasOwn(ceilings, name)) { + if (uncapped[name] === 'carried') flags.push('carried: no ceiling'); + return ['—', flags.join('; ')]; + } + const ceiling = ceilings[name]; + if (values && !provisional.has(name)) { + const fresh = ceilingOf(values, headroom); + if (fresh !== ceiling) flags.push(`held; this window: ${fmt(fresh)}`); + if (Math.max(...values) > ceiling) flags.push('⚠ a window run is OVER its held ceiling'); + } + return [fmt(ceiling), flags.join('; ')]; + }; const rows = []; for (const name of Object.keys(dataset.packages)) { const values = bySample.get(name); - const flag = provisional.has(name) ? 'provisional' : ''; + const [ceilingCell, flag] = ceilingCells(name, values); if (!values) { - rows.push({ ratio: -1, name, cells: [`\`${name}\``, '0', 'carried', 'carried', 'carried', '—', flag] }); + rows.push({ ratio: -1, name, cells: [`\`${name}\``, '0', 'carried', 'carried', 'carried', '—', ceilingCell, flag] }); continue; } const lo = Math.min(...values); @@ -700,6 +829,7 @@ export function renderSpread({ bySample, dataset }) { fmt(Math.round(median(values) * 100) / 100), fmt(hi), Number.isFinite(ratio) ? `${ratio.toFixed(2)}×` : '∞', + ceilingCell, flag, ], }); @@ -707,10 +837,12 @@ export function renderSpread({ bySample, dataset }) { rows.sort((a, b) => b.ratio - a.ratio || a.name.localeCompare(b.name, 'en')); const minimumRuns = dataset.provenance?.minimumRuns; const lines = [ - `Each package's executed runs in this refresh (minimum ${minimumRuns}), and the spread of its readings in seconds.`, + `Each package's executed runs in this refresh (minimum ${minimumRuns}), the spread of its readings in seconds, ` + + `and the suite-duration ceiling \`Test Core\` grades it against: its slowest executed run x ${headroom} when ` + + 'first set, then held by every refresh -- a ceiling is raised only by a ruling.', '', - '| package | runs | min s | median s | max s | max/min | |', - '|---|--:|--:|--:|--:|--:|---|', + '| package | runs | min s | median s | max s | max/min | ceiling s | |', + '|---|--:|--:|--:|--:|--:|--:|---|', ...rows.map((r) => `| ${r.cells.join(' | ')} |`), ]; return `${lines.join('\n')}\n`; @@ -748,11 +880,13 @@ const SELF_TEST_BATTERIES = Object.freeze({ 'env-carried slices (#19278)': 11, // #22014: the per-package minimum of executed runs, and the spread table. 'minimum executed runs per package': 14, + // #16468: the per-package suite-duration ceilings the refresh writes. + 'suite-duration ceilings (#16468)': 14, }); // DELETING an entry silences that battery's floor exactly as effectively as // zeroing it, so the roster's own size is pinned too. -const SELF_TEST_BATTERY_FLOOR = 3; +const SELF_TEST_BATTERY_FLOOR = 4; // The key an assertion is filed under when no battery is open. It is not a // declared battery, so it reds by the same set difference rather than silently @@ -1664,14 +1798,116 @@ function selfTest() { const table = renderSpread({ bySample: collectSamples(runs).bySample, dataset: ds }); const body = table.split('\n').filter((l) => l.startsWith('| `')); if (body.length !== 3) throw new Error(`spread: expected one row per package, got ${body.length}:\n${table}`); - if (body[0] !== '| `a` | 3 | 10.00 | 20.00 | 30.00 | 3.00× | |') throw new Error(`spread: the widest row is wrong: ${body[0]}`); - if (!body.some((l) => l.startsWith('| `b` | 1 | 20.00 | 20.00 | 20.00 | 1.00× | provisional |'))) { + if (body[0] !== '| `a` | 3 | 10.00 | 20.00 | 30.00 | 3.00× | 37.50 | |') throw new Error(`spread: the widest row is wrong: ${body[0]}`); + if (!body.some((l) => l.startsWith('| `b` | 1 | 20.00 | 20.00 | 20.00 | 1.00× | — | provisional |'))) { throw new Error(`spread: the provisional row is wrong:\n${table}`); } if (!body.some((l) => l.startsWith('| `cached` | 0 | carried |'))) throw new Error(`spread: the carried row is wrong:\n${table}`); if (!table.includes(`minimum ${MINIMUM_EXECUTED_RUNS}`)) throw new Error('spread: the table does not state the minimum'); }); + // -- The suite-duration ceilings (#16468) ------------------------------- + // + // The ruled reading: a package's ceiling is its SLOWEST executed run in the + // refresh window x 1.25 (not its median, which is the weight), a provisional + // package gets none, and a refresh never moves a ceiling it already set. + battery('suite-duration ceilings (#16468)'); + const windowRuns = [ + runOf('r1', [testTask('a', 0, 10_000), testTask('b', 0, 20_000), testTask('cached', 0, 90, 'HIT')]), + runOf('r2', [testTask('a', 0, 30_000), testTask('cached', 0, 90, 'HIT')]), + runOf('r3', [testTask('a', 0, 20_000), testTask('cached', 0, 90, 'HIT')]), + ]; + const ceilingSet = (extra = {}) => depthSet(windowRuns, { carryFrom: { cached: 500 }, carryProvisional: [], ...extra }); + check(() => { + // The ruled headroom. Changing it is a ruling, not a tune. + if (CEILING_HEADROOM !== 1.25) throw new Error(`ceilings: CEILING_HEADROOM is ${CEILING_HEADROOM}, not the ruled 1.25`); + }); + check(() => { + // THE RULED READING: the window MAXIMUM (30s) x 1.25 = 37.5s, not the + // median (20s) x 1.25 = 25s -- which the window's own slowest run is over. + const first = ceilingSet(); + if (first.ceilings.a !== 37.5) throw new Error(`ceilings: a read ${first.ceilings.a}, not its slowest run x 1.25 (37.5)`); + }); + check(() => { + // The weights are untouched: `a` still weighs its median. + if (ceilingSet().packages.a !== 20) throw new Error(`ceilings: the median weight moved (${ceilingSet().packages.a})`); + }); + check(() => { + // Provisional: one executed run is not a baseline -- no ceiling, named why. + const first = ceilingSet(); + if (Object.hasOwn(first.ceilings, 'b') || first.uncapped.b !== 'provisional') { + throw new Error(`ceilings: a provisional package got ${JSON.stringify(first.ceilings.b)} / ${JSON.stringify(first.uncapped.b)}`); + } + }); + check(() => { + // Carried and not provisional, with nothing to hold: no slowest run exists + // in this window, so no ceiling, and the reason is named. + const first = ceilingSet(); + if (Object.hasOwn(first.ceilings, 'cached') || first.uncapped.cached !== 'carried') { + throw new Error(`ceilings: a carried package got ${JSON.stringify(first.ceilings.cached)} / ${JSON.stringify(first.uncapped.cached)}`); + } + }); + check(() => { + // Every package lands in exactly one of the two maps. + const first = ceilingSet(); + const names = Object.keys(first.packages); + const both = names.filter((n) => Object.hasOwn(first.ceilings, n) === Object.hasOwn(first.uncapped, n)); + if (both.length) throw new Error(`ceilings: ${both.join(', ')} is in both maps or in neither`); + }); + check(() => { + // HELD, never raised: a prior 30s stays 30s although this window gives 37.5. + const held = ceilingSet({ carryCeilings: { a: 30 } }); + if (held.ceilings.a !== 30) throw new Error(`ceilings: a refresh raised a held ceiling to ${held.ceilings.a}`); + }); + check(() => { + // HELD, never lowered: a quieter window is variance, not a faster suite. + const held = ceilingSet({ carryCeilings: { a: 100 } }); + if (held.ceilings.a !== 100) throw new Error(`ceilings: a refresh lowered a held ceiling to ${held.ceilings.a}`); + }); + check(() => { + // A carried package keeps the ceiling an earlier window gave it, and a + // ceiling for a package gone from the dataset goes with it. + const held = ceilingSet({ carryCeilings: { cached: 640, gone: 9 } }); + if (held.ceilings.cached !== 640 || Object.hasOwn(held.uncapped, 'cached')) { + throw new Error(`ceilings: a carried package lost its held ceiling (${JSON.stringify(held.ceilings)})`); + } + if (Object.hasOwn(held.ceilings, 'gone')) throw new Error('ceilings: a ceiling outlived its package'); + }); + check(() => { + // A malformed prior table is refused, never guessed around. + if (!threw(() => ceilingSet({ carryCeilings: [30] }))) throw new Error('ceilings: a prior list was read as a map'); + if (!threw(() => ceilingSet({ carryCeilings: { a: -1 } }))) throw new Error('ceilings: a negative prior ceiling was held'); + }); + check(() => { + // A file-sharded package's ceiling is from its per-run SUMS: the slowest + // whole run (650s), never the slowest slice (330s). + const sliceRun = (run, x, y) => [ + runOf(run, [slicedTask('cli', 0, x, 1, 2)]), + runOf(run, [slicedTask('cli', 0, y, 2, 2)]), + ]; + const ds = depthSet([...sliceRun('r1', 300_000, 300_000), ...sliceRun('r2', 310_000, 310_000), ...sliceRun('r3', 320_000, 330_000)]); + if (ds.ceilings.cli !== 812.5) throw new Error(`ceilings: a sliced package read ${ds.ceilings.cli}, not its slowest whole run x 1.25 (812.5)`); + }); + check(() => { + // The headroom travels with the table, like the minimum. + if (ceilingSet().provenance.ceilingHeadroom !== CEILING_HEADROOM) throw new Error('ceilings: provenance does not record the headroom'); + }); + check(() => { + // Rounding is UP, so the window's slowest run is never over its own line. + if (ceilingOf([1.0001]) < 1.0001 * CEILING_HEADROOM) throw new Error(`ceilings: rounding put the line under max x headroom (${ceilingOf([1.0001])})`); + if (!threw(() => ceilingOf([]))) throw new Error('ceilings: an empty reading set produced a ceiling'); + }); + check(() => { + // The spread table shows a held ceiling this window would have set + // differently, and a run already over it. + const held = ceilingSet({ carryCeilings: { a: 25 } }); + const table = renderSpread({ bySample: collectSamples(windowRuns).bySample, dataset: held }); + const row = table.split('\n').find((l) => l.startsWith('| `a` |')); + if (row !== '| `a` | 3 | 10.00 | 20.00 | 30.00 | 3.00× | 25.00 | held; this window: 37.50; ⚠ a window run is OVER its held ceiling |') { + throw new Error(`spread: the held row is wrong: ${row}`); + } + }); + // -- The floor: every declared battery RAN, and ran its cases (#13489) ---- // // Evaluated after every battery has had its chance and BEFORE the verdict, so @@ -1840,6 +2076,7 @@ function main() { // from the numbers actually in the file rather than from a subset of them. let carryFrom = null; let carryProvisional = null; + let carryCeilings = null; if (mergeInto !== null) { const prior = JSON.parse(readFileSync(mergeInto, 'utf8')); if (!prior || typeof prior.packages !== 'object' || prior.packages === null) { @@ -1853,6 +2090,10 @@ function main() { // dataset with no `provisional` key never recorded one, which buildDataset // reads as "not shown to meet the minimum" -- null, deliberately not []. carryProvisional = Object.hasOwn(prior, 'provisional') ? prior.provisional : null; + // The ceilings it set are HELD (#16468): a refresh never raises or lowers + // one. A prior dataset with no `ceilings` key predates them -- nothing to + // hold, so this refresh sets the first table. + carryCeilings = Object.hasOwn(prior, 'ceilings') ? prior.ceilings : null; } const fileCounts = new Map(); @@ -1870,6 +2111,7 @@ function main() { fileCounts, carryFrom, carryProvisional, + carryCeilings, provenance: { measuredAt: new Date().toISOString().slice(0, 10), summaries: inputs.map((i) => path.basename(i.file)), @@ -1926,6 +2168,26 @@ function main() { 'read them as balancing inputs only.' ); } + // The ceilings (#16468), on the same stderr: how many a package now has, which + // packages have none and why, and -- the line a reviewer most needs -- any + // HELD ceiling a run of this very window already went over, which on `main` + // is the brake having fired, not a number to absorb. + const uncappedNames = Object.entries(dataset.uncapped); + const bySampleNow = collectSamples(perSummary).bySample; + const over = Object.entries(dataset.ceilings) + .filter(([name, ceiling]) => !dataset.provisional.includes(name) && Math.max(...(bySampleNow.get(name) ?? [0])) > ceiling) + .map(([name, ceiling]) => `${name} (${Math.max(...bySampleNow.get(name)).toFixed(2)}s > ${ceiling}s)`); + console.error( + `measure-test-shard-timings: ${Object.keys(dataset.ceilings).length} suite-duration ceiling(s)` + + `${carryCeilings === null ? ' -- the FIRST table: every one is its slowest executed run x ' + dataset.provenance.ceilingHeadroom : ', the prior ones held'}` + + `; ${uncappedNames.length} uncapped${uncappedNames.length ? ` (${uncappedNames.map(([n, why]) => `${n}: ${why}`).join(', ')})` : ''}.` + ); + if (over.length > 0) { + console.error( + `measure-test-shard-timings: ⚠ ${over.length} package(s) ran OVER their held ceiling in this window: ` + + `${over.join(', ')}. A refresh never raises a ceiling; that takes a ruling.` + ); + } if (dataset.skippedIncompleteSlices.length > 0) { console.error( `measure-test-shard-timings: ⚠ ${dataset.skippedIncompleteSlices.length} package(s) reported an ` + From fa972f19cef100ba3747c3df0a8ce474a85d9dcf Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 11:38:26 +0000 Subject: [PATCH 2/4] feat(ci): check-test-suite-ceilings grades each package's suite against its ceiling Reads the per-shard timing captures, sums a file-sliced package's parts across shards, and grades every executed package against the generated ceiling in scripts/test-shard-timings.json: over its ceiling, or absent from the dataset, is red, naming the package and its slowest test files; a provisional or carried package prints why it has no ceiling and is not red; a dataset without the `ceilings` field reads NOT MEASURED. A ceiling rises only by a RULED_CEILING_RAISES entry naming the ruling comment. Claude-Session: https://claude.ai/code/session_0115N1oNnQS5WqofZ2DzaT3q Co-authored-by: Claude --- scripts/check-test-suite-ceilings.mjs | 801 ++++++++++++++++++++++++++ 1 file changed, 801 insertions(+) create mode 100644 scripts/check-test-suite-ceilings.mjs diff --git a/scripts/check-test-suite-ceilings.mjs b/scripts/check-test-suite-ceilings.mjs new file mode 100644 index 00000000000..324d1b0abc7 --- /dev/null +++ b/scripts/check-test-suite-ceilings.mjs @@ -0,0 +1,801 @@ +#!/usr/bin/env node +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Per-package suite-duration ratchet (#16468): a run in which a package's test + * suite took longer than its measured ceiling is RED, naming the package and + * its slowest test files, and a ceiling rises only by a ruling. + * + * node scripts/check-test-suite-ceilings.mjs --captures [--label ] + * node scripts/check-test-suite-ceilings.mjs --self-test + * + * ## Why it exists + * + * `@objectstack/cli` measured 458 s on 2026-08-24, and the shard carrying it + * measured 28m46s two weeks later; nothing turned red on the pull requests that + * made it slower. CI time is the Actions bill and every seat's wait, and until + * this check nothing fired at the moment a package got slower. + * + * ## The ruling it implements (letter A, maintainer-approved) + * + * - The ceiling is the package's SLOWEST executed run in the refresh window + * x 1.25 -- not its median, which 31 of 71 packages' own window maximum + * already exceeded by more than 25%. It is GENERATED, never typed: the + * `ceilings` field of scripts/test-shard-timings.json, written by + * scripts/measure-test-shard-timings.mjs at every refresh (CEILING_HEADROOM + * and suiteCeilings() there), and held by every later refresh. + * - A package in the dataset's `provisional` list prints "no ceiling: + * provisional" and is not red. A package `uncapped` as `carried` (a cache HIT + * re-confirmed it, but no run of the window executed it) is not red either: + * it is in the dataset, and the next refresh that executes it gives it one. + * - ONLY a package absent from the dataset is red for having no ceiling -- the + * new-package shape. A package that has never landed is never measured by a + * refresh, so for a new package that red is lifted by a ruled raise below, + * not by waiting. + * - A ceiling rises only by a ruling: RULED_CEILING_RAISES, each entry naming + * the ruling's comment URL. Nothing else in this file moves a ceiling. + * + * ## No table yet is NOT MEASURED, never red + * + * The ceilings can only be generated where the run-summary artifacts can be + * downloaded -- the refresh workflow on a GitHub-hosted runner. An agent + * container's egress answers 403 to the artifact host (blob storage behind + * `GET /actions/artifacts/{id}/zip`), so no seat can seed the table, and a + * hand-typed one is exactly what the generator exists to prevent. Until the + * first refresh after this check landed writes `ceilings`, a dataset WITHOUT + * the key is read as NOT MEASURED ("no ceiling table yet") and exits 0. That + * is distinct from a package absent from a dataset that HAS the key (red), and + * the reading is printed as a warning annotation on every run, because a + * persistent NOT MEASURED is a defect: it means the refresh stopped writing + * ceilings. + * + * ## Where it runs, and why the CLI's slices are SUMMED + * + * In `Test Core` -- the aggregator job ci.yml runs after all six shards -- as + * the last step. It reads the per-shard captures `report-test-timings.mjs + * --capture` already writes (#16454): each capture carries every executed + * package's turbo execution window, read by `samplesFromSummary` -- the reader + * the drift step and the generator use, so there is no second reader of the + * turbo summary format here -- and every vitest file line with its duration, + * which is where "its slowest files" comes from. + * + * A file-sharded package (the CLI, cut into 3 slices on 3 shards whenever a + * run's own list makes it the serial floor) arrives as three partial windows + * on three shards. They are SUMMED across shards (mergeCaptures, which marks a + * set missing a part as incomplete) and the sum is graded against the whole + * package's ceiling, because that is the quantity the ceiling was measured in: + * the generator records a sliced package as the per-run SUM of its slices, and + * takes the slowest of those sums. Comparing each slice to a third of the + * ceiling was measured and rejected: vitest cuts the file list by a hash of + * each path, so the three slices carry 0.81 / 1.17 / 1.02 of an even third + * (PR #22456), and the heaviest slice against a third of `max x 1.25` keeps + * only ~7% headroom over the window maximum instead of 25% -- and a new or + * renamed file can move a slow file between slices without the suite getting + * one second slower. A set missing a part is NOT MEASURED for that package, + * never a part graded as a whole. + * + * ## The density the ceilings assume + * + * A package's window is CONTENDED wall clock: up to four suites share a 4-vCPU + * runner (`--concurrency=4`), so its window grows with how much its shard + * carries. Round 2 of the CLI split packed ~2641 s of predicted windows onto + * three shards (1.49x the density cap) and those shards' windows read 1.5-1.9x; + * on the one full run under it (37902633826), two packages read over this + * ratchet's ceilings. The ceilings are measured at the density the dataset's + * own split runs at, which the live split holds under its density cap + * (`densityCap()` in partition-test-shards.mjs). A future split that raises + * the density will read red here on packages it did not touch; that red is + * the split making suites slower, and the remedy is the split, not a raise. + * + * ## Exit codes + * + * 0 OK, or NOT MEASURED (no table yet; nothing executed; no captures) + * 1 a package over its ceiling, a package absent from the dataset, or a + * refusal (a malformed ceiling table or raise record) + */ + +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import process from 'node:process'; + +import { isEntrypoint } from './invoked-as.mjs'; +import { escapeWorkflowCommandMessage } from './partition-test-shards.mjs'; +import { mergeCaptures } from './report-test-timings.mjs'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const DATASET_PATH = path.join(REPO_ROOT, 'scripts', 'test-shard-timings.json'); +const DATASET_REL = 'scripts/test-shard-timings.json'; + +/** How many of a red package's slowest test files the verdict names. */ +export const SLOWEST_FILES_NAMED = 5; + +// ── The ruled raises ────────────────────────────────────────────────────── +// +// A ceiling rises ONLY here, and only on a ruling. One entry per package: +// +// '': { seconds: , ruling: '' } +// +// `ruling` must be the URL of the issue or pull-request comment that records +// the ruling (`.../issues/#issuecomment-` or `.../pull/#...`): the +// raise is reviewable only if the words that authorised it can be read. The +// effective ceiling is the larger of the generated one and the raise, so a raise +// below the generated ceiling changes nothing and is reported as stale. An entry +// for a package absent from the dataset gives a new package its first ceiling -- +// the only way a package that has never landed can get one, since a refresh only +// measures what runs on `main`. +// +// ⛔ Never raise a ceiling by editing `ceilings` in scripts/test-shard-timings.json: +// that file is generated, and the next refresh holds whatever it finds. +export const RULED_CEILING_RAISES = Object.freeze({}); + +const RULING_URL = /^https:\/\/github\.com\/objectstack-ai\/[A-Za-z0-9._-]+\/(?:issues|pull)\/\d+#issuecomment-\d+$/; + +/** Problems with a raise record; empty when every entry is well-formed. */ +export function raiseProblems(raises) { + const problems = []; + if (!raises || typeof raises !== 'object' || Array.isArray(raises)) { + return [`the ruled-raise record is ${JSON.stringify(raises)}, not a map of package -> { seconds, ruling }`]; + } + for (const [name, entry] of Object.entries(raises)) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + problems.push(`${name}: the raise is ${JSON.stringify(entry)}, not { seconds, ruling }`); + continue; + } + if (typeof entry.seconds !== 'number' || !Number.isFinite(entry.seconds) || !(entry.seconds > 0)) { + problems.push(`${name}: the raised ceiling is ${JSON.stringify(entry.seconds)}, not a positive number of seconds`); + } + if (typeof entry.ruling !== 'string' || !RULING_URL.test(entry.ruling)) { + problems.push( + `${name}: the raise names no ruling (${JSON.stringify(entry.ruling)}). A ceiling rises only by a ruling, ` + + 'so the entry carries the URL of the comment that records it.' + ); + } + } + return problems; +} + +// ── Reading the dataset's ceiling table ─────────────────────────────────── + +/** + * The parts of the dataset this check reads, or `{ table: null }` when the + * dataset predates the ceilings (no `ceilings` key) -- the NOT MEASURED + * bootstrap reading. A dataset that HAS the key but carries a malformed table + * is a refusal: it was generated, so a malformed one is a defect to fix, and + * grading against a guess would be a pass nobody earned. + */ +export function readCeilingTable(dataset) { + if (!dataset || typeof dataset !== 'object') { + throw new Error(`${DATASET_REL} is not a JSON object`); + } + const packages = dataset.packages; + if (!packages || typeof packages !== 'object' || Array.isArray(packages)) { + throw new Error(`${DATASET_REL}: expected {packages:{"":}}`); + } + if (!Object.hasOwn(dataset, 'ceilings')) return { table: null }; + const { ceilings, uncapped = {}, provisional = [] } = dataset; + if (!ceilings || typeof ceilings !== 'object' || Array.isArray(ceilings)) { + throw new Error(`${DATASET_REL}: \`ceilings\` is ${JSON.stringify(ceilings)}, not a map of package -> seconds`); + } + for (const [name, seconds] of Object.entries(ceilings)) { + if (typeof seconds !== 'number' || !Number.isFinite(seconds) || !(seconds > 0)) { + throw new Error(`${DATASET_REL}: the ceiling for ${name} is ${JSON.stringify(seconds)}, not a positive duration`); + } + } + if (!uncapped || typeof uncapped !== 'object' || Array.isArray(uncapped)) { + throw new Error(`${DATASET_REL}: \`uncapped\` is ${JSON.stringify(uncapped)}, not a map of package -> reason`); + } + if (!Array.isArray(provisional)) { + throw new Error(`${DATASET_REL}: \`provisional\` is ${JSON.stringify(provisional)}, not a list of package names`); + } + // Every package the dataset weighs is either capped or uncapped with a reason: + // the generator guarantees it, so a package in neither map is a table that + // was not written by it, and is refused rather than read as "no ceiling". + const uncovered = Object.keys(packages).filter((n) => !Object.hasOwn(ceilings, n) && !Object.hasOwn(uncapped, n)); + if (uncovered.length > 0) { + throw new Error( + `${DATASET_REL}: ${uncovered.length} package(s) carry a weight but neither a ceiling nor an \`uncapped\` ` + + `reason (${uncovered.join(', ')}). The generator writes one or the other for every package; regenerate ` + + 'the dataset rather than editing it.' + ); + } + return { + table: { + packages, + ceilings, + uncapped, + provisional: new Set(provisional), + headroom: dataset.provenance?.ceilingHeadroom ?? null, + measuredAt: dataset.provenance?.measuredAt ?? null, + }, + }; +} + +// ── Grading ─────────────────────────────────────────────────────────────── + +/** + * Grade one run. `merged` is mergeCaptures()' result: every executed package's + * window, a sliced package already summed across shards with `complete` saying + * whether every part turned up, and every file line. + * + * Returns `{ table, rows, problems }`; each row is one executed package with a + * `status`: + * + * over graded, over its effective ceiling RED + * absent not in the dataset, and no ruled raise RED + * ok graded, within its effective ceiling + * provisional in the dataset's `provisional` list no ceiling, not red + * uncapped `uncapped` with a reason (`carried`) no ceiling, not red + * partial a sliced package missing a part this run NOT MEASURED + * + * `problems` are refusals (a malformed raise record) and red the run on their own. + */ +export function gradeCeilings({ merged, dataset, raises = RULED_CEILING_RAISES }) { + const problems = raiseProblems(raises); + const { table } = readCeilingTable(dataset); + const rows = []; + if (table === null) return { table, rows, problems, staleRaises: [] }; + + const staleRaises = []; + for (const [name, raise] of Object.entries(problems.length ? {} : raises)) { + const generated = table.ceilings[name]; + if (generated !== undefined && raise.seconds <= generated) { + staleRaises.push(`${name} (raised to ${raise.seconds}s, generated ${generated}s)`); + } + } + + for (const pkg of merged.packages) { + const name = pkg.pkg; + const seconds = pkg.seconds; + const slices = pkg.sliceCount === null ? null : { count: pkg.sliceCount, parts: pkg.parts }; + const files = merged.files + .filter((f) => f.pkg === name) + .sort((a, b) => b.ms - a.ms) + .slice(0, SLOWEST_FILES_NAMED); + const row = { name, seconds, slices, files, ceiling: null, generated: null, raise: null }; + rows.push(row); + if (!pkg.complete) { + row.status = 'partial'; + continue; + } + const raise = problems.length === 0 && Object.hasOwn(raises, name) ? raises[name] : null; + const inDataset = Object.hasOwn(table.packages, name); + if (inDataset && table.provisional.has(name)) { + row.status = 'provisional'; + continue; + } + const generated = inDataset && Object.hasOwn(table.ceilings, name) ? table.ceilings[name] : null; + row.generated = generated; + row.raise = raise; + if (generated === null && raise === null) { + if (!inDataset) { + row.status = 'absent'; + } else { + row.status = 'uncapped'; + row.reason = table.uncapped[name]; + } + continue; + } + row.ceiling = Math.max(generated ?? 0, raise?.seconds ?? 0); + // Strict `>`: a run AT its ceiling is within it, like the drift bound. + row.status = seconds > row.ceiling ? 'over' : 'ok'; + } + rows.sort((a, b) => a.name.localeCompare(b.name, 'en')); + return { table, rows, problems, staleRaises }; +} + +// ── Rendering ───────────────────────────────────────────────────────────── + +const RED_STATUSES = new Set(['over', 'absent']); + +function sliceNote(row) { + if (!row.slices) return ''; + return row.slices.parts.length === row.slices.count + ? ` (all ${row.slices.count} slices, summed across shards)` + : ` (slices ${row.slices.parts.join('+') || 'none'} of ${row.slices.count})`; +} + +function filesText(row) { + if (row.files.length === 0) return ' (no vitest file line was captured for it)'; + return row.files + .map((f) => ` ${(f.ms / 1000).toFixed(2)}s ${f.file}${f.project ? ` [${f.project}]` : ''}`) + .join('\n'); +} + +function ceilingText(row) { + const parts = []; + if (row.generated !== null) parts.push(`generated ${row.generated.toFixed(2)}s`); + if (row.raise) parts.push(`ruled raise ${row.raise.seconds.toFixed(2)}s, ${row.raise.ruling}`); + return parts.join('; '); +} + +/** + * The verdict a runner receives, rendered without printing it so the + * self-test can read it: `out` is stdout (workflow-command annotations), + * `err` the human log. + */ +export function renderCeilingVerdict(report, label) { + const head = `suite-ceiling`; + if (report.problems.length > 0) { + return { + verdict: 'REFUSED', + exitCode: 1, + out: [`::error title=Suite-duration ceilings::${escapeWorkflowCommandMessage(`${label}: the ruled-raise record is malformed: ${report.problems.join('; ')}`)}`], + err: [ + `${head}: REFUSED -- ${label}: RULED_CEILING_RAISES in scripts/check-test-suite-ceilings.mjs is malformed:\n` + + report.problems.map((p) => ` - ${p}`).join('\n'), + ], + }; + } + + if (report.table === null) { + const text = + `NOT MEASURED -- ${label}: no ceiling table yet. ${DATASET_REL} carries no \`ceilings\`, because it was ` + + 'written before the suite-duration ceilings existed; the first run of .github/workflows/shard-timings-refresh.yml ' + + 'after this check landed writes them. Nothing was graded and nothing failed. If this still prints after that ' + + 'refresh has merged, the refresh is not writing ceilings: that is a defect, file it.'; + return { + verdict: 'NOT MEASURED', + exitCode: 0, + out: [`::warning title=Suite-duration ceilings::${escapeWorkflowCommandMessage(text)}`], + err: [`${head}: ${text}`], + }; + } + + const graded = report.rows.filter((r) => r.status === 'ok' || r.status === 'over'); + const red = report.rows.filter((r) => RED_STATUSES.has(r.status)); + const lines = report.rows.map((r) => { + switch (r.status) { + case 'ok': + case 'over': + return ( + ` ${r.status === 'over' ? 'OVER' : 'ok '} ${r.name}: ${r.seconds.toFixed(2)}s against ${r.ceiling.toFixed(2)}s ` + + `(${(r.seconds / r.ceiling).toFixed(2)}x)${sliceNote(r)}` + ); + case 'provisional': + return ` -- ${r.name}: ${r.seconds.toFixed(2)}s -- no ceiling: provisional`; + case 'uncapped': + return ` -- ${r.name}: ${r.seconds.toFixed(2)}s -- no ceiling: ${r.reason}`; + case 'partial': + return ` -- ${r.name}: NOT MEASURED -- a part of its suite is missing from this run${sliceNote(r)}; a part is never graded as the whole`; + case 'absent': + return ` NEW ${r.name}: ${r.seconds.toFixed(2)}s -- no ceiling: not in ${DATASET_REL} (a new package)`; + default: + throw new Error(`unknown row status ${r.status}`); + } + }); + const stale = report.staleRaises.length + ? `\n Ruled raises at or below the generated ceiling, so changing nothing (delete them): ${report.staleRaises.join(', ')}.` + : ''; + const basis = + `ceilings from ${DATASET_REL}${report.table.measuredAt ? ` (refreshed ${report.table.measuredAt})` : ''}: ` + + `each package's slowest executed run x ${report.table.headroom ?? '?'} when first set, held since`; + + if (red.length === 0) { + const verdict = graded.length === 0 ? 'NOT MEASURED' : 'OK'; + const summaryLine = + graded.length === 0 + ? `${head}: NOT MEASURED -- ${label}: no executed package carried a ceiling to grade against (${report.rows.length} executed).` + : `${head}: OK -- ${label}: ${graded.length} package(s) graded, every one within its ceiling.`; + return { + verdict, + exitCode: 0, + out: [], + err: [`${summaryLine}\n ${basis}.${lines.length ? `\n${lines.join('\n')}` : ''}${stale}`], + }; + } + + const detail = red + .map((r) => + r.status === 'over' + ? ` ${r.name}: ${r.seconds.toFixed(2)}s measured against its ${r.ceiling.toFixed(2)}s ceiling ` + + `(${(r.seconds / r.ceiling).toFixed(2)}x; ${ceilingText(r)})${sliceNote(r)}. Its slowest test files:\n${filesText(r)}` + : ` ${r.name}: ${r.seconds.toFixed(2)}s and no ceiling -- it is not in ${DATASET_REL}, so it is a new package. ` + + 'It stays red until a ruled raise gives it a first ceiling (a refresh only measures what runs on main). ' + + `Its slowest test files:\n${filesText(r)}` + ) + .join('\n'); + return { + verdict: 'OVER', + exitCode: 1, + out: red.map( + (r) => + `::error title=Suite-duration ceiling::${escapeWorkflowCommandMessage( + r.status === 'over' + ? `${r.name}: ${r.seconds.toFixed(2)}s, over its ${r.ceiling.toFixed(2)}s ceiling. Slowest file: ` + + `${r.files[0] ? `${r.files[0].file} ${(r.files[0].ms / 1000).toFixed(2)}s` : '(none captured)'}.` + : `${r.name}: ${r.seconds.toFixed(2)}s and no ceiling -- a package absent from ${DATASET_REL}.` + )}` + ), + err: [ + `${head}: OVER -- ${label}: ${red.length} package(s) over their suite-duration ceiling or without one.\n` + + `${detail}\n` + + ` ${basis}.\n` + + ' Make the suite faster, or take the raise to a ruling: a ceiling rises only by an entry in\n' + + ' RULED_CEILING_RAISES (scripts/check-test-suite-ceilings.mjs) naming the ruling comment. ⛔ Never\n' + + ` hand-edit \`ceilings\` in ${DATASET_REL} -- it is generated, and the next refresh holds what it finds.\n` + + ` Every executed package this run:\n${lines.join('\n')}${stale}`, + ], + }; +} + +// ── Reading the shard captures ──────────────────────────────────────────── + +/** Every capture JSON under `dir`, recursively; unreadable files become problems. */ +export function readCaptures(dir) { + const captures = []; + const problems = []; + const walk = (d) => { + let entries; + try { + entries = readdirSync(d); + } catch (err) { + problems.push(`capture directory unreadable at ${d}: ${err?.message ?? String(err)}`); + return; + } + for (const entry of entries.sort()) { + const p = path.join(d, entry); + let s; + try { + s = statSync(p); + } catch { + continue; + } + if (s.isDirectory()) { + walk(p); + continue; + } + if (!entry.endsWith('.json')) continue; + try { + captures.push(JSON.parse(readFileSync(p, 'utf8'))); + } catch (err) { + problems.push(`${entry}: ${err?.message ?? String(err)}`); + } + } + }; + walk(dir); + return { captures, problems }; +} + +// ── Self-test ───────────────────────────────────────────────────────────── + +// Set by `selfTest()` only after its verdict is printed, and read at the +// dispatch: a `return` above that line prints nothing and still exits 0 -- a +// self-test that never finished, reported as one that passed. +let selfTestReachedVerdict = false; + +// The roster: battery name -> minimum case count. A FLOOR, not an equality -- +// adding cases is ordinary work. A battery below its floor means cases stopped +// running; the remedy is to find what stopped registering, never to lower it. +const SELF_TEST_BATTERIES = Object.freeze({ + 'over the ceiling is red, naming the package and its slowest files': 6, + 'provisional prints "no ceiling: provisional" and is not red': 3, + 'a package absent from the dataset is red; an uncapped one is not': 4, + 'no ceiling table yet is NOT MEASURED, never red': 5, + 'file-sliced packages are summed across shards': 5, + 'a ruled raise lifts a ceiling, and only a well-formed one does': 7, + 'a malformed ceiling table is refused, never read as a pass': 4, +}); + +// Deleting an entry silences that battery's floor as effectively as zeroing +// it, so the roster's own size is pinned too. +const SELF_TEST_BATTERY_FLOOR = 7; + +const UNATTRIBUTED_BATTERY = '(no battery open)'; + +function selfTest() { + const batterySeen = new Map(); + let openBattery = null; + const battery = (name) => { + openBattery = name; + }; + const check = (fn) => { + const b = openBattery ?? UNATTRIBUTED_BATTERY; + batterySeen.set(b, (batterySeen.get(b) ?? 0) + 1); + fn(); + }; + const threw = (fn) => { + try { + fn(); + return false; + } catch { + return true; + } + }; + + // Fixtures in the shapes the live inputs have: a capture as + // `report-test-timings.mjs --capture` writes it, a dataset as the generator does. + const capture = (shard, packages, files = []) => ({ + schema: 'test-timing-capture/1', + shard, + files, + packages: packages.map(([pkg, seconds, slice = null]) => ({ + pkg, + seconds, + sliceCount: slice ? slice[1] : null, + parts: slice ? [slice[0]] : [], + })), + cached: [], + problems: [], + coverage: { declaredFiles: files.length, unattributed: 0, summariesRead: 1 }, + }); + const file = (pkg, name, ms) => ({ pkg, file: name, project: null, tests: 1, ms, state: 'pass', replayed: false }); + const dataset = (extra = {}) => ({ + packages: { a: 100, b: 50, prov: 10, carried: 20, cli: 1500 }, + provisional: ['prov'], + ceilings: { a: 150, b: 80, cli: 2000 }, + uncapped: { prov: 'provisional', carried: 'carried' }, + provenance: { measuredAt: '2026-10-12', ceilingHeadroom: 1.25 }, + ...extra, + }); + const run = (captures, ds = dataset(), raises = {}) => { + const report = gradeCeilings({ merged: mergeCaptures(captures), dataset: ds, raises }); + return { report, ...renderCeilingVerdict(report, 'Test Core') }; + }; + const status = (r, name) => r.report.rows.find((row) => row.name === name)?.status; + const raiseUrl = 'https://github.com/objectstack-ai/objectstack/issues/1#issuecomment-2'; + + battery('over the ceiling is red, naming the package and its slowest files'); + const over = run([ + capture('1/6', [['a', 151]], [file('a', 'fast.test.ts', 1_000), file('a', 'slow.test.ts', 90_000), file('a', 'mid.test.ts', 30_000)]), + ]); + check(() => { + if (over.exitCode !== 1 || over.verdict !== 'OVER') throw new Error(`over: exit ${over.exitCode}, verdict ${over.verdict}`); + }); + check(() => { + if (!over.err[0].includes('a: 151.00s measured against its 150.00s ceiling')) throw new Error(`over: the package is not named:\n${over.err[0]}`); + }); + check(() => { + // The slowest files, slowest first. + const text = over.err[0]; + const [i, j, k] = ['slow.test.ts', 'mid.test.ts', 'fast.test.ts'].map((n) => text.indexOf(n)); + if (!(i >= 0 && i < j && j < k)) throw new Error(`over: the slowest files are not named slowest first:\n${text}`); + }); + check(() => { + if (!over.out.some((l) => l.startsWith('::error title=Suite-duration ceiling::a: 151.00s'))) { + throw new Error(`over: no error annotation names the package: ${JSON.stringify(over.out)}`); + } + }); + check(() => { + // AT the ceiling is within it (strict `>`), and under is OK. + const at = run([capture('1/6', [['a', 150], ['b', 10]])]); + if (at.exitCode !== 0 || at.verdict !== 'OK' || status(at, 'a') !== 'ok') throw new Error(`over: a run AT its ceiling read ${at.verdict}`); + }); + check(() => { + // Only the files of the red package are named, at most SLOWEST_FILES_NAMED. + const many = Array.from({ length: 9 }, (_, i) => file('a', `f${i}.test.ts`, 1000 * (i + 1))); + const r = run([capture('1/6', [['a', 999], ['b', 1]], [...many, file('b', 'other.test.ts', 999_999)])]); + const row = r.report.rows.find((x) => x.name === 'a'); + if (row.files.length !== SLOWEST_FILES_NAMED || row.files[0].file !== 'f8.test.ts' || r.err[0].includes('other.test.ts ')) { + throw new Error(`over: the named files are wrong: ${JSON.stringify(row.files.map((f) => f.file))}`); + } + }); + + battery('provisional prints "no ceiling: provisional" and is not red'); + const prov = run([capture('1/6', [['prov', 9999], ['a', 10]])]); + check(() => { + if (prov.exitCode !== 0 || status(prov, 'prov') !== 'provisional') throw new Error(`provisional: exit ${prov.exitCode}, ${status(prov, 'prov')}`); + }); + check(() => { + if (!prov.err[0].includes('prov: 9999.00s -- no ceiling: provisional')) throw new Error(`provisional: the line is wrong:\n${prov.err[0]}`); + }); + check(() => { + // Provisional wins even over a ceiling the table still holds for it. + const held = run([capture('1/6', [['prov', 9999]])], dataset({ ceilings: { a: 150, b: 80, cli: 2000, prov: 12 }, uncapped: { carried: 'carried' } })); + if (held.exitCode !== 0 || status(held, 'prov') !== 'provisional') throw new Error(`provisional: a held ceiling overrode the provisional exemption (${held.verdict})`); + }); + + battery('a package absent from the dataset is red; an uncapped one is not'); + const fresh = run([capture('2/6', [['brand-new', 5], ['a', 10]], [file('brand-new', 'n.test.ts', 4_000)])]); + check(() => { + if (fresh.exitCode !== 1 || status(fresh, 'brand-new') !== 'absent') throw new Error(`absent: exit ${fresh.exitCode}, ${status(fresh, 'brand-new')}`); + }); + check(() => { + if (!fresh.err[0].includes('brand-new: 5.00s and no ceiling') || !fresh.err[0].includes('n.test.ts')) { + throw new Error(`absent: the package or its files are not named:\n${fresh.err[0]}`); + } + }); + check(() => { + const carried = run([capture('2/6', [['carried', 9999]])]); + if (carried.exitCode !== 0 || status(carried, 'carried') !== 'uncapped') throw new Error(`uncapped: a carried package read ${carried.verdict}`); + }); + check(() => { + const carried = run([capture('2/6', [['carried', 9999], ['a', 1]])]); + if (!carried.err[0].includes('carried: 9999.00s -- no ceiling: carried')) throw new Error(`uncapped: the reason is not named:\n${carried.err[0]}`); + }); + + battery('no ceiling table yet is NOT MEASURED, never red'); + const { ceilings: _c, uncapped: _u, ...preCeilings } = dataset(); + check(() => { + // Even a package absent from the dataset is not red before the table exists. + const none = run([capture('1/6', [['a', 9999], ['brand-new', 5]])], preCeilings); + if (none.exitCode !== 0 || none.verdict !== 'NOT MEASURED') throw new Error(`no table: exit ${none.exitCode}, verdict ${none.verdict}`); + }); + check(() => { + const none = run([capture('1/6', [['a', 9999]])], preCeilings); + if (!none.err[0].includes('no ceiling table yet')) throw new Error(`no table: the reason is not named:\n${none.err[0]}`); + }); + check(() => { + // Said as a warning annotation, so a persistent one is seen. + const none = run([capture('1/6', [['a', 9999]])], preCeilings); + if (!none.out.some((l) => l.startsWith('::warning title=Suite-duration ceilings::NOT MEASURED'))) { + throw new Error(`no table: no warning annotation: ${JSON.stringify(none.out)}`); + } + }); + check(() => { + // No capture at all, and only cached packages, grade nothing: NOT MEASURED. + const empty = run([]); + if (empty.exitCode !== 0 || empty.verdict !== 'NOT MEASURED') throw new Error(`no captures: verdict ${empty.verdict}`); + }); + check(() => { + const onlyProv = run([capture('1/6', [['prov', 1]])]); + if (onlyProv.exitCode !== 0 || onlyProv.verdict !== 'NOT MEASURED') throw new Error(`nothing graded: verdict ${onlyProv.verdict}`); + }); + + battery('file-sliced packages are summed across shards'); + check(() => { + // Three parts on three shards: summed, graded against the WHOLE ceiling. + const r = run([capture('2/6', [['cli', 600, [1, 3]]]), capture('3/6', [['cli', 700, [2, 3]]]), capture('4/6', [['cli', 650, [3, 3]]])]); + const row = r.report.rows.find((x) => x.name === 'cli'); + if (row.seconds !== 1950 || row.status !== 'ok' || r.exitCode !== 0) throw new Error(`slices: read ${row.seconds}s, ${row.status}`); + }); + check(() => { + // THE SKEW CASE: the heavy slice is past a third of the ceiling (700 > + // 2000/3 = 666.67), but the suite is within it -- not red. + const r = run([capture('2/6', [['cli', 560, [1, 3]]]), capture('3/6', [['cli', 700, [2, 3]]]), capture('4/6', [['cli', 600, [3, 3]]])]); + if (r.exitCode !== 0 || status(r, 'cli') !== 'ok') throw new Error(`slices: a skewed slice redded a suite within its ceiling (${r.verdict})`); + }); + check(() => { + // The sum over the whole ceiling is red, said as a summed set. + const r = run([capture('2/6', [['cli', 700, [1, 3]]]), capture('3/6', [['cli', 700, [2, 3]]]), capture('4/6', [['cli', 700, [3, 3]]])]); + if (r.exitCode !== 1 || !r.err[0].includes('cli: 2100.00s measured against its 2000.00s ceiling') || !r.err[0].includes('all 3 slices, summed across shards')) { + throw new Error(`slices: an over-ceiling sum was not red, or not said as a sum:\n${r.err[0]}`); + } + }); + check(() => { + // A part missing: NOT MEASURED for the package, never a part graded as the whole. + const r = run([capture('2/6', [['cli', 9999, [1, 3]]]), capture('3/6', [['cli', 9999, [2, 3]]])]); + if (r.exitCode !== 0 || status(r, 'cli') !== 'partial') throw new Error(`slices: an incomplete set read ${status(r, 'cli')} / ${r.verdict}`); + }); + check(() => { + const r = run([capture('2/6', [['cli', 10, [1, 3]]]), capture('3/6', [['a', 1]])]); + if (!r.err[0].includes('cli: NOT MEASURED') || !r.err[0].includes('slices 1 of 3')) throw new Error(`slices: the partial line is wrong:\n${r.err[0]}`); + }); + + battery('a ruled raise lifts a ceiling, and only a well-formed one does'); + check(() => { + // Over the generated 150, within the ruled 200: not red, and the ruling is named. + const r = run([capture('1/6', [['a', 180]])], dataset(), { a: { seconds: 200, ruling: raiseUrl } }); + const row = r.report.rows.find((x) => x.name === 'a'); + if (r.exitCode !== 0 || row.status !== 'ok' || row.ceiling !== 200) throw new Error(`raise: ${row.status} against ${row.ceiling}`); + }); + check(() => { + const r = run([capture('1/6', [['a', 201]])], dataset(), { a: { seconds: 200, ruling: raiseUrl } }); + if (r.exitCode !== 1 || !r.err[0].includes(raiseUrl)) throw new Error(`raise: over the raise was not red, or the ruling not named:\n${r.err[0]}`); + }); + check(() => { + // A new package's first ceiling, before any refresh could give it one. + const r = run([capture('1/6', [['brand-new', 5]])], dataset(), { 'brand-new': { seconds: 30, ruling: raiseUrl } }); + if (r.exitCode !== 0 || status(r, 'brand-new') !== 'ok') throw new Error(`raise: a ruled first ceiling did not lift the new-package red (${status(r, 'brand-new')})`); + }); + check(() => { + // A raise at or under the generated ceiling changes nothing, and says so. + const r = run([capture('1/6', [['a', 151]])], dataset(), { a: { seconds: 120, ruling: raiseUrl } }); + if (r.exitCode !== 1 || !r.err[0].includes('changing nothing')) throw new Error(`raise: a stale raise lowered or silently stayed:\n${r.err[0]}`); + }); + check(() => { + // No ruling, no raise: the record is refused and the run is red. + const r = run([capture('1/6', [['a', 180]])], dataset(), { a: { seconds: 200, ruling: 'the maintainer said so' } }); + if (r.exitCode !== 1 || r.verdict !== 'REFUSED') throw new Error(`raise: a raise naming no ruling was applied (${r.verdict})`); + }); + check(() => { + if (raiseProblems({ a: { seconds: -5, ruling: raiseUrl } }).length !== 1) throw new Error('raise: a negative raise was accepted'); + if (raiseProblems([]).length !== 1) throw new Error('raise: a list was read as the record'); + }); + check(() => { + // The live record is well-formed. + const live = raiseProblems(RULED_CEILING_RAISES); + if (live.length) throw new Error(`raise: RULED_CEILING_RAISES is malformed: ${live.join('; ')}`); + }); + + battery('a malformed ceiling table is refused, never read as a pass'); + check(() => { + if (!threw(() => run([capture('1/6', [['a', 1]])], dataset({ ceilings: [150] })))) throw new Error('table: a list was read as `ceilings`'); + }); + check(() => { + if (!threw(() => run([capture('1/6', [['a', 1]])], dataset({ ceilings: { a: 0, b: 80, cli: 2000 } })))) throw new Error('table: a zero ceiling was graded against'); + }); + check(() => { + // A weighed package in neither map is a table the generator did not write. + if (!threw(() => run([capture('1/6', [['a', 1]])], dataset({ uncapped: { prov: 'provisional' } })))) { + throw new Error('table: a package with neither a ceiling nor a reason was read as uncapped'); + } + }); + check(() => { + if (!threw(() => readCeilingTable({ ceilings: {} }))) throw new Error('table: a dataset with no `packages` was accepted'); + }); + + // The floor: every declared battery ran, and ran its cases. + const floorFailures = []; + const declared = Object.keys(SELF_TEST_BATTERIES); + if (declared.length < SELF_TEST_BATTERY_FLOOR) { + floorFailures.push(`SELF_TEST_BATTERIES declares ${declared.length} batteries, below the pinned ${SELF_TEST_BATTERY_FLOOR}`); + } + for (const [name, count] of batterySeen) { + if (!declared.includes(name)) floorFailures.push(`battery "${name}" registered ${count} case(s) but is not declared`); + } + for (const name of declared) { + const count = batterySeen.get(name) ?? 0; + if (count < SELF_TEST_BATTERIES[name]) { + floorFailures.push( + count === 0 + ? `battery "${name}" DID NOT RUN -- 0 cases registered, ${SELF_TEST_BATTERIES[name]} pinned` + : `battery "${name}" registered ${count} case(s), below its floor of ${SELF_TEST_BATTERIES[name]}` + ); + } + } + if (floorFailures.length > 0) { + throw new Error( + `check-test-suite-ceilings self-test floor (${floorFailures.length} breach(es)):\n` + + floorFailures.map((f) => ` - ${f}`).join('\n') + + '\n A battery below its floor means cases STOPPED RUNNING; find what stopped registering and restore it.' + ); + } + + const total = [...batterySeen.values()].reduce((a, b) => a + b, 0); + console.log(`check-test-suite-ceilings: self-test OK (${total} cases across ${declared.length} batteries)`); + selfTestReachedVerdict = true; +} + +// ── Entry point ─────────────────────────────────────────────────────────── + +function main() { + const argv = process.argv.slice(2); + if (argv.includes('--self-test')) { + selfTest(); + if (!selfTestReachedVerdict) { + console.error( + '\n✗ check-test-suite-ceilings self-test: selfTest() returned without reaching its verdict,\n' + + 'so no success line was printed. Exiting 0 here would report a self-test\n' + + 'that never finished as a self-test that passed.\n' + ); + process.exit(1); + } + return; + } + + let capturesDir = null; + let label = 'Test Core'; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--captures') capturesDir = argv[++i]; + else if (argv[i] === '--label') label = argv[++i]; + else throw new Error(`unrecognized argument: ${argv[i]}`); + } + if (!capturesDir) { + console.error('usage: check-test-suite-ceilings.mjs --captures [--label ] | --self-test'); + process.exit(1); + } + + const { captures, problems: readProblems } = readCaptures(capturesDir); + const merged = mergeCaptures(captures); + const dataset = JSON.parse(readFileSync(DATASET_PATH, 'utf8')); + const report = gradeCeilings({ merged, dataset }); + const rendered = renderCeilingVerdict(report, label); + for (const line of rendered.out) console.log(line); + for (const line of rendered.err) console.error(line); + // Not a verdict of their own: a shard capture that could not be read leaves + // its packages ungraded, which the counts above already show. Said, so an + // OK over five of six shards never reads as an OK over six. + const notes = [...readProblems, ...merged.problems]; + const shards = merged.shards.length; + console.error( + `suite-ceiling: read ${captures.length} shard capture(s) (${shards ? merged.shards.join(', ') : 'none'})` + + `${notes.length ? `; ${notes.length} could not be fully read, so their packages are ungraded: ${notes.join('; ')}` : ''}.` + ); + if (rendered.exitCode !== 0) process.exit(rendered.exitCode); +} + +// Exports bindings, so an import for those exports alone must run nothing. +if (isEntrypoint(import.meta.url)) { + main(); +} From 8ab0fb8f0aa07e3564489b334fc479b355d94909 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 11:39:52 +0000 Subject: [PATCH 3/4] ci: wire the suite-duration ceiling check into Test Core and the refresh PR body Test Core's aggregator runs check-test-suite-ceilings.mjs as its last step, with no continue-on-error, over the shard captures it already downloads; lint.yml runs the check's self-test. The refresh lane's PR body says whether it writes the first ceiling table or holds the existing one, and names the uncapped packages and any package that ran over its held ceiling in the window. Claude-Session: https://claude.ai/code/session_0115N1oNnQS5WqofZ2DzaT3q Co-authored-by: Claude --- .github/workflows/ci.yml | 37 ++++++++++++- .github/workflows/lint.yml | 10 ++++ .github/workflows/shard-timings-refresh.yml | 61 ++++++++++++++++++++- 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9ae0f44ad5..6993458becb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1391,7 +1391,10 @@ jobs: # redden it. Ordering after the verdict is what makes "cannot delay" # true, and the two flags are what make "cannot redden" true — a # diagnostics fault here (an artifact-service 403 is the measured one, - # #14469) must not evict a green run. + # #14469) must not evict a green run. The suite-duration ceiling check + # at the end of this job reads the same captures and is the one step + # here that DOES red the job, on purpose: it is a gate, not a report, and + # a capture that never arrived reads NOT MEASURED there, never red. # # ⛔ Steps, deliberately, NOT a new `needs:` entry: this job's # `OS_AGGREGATOR_MEMBERS` / `OS_AGGREGATOR_NON_MEMBERS` block is asserted @@ -1447,6 +1450,38 @@ jobs: if-no-files-found: ignore retention-days: 14 + # ── The per-package suite-duration ratchet (#16468) ────────────────── + # + # A GATE, unlike the two reporting steps above: NO `continue-on-error`, + # so a package whose suite ran past its ceiling reds `Test Core`, naming + # the package and its slowest test files. The ceiling is the package's + # slowest executed run in the refresh window x 1.25, generated into + # scripts/test-shard-timings.json (`ceilings`) by the refresh lane and + # held by every later refresh; it rises only by a ruled entry in the + # script's RULED_CEILING_RAISES. A `provisional` package prints "no + # ceiling: provisional" and is not red; a package absent from the dataset + # is red (the new-package shape). See the script header for the ruling. + # + # HERE, in the aggregator, rather than in each shard, because the CLI's + # file-level slices land on three shards: the three windows are SUMMED + # and the sum graded against the whole package's ceiling -- the quantity + # the ceiling was measured in. It reads the shard captures the download + # step above already fetched (each package's turbo execution window via + # samplesFromSummary, and every vitest file line); a capture that did not + # arrive leaves its packages ungraded and says so, never red. + # + # NOT MEASURED, exit 0, while the dataset carries no `ceilings`: they are + # generated only where the run-summary artifacts are downloadable (the + # refresh lane), so the first refresh after this lands writes them, and + # until then every run prints a warning annotation saying so. + # + # LAST, after the reporting steps, so the timing table is published + # whatever this decides. `always()`: it reads only what the shards already + # captured, so a red shard's packages are graded too. + - name: Check the per-package suite-duration ceilings + if: always() + run: node scripts/check-test-suite-ceilings.mjs --captures "$RUNNER_TEMP/test-timings" --label 'Test Core' + # ── Temporal conformance against live, non-UTC servers (ADR-0053 D-A3) ───── # diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index deb5d9ea8e8..e218efdd122 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4369,6 +4369,16 @@ jobs: - name: Test-timing parser self-test run: node scripts/report-test-timings.mjs --self-test + # The per-package suite-duration ratchet's own instrument (#16468). The + # gate runs in ci.yml's `Test Core` aggregator, where a clean tree with + # every suite under its ceiling reads OK whether or not the over-ceiling, + # absent-package and summed-slice rules still work -- so its self-test, + # which feeds those shapes in, runs here with teeth. Invoked as `node` for + # the GATE INVOCATION IDIOM reason at the top of this file. In-process + # fixtures, no IO, milliseconds. + - name: Suite-duration ceiling check self-test + run: node scripts/check-test-suite-ceilings.mjs --self-test + # Hand-written `.d.mts` mirrors (#10549). `scripts/js-comment-mask.mjs` # and `scripts/check-regen-pending.mjs` are untyped `.mjs` that each ship # a hand-written declaration beside them, and both files say "keep this in diff --git a/.github/workflows/shard-timings-refresh.yml b/.github/workflows/shard-timings-refresh.yml index 5e45a000268..527631fb8bf 100644 --- a/.github/workflows/shard-timings-refresh.yml +++ b/.github/workflows/shard-timings-refresh.yml @@ -33,6 +33,16 @@ # quoted verbatim into the PR body and a human decides. See "THE PINS" below. # ⛔ It never opens a PR when the regenerated file is byte-identical. A weekly # no-op PR trains everyone to ignore this PR. +# ⛔ It never moves a suite-duration ceiling (#16468). The dataset also +# carries `ceilings`, which `Test Core` grades every executed package +# against (scripts/check-test-suite-ceilings.mjs). The generator writes +# them in the same pass and HOLDS every ceiling the committed dataset +# already has; a package gets a new one only when it had none (its slowest +# executed run in this window x 1.25). A ceiling rises only by a ruling, +# recorded in that script, so a refresh whose window ran slower REPORTS +# the gap in the PR body instead of absorbing it. The first refresh after +# the check landed writes the first table; until it merges the check reads +# NOT MEASURED. # # HOW IT GETS ITS INPUTS, AND WHY IT RUNS HERE RATHER THAN IN AN AGENT WORKTREE # ---------------------------------------------------------------------------- @@ -634,6 +644,53 @@ jobs: } ') + # The suite-duration ceilings (#16468), read out of the dataset like the + # two lines above. The prior table is the COMMITTED dataset, read from + # git because the bins step has already copied the refreshed file over + # it. A refresh never moves a held ceiling, so the line counts what is + # new and what is uncapped, and the spread table below marks every + # held ceiling this window would have set differently -- and every one + # a run of this window already went over. + CEILING_LINE=$(node -e ' + const fs = require("fs"); + const { execFileSync } = require("child_process"); + const d = JSON.parse(fs.readFileSync(process.env.RUNNER_TEMP + "/refresh/refreshed.json", "utf8")); + if (!d.ceilings || typeof d.ceilings !== "object" || !d.uncapped || typeof d.uncapped !== "object") { + throw new Error("the refreshed dataset carries no `ceilings` / `uncapped`"); + } + const prior = JSON.parse(execFileSync("git", ["show", "HEAD:scripts/test-shard-timings.json"], { encoding: "utf8" })); + const held = prior.ceilings && typeof prior.ceilings === "object" ? prior.ceilings : null; + const names = Object.keys(d.ceilings); + const fresh = held === null ? names : names.filter((n) => !Object.hasOwn(held, n)); + const uncapped = Object.entries(d.uncapped).map(([n, why]) => n + " (" + why + ")"); + const h = d.provenance.ceilingHeadroom; + const over = (fs.readFileSync(process.env.RUNNER_TEMP + "/refresh/spread.md", "utf8").match(/OVER its held ceiling/g) || []).length; + const lines = []; + if (held === null) { + lines.push( + "⚠️ This refresh writes the FIRST suite-duration ceiling table: " + names.length + " package(s) get a " + + "ceiling, each its slowest executed run in this window x " + h + ". From the merge of this PR the `Test Core` " + + "check (scripts/check-test-suite-ceilings.mjs) grades every executed package against it -- until then it " + + "reads NOT MEASURED. Every later refresh HOLDS these numbers and a ceiling rises only by a ruling, so " + + "review the ceiling column below before merging." + ); + } else { + lines.push( + "Suite-duration ceilings: " + (names.length - fresh.length) + " held unchanged (a refresh never moves one; " + + "a raise is a ruling), " + fresh.length + " new" + (fresh.length ? " (" + fresh.join(", ") + ")" : "") + + ", each new one its slowest executed run in this window x " + h + "." + ); + } + lines.push(uncapped.length ? "No ceiling: " + uncapped.join(", ") + "." : "Every package carries a ceiling."); + if (over > 0) { + lines.push( + "⚠️ " + over + " package(s) ran OVER their held ceiling in this window (marked in the table): the brake " + + "fired on main. This refresh does not raise them; that takes a ruling." + ); + } + console.log(lines.join("\n\n")); + ') + SHARDS=$(node -e ' const rs = JSON.parse(require("fs").readFileSync(process.env.RUNNER_TEMP + "/candidates.json", "utf8")); const r = rs.find((x) => String(x.run_id) === process.env.RUN_ID); @@ -675,10 +732,12 @@ jobs: echo "because a weekly lane cannot know which cards a given run ought to retire. If this is" echo "the first refresh to land, retire those two by hand as part of merging it." echo - echo "## Sample depth and run-to-run spread" + echo "## Sample depth, run-to-run spread and suite-duration ceilings" echo echo "$DEPTH_LINE" echo + echo "$CEILING_LINE" + echo cat "$RUNNER_TEMP/refresh/spread.md" echo echo "## Measured per-shard suite time on the newest run in the set" From 68ca4e88d7d056520cfddfa19cb99e18d2fd8e42 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 12:01:18 +0000 Subject: [PATCH 4/4] fix(ci): the ceiling check folds the shard captures itself Importing report-test-timings.mjs from a gate family stops that family inheriting the module's declared populations in the dispatch-gates derivation (the module is a gate file only through its self-test), which the derivation's promotion invariant reds on. The fold is the same sum of slice parts with the same completeness mark, and a capture in an unknown schema is named and left out. Claude-Session: https://claude.ai/code/session_0115N1oNnQS5WqofZ2DzaT3q Co-authored-by: Claude --- scripts/check-test-suite-ceilings.mjs | 65 ++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/scripts/check-test-suite-ceilings.mjs b/scripts/check-test-suite-ceilings.mjs index 324d1b0abc7..1f01cb9c954 100644 --- a/scripts/check-test-suite-ceilings.mjs +++ b/scripts/check-test-suite-ceilings.mjs @@ -61,7 +61,7 @@ * * A file-sharded package (the CLI, cut into 3 slices on 3 shards whenever a * run's own list makes it the serial floor) arrives as three partial windows - * on three shards. They are SUMMED across shards (mergeCaptures, which marks a + * on three shards. They are SUMMED across shards (foldCaptures, which marks a * set missing a part as incomplete) and the sum is graded against the whole * package's ceiling, because that is the quantity the ceiling was measured in: * the generator records a sliced package as the per-run SUM of its slices, and @@ -101,7 +101,6 @@ import process from 'node:process'; import { isEntrypoint } from './invoked-as.mjs'; import { escapeWorkflowCommandMessage } from './partition-test-shards.mjs'; -import { mergeCaptures } from './report-test-timings.mjs'; const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const DATASET_PATH = path.join(REPO_ROOT, 'scripts', 'test-shard-timings.json'); @@ -214,7 +213,7 @@ export function readCeilingTable(dataset) { // ── Grading ─────────────────────────────────────────────────────────────── /** - * Grade one run. `merged` is mergeCaptures()' result: every executed package's + * Grade one run. `merged` is foldCaptures()' result: every executed package's * window, a sliced package already summed across shards with `complete` saying * whether every part turned up, and every file line. * @@ -421,6 +420,52 @@ export function renderCeilingVerdict(report, label) { // ── Reading the shard captures ──────────────────────────────────────────── +/** The capture schema `report-test-timings.mjs --capture` writes; any other is refused. */ +export const CAPTURE_SCHEMA = 'test-timing-capture/1'; + +/** + * Fold the per-shard captures into one run: every executed package's window, + * a file-sliced package SUMMED across the shards that ran its parts and marked + * `complete` only when every part turned up, and every file line. + * + * The same fold `report-test-timings.mjs` makes for its table, kept here rather + * than imported: that module is a gate file only through its own self-test, and + * a gate family importing it would silently stop inheriting its declared + * populations in the dispatch-gates derivation (its promotion invariant reds on + * exactly that). The windows themselves are not re-read here -- each capture's + * package rows are `samplesFromSummary`'s, taken on the shard. + */ +export function foldCaptures(captures) { + const files = []; + const byName = new Map(); + const problems = []; + const shards = []; + for (const capture of captures) { + if (capture?.schema !== CAPTURE_SCHEMA) { + problems.push(`a capture declares schema ${JSON.stringify(capture?.schema ?? null)}, not ${CAPTURE_SCHEMA}`); + continue; + } + if (capture.shard) shards.push(capture.shard); + files.push(...(capture.files ?? [])); + for (const p of capture.problems ?? []) problems.push(p); + for (const row of capture.packages ?? []) { + const acc = byName.get(row.pkg) ?? { pkg: row.pkg, seconds: 0, sliceCount: null, parts: [] }; + acc.seconds += row.seconds; + if (row.sliceCount) { + acc.sliceCount = row.sliceCount; + acc.parts.push(...(row.parts ?? [])); + } + byName.set(row.pkg, acc); + } + } + const packages = [...byName.values()].map((row) => { + const parts = [...new Set(row.parts)].sort((a, b) => a - b); + return { ...row, parts, complete: row.sliceCount === null || parts.length === row.sliceCount }; + }); + return { files, packages, problems, shards: shards.sort() }; +} + + /** Every capture JSON under `dir`, recursively; unreadable files become problems. */ export function readCaptures(dir) { const captures = []; @@ -472,7 +517,7 @@ const SELF_TEST_BATTERIES = Object.freeze({ 'provisional prints "no ceiling: provisional" and is not red': 3, 'a package absent from the dataset is red; an uncapped one is not': 4, 'no ceiling table yet is NOT MEASURED, never red': 5, - 'file-sliced packages are summed across shards': 5, + 'file-sliced packages are summed across shards': 6, 'a ruled raise lifts a ceiling, and only a well-formed one does': 7, 'a malformed ceiling table is refused, never read as a pass': 4, }); @@ -529,7 +574,7 @@ function selfTest() { ...extra, }); const run = (captures, ds = dataset(), raises = {}) => { - const report = gradeCeilings({ merged: mergeCaptures(captures), dataset: ds, raises }); + const report = gradeCeilings({ merged: foldCaptures(captures), dataset: ds, raises }); return { report, ...renderCeilingVerdict(report, 'Test Core') }; }; const status = (r, name) => r.report.rows.find((row) => row.name === name)?.status; @@ -661,6 +706,14 @@ function selfTest() { const r = run([capture('2/6', [['cli', 10, [1, 3]]]), capture('3/6', [['a', 1]])]); if (!r.err[0].includes('cli: NOT MEASURED') || !r.err[0].includes('slices 1 of 3')) throw new Error(`slices: the partial line is wrong:\n${r.err[0]}`); }); + check(() => { + // A capture in a shape this fold does not know contributes nothing and is + // named, never read as a set of windows. + const folded = foldCaptures([{ ...capture('2/6', [['cli', 600, [1, 3]]]), schema: 'test-timing-capture/2' }, capture('3/6', [['a', 1]])]); + if (folded.packages.some((p) => p.pkg === 'cli') || !folded.problems.some((p) => p.includes('test-timing-capture/2'))) { + throw new Error(`slices: an unknown capture schema was folded in: ${JSON.stringify(folded)}`); + } + }); battery('a ruled raise lifts a ceiling, and only a well-formed one does'); check(() => { @@ -777,7 +830,7 @@ function main() { } const { captures, problems: readProblems } = readCaptures(capturesDir); - const merged = mergeCaptures(captures); + const merged = foldCaptures(captures); const dataset = JSON.parse(readFileSync(DATASET_PATH, 'utf8')); const report = gradeCeilings({ merged, dataset }); const rendered = renderCeilingVerdict(report, label);