diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9ae0f44ad5..6993458becb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1391,7 +1391,10 @@ jobs: # redden it. Ordering after the verdict is what makes "cannot delay" # true, and the two flags are what make "cannot redden" true — a # diagnostics fault here (an artifact-service 403 is the measured one, - # #14469) must not evict a green run. + # #14469) must not evict a green run. The suite-duration ceiling check + # at the end of this job reads the same captures and is the one step + # here that DOES red the job, on purpose: it is a gate, not a report, and + # a capture that never arrived reads NOT MEASURED there, never red. # # ⛔ Steps, deliberately, NOT a new `needs:` entry: this job's # `OS_AGGREGATOR_MEMBERS` / `OS_AGGREGATOR_NON_MEMBERS` block is asserted @@ -1447,6 +1450,38 @@ jobs: if-no-files-found: ignore retention-days: 14 + # ── The per-package suite-duration ratchet (#16468) ────────────────── + # + # A GATE, unlike the two reporting steps above: NO `continue-on-error`, + # so a package whose suite ran past its ceiling reds `Test Core`, naming + # the package and its slowest test files. The ceiling is the package's + # slowest executed run in the refresh window x 1.25, generated into + # scripts/test-shard-timings.json (`ceilings`) by the refresh lane and + # held by every later refresh; it rises only by a ruled entry in the + # script's RULED_CEILING_RAISES. A `provisional` package prints "no + # ceiling: provisional" and is not red; a package absent from the dataset + # is red (the new-package shape). See the script header for the ruling. + # + # HERE, in the aggregator, rather than in each shard, because the CLI's + # file-level slices land on three shards: the three windows are SUMMED + # and the sum graded against the whole package's ceiling -- the quantity + # the ceiling was measured in. It reads the shard captures the download + # step above already fetched (each package's turbo execution window via + # samplesFromSummary, and every vitest file line); a capture that did not + # arrive leaves its packages ungraded and says so, never red. + # + # NOT MEASURED, exit 0, while the dataset carries no `ceilings`: they are + # generated only where the run-summary artifacts are downloadable (the + # refresh lane), so the first refresh after this lands writes them, and + # until then every run prints a warning annotation saying so. + # + # LAST, after the reporting steps, so the timing table is published + # whatever this decides. `always()`: it reads only what the shards already + # captured, so a red shard's packages are graded too. + - name: Check the per-package suite-duration ceilings + if: always() + run: node scripts/check-test-suite-ceilings.mjs --captures "$RUNNER_TEMP/test-timings" --label 'Test Core' + # ── Temporal conformance against live, non-UTC servers (ADR-0053 D-A3) ───── # diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index deb5d9ea8e8..e218efdd122 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4369,6 +4369,16 @@ jobs: - name: Test-timing parser self-test run: node scripts/report-test-timings.mjs --self-test + # The per-package suite-duration ratchet's own instrument (#16468). The + # gate runs in ci.yml's `Test Core` aggregator, where a clean tree with + # every suite under its ceiling reads OK whether or not the over-ceiling, + # absent-package and summed-slice rules still work -- so its self-test, + # which feeds those shapes in, runs here with teeth. Invoked as `node` for + # the GATE INVOCATION IDIOM reason at the top of this file. In-process + # fixtures, no IO, milliseconds. + - name: Suite-duration ceiling check self-test + run: node scripts/check-test-suite-ceilings.mjs --self-test + # Hand-written `.d.mts` mirrors (#10549). `scripts/js-comment-mask.mjs` # and `scripts/check-regen-pending.mjs` are untyped `.mjs` that each ship # a hand-written declaration beside them, and both files say "keep this in diff --git a/.github/workflows/shard-timings-refresh.yml b/.github/workflows/shard-timings-refresh.yml index 5e45a000268..527631fb8bf 100644 --- a/.github/workflows/shard-timings-refresh.yml +++ b/.github/workflows/shard-timings-refresh.yml @@ -33,6 +33,16 @@ # quoted verbatim into the PR body and a human decides. See "THE PINS" below. # ⛔ It never opens a PR when the regenerated file is byte-identical. A weekly # no-op PR trains everyone to ignore this PR. +# ⛔ It never moves a suite-duration ceiling (#16468). The dataset also +# carries `ceilings`, which `Test Core` grades every executed package +# against (scripts/check-test-suite-ceilings.mjs). The generator writes +# them in the same pass and HOLDS every ceiling the committed dataset +# already has; a package gets a new one only when it had none (its slowest +# executed run in this window x 1.25). A ceiling rises only by a ruling, +# recorded in that script, so a refresh whose window ran slower REPORTS +# the gap in the PR body instead of absorbing it. The first refresh after +# the check landed writes the first table; until it merges the check reads +# NOT MEASURED. # # HOW IT GETS ITS INPUTS, AND WHY IT RUNS HERE RATHER THAN IN AN AGENT WORKTREE # ---------------------------------------------------------------------------- @@ -634,6 +644,53 @@ jobs: } ') + # The suite-duration ceilings (#16468), read out of the dataset like the + # two lines above. The prior table is the COMMITTED dataset, read from + # git because the bins step has already copied the refreshed file over + # it. A refresh never moves a held ceiling, so the line counts what is + # new and what is uncapped, and the spread table below marks every + # held ceiling this window would have set differently -- and every one + # a run of this window already went over. + CEILING_LINE=$(node -e ' + const fs = require("fs"); + const { execFileSync } = require("child_process"); + const d = JSON.parse(fs.readFileSync(process.env.RUNNER_TEMP + "/refresh/refreshed.json", "utf8")); + if (!d.ceilings || typeof d.ceilings !== "object" || !d.uncapped || typeof d.uncapped !== "object") { + throw new Error("the refreshed dataset carries no `ceilings` / `uncapped`"); + } + const prior = JSON.parse(execFileSync("git", ["show", "HEAD:scripts/test-shard-timings.json"], { encoding: "utf8" })); + const held = prior.ceilings && typeof prior.ceilings === "object" ? prior.ceilings : null; + const names = Object.keys(d.ceilings); + const fresh = held === null ? names : names.filter((n) => !Object.hasOwn(held, n)); + const uncapped = Object.entries(d.uncapped).map(([n, why]) => n + " (" + why + ")"); + const h = d.provenance.ceilingHeadroom; + const over = (fs.readFileSync(process.env.RUNNER_TEMP + "/refresh/spread.md", "utf8").match(/OVER its held ceiling/g) || []).length; + const lines = []; + if (held === null) { + lines.push( + "⚠️ This refresh writes the FIRST suite-duration ceiling table: " + names.length + " package(s) get a " + + "ceiling, each its slowest executed run in this window x " + h + ". From the merge of this PR the `Test Core` " + + "check (scripts/check-test-suite-ceilings.mjs) grades every executed package against it -- until then it " + + "reads NOT MEASURED. Every later refresh HOLDS these numbers and a ceiling rises only by a ruling, so " + + "review the ceiling column below before merging." + ); + } else { + lines.push( + "Suite-duration ceilings: " + (names.length - fresh.length) + " held unchanged (a refresh never moves one; " + + "a raise is a ruling), " + fresh.length + " new" + (fresh.length ? " (" + fresh.join(", ") + ")" : "") + + ", each new one its slowest executed run in this window x " + h + "." + ); + } + lines.push(uncapped.length ? "No ceiling: " + uncapped.join(", ") + "." : "Every package carries a ceiling."); + if (over > 0) { + lines.push( + "⚠️ " + over + " package(s) ran OVER their held ceiling in this window (marked in the table): the brake " + + "fired on main. This refresh does not raise them; that takes a ruling." + ); + } + console.log(lines.join("\n\n")); + ') + SHARDS=$(node -e ' const rs = JSON.parse(require("fs").readFileSync(process.env.RUNNER_TEMP + "/candidates.json", "utf8")); const r = rs.find((x) => String(x.run_id) === process.env.RUN_ID); @@ -675,10 +732,12 @@ jobs: echo "because a weekly lane cannot know which cards a given run ought to retire. If this is" echo "the first refresh to land, retire those two by hand as part of merging it." echo - echo "## Sample depth and run-to-run spread" + echo "## Sample depth, run-to-run spread and suite-duration ceilings" echo echo "$DEPTH_LINE" echo + echo "$CEILING_LINE" + echo cat "$RUNNER_TEMP/refresh/spread.md" echo echo "## Measured per-shard suite time on the newest run in the set" diff --git a/scripts/check-test-suite-ceilings.mjs b/scripts/check-test-suite-ceilings.mjs new file mode 100644 index 00000000000..1f01cb9c954 --- /dev/null +++ b/scripts/check-test-suite-ceilings.mjs @@ -0,0 +1,854 @@ +#!/usr/bin/env node +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Per-package suite-duration ratchet (#16468): a run in which a package's test + * suite took longer than its measured ceiling is RED, naming the package and + * its slowest test files, and a ceiling rises only by a ruling. + * + * node scripts/check-test-suite-ceilings.mjs --captures [--label ] + * node scripts/check-test-suite-ceilings.mjs --self-test + * + * ## Why it exists + * + * `@objectstack/cli` measured 458 s on 2026-08-24, and the shard carrying it + * measured 28m46s two weeks later; nothing turned red on the pull requests that + * made it slower. CI time is the Actions bill and every seat's wait, and until + * this check nothing fired at the moment a package got slower. + * + * ## The ruling it implements (letter A, maintainer-approved) + * + * - The ceiling is the package's SLOWEST executed run in the refresh window + * x 1.25 -- not its median, which 31 of 71 packages' own window maximum + * already exceeded by more than 25%. It is GENERATED, never typed: the + * `ceilings` field of scripts/test-shard-timings.json, written by + * scripts/measure-test-shard-timings.mjs at every refresh (CEILING_HEADROOM + * and suiteCeilings() there), and held by every later refresh. + * - A package in the dataset's `provisional` list prints "no ceiling: + * provisional" and is not red. A package `uncapped` as `carried` (a cache HIT + * re-confirmed it, but no run of the window executed it) is not red either: + * it is in the dataset, and the next refresh that executes it gives it one. + * - ONLY a package absent from the dataset is red for having no ceiling -- the + * new-package shape. A package that has never landed is never measured by a + * refresh, so for a new package that red is lifted by a ruled raise below, + * not by waiting. + * - A ceiling rises only by a ruling: RULED_CEILING_RAISES, each entry naming + * the ruling's comment URL. Nothing else in this file moves a ceiling. + * + * ## No table yet is NOT MEASURED, never red + * + * The ceilings can only be generated where the run-summary artifacts can be + * downloaded -- the refresh workflow on a GitHub-hosted runner. An agent + * container's egress answers 403 to the artifact host (blob storage behind + * `GET /actions/artifacts/{id}/zip`), so no seat can seed the table, and a + * hand-typed one is exactly what the generator exists to prevent. Until the + * first refresh after this check landed writes `ceilings`, a dataset WITHOUT + * the key is read as NOT MEASURED ("no ceiling table yet") and exits 0. That + * is distinct from a package absent from a dataset that HAS the key (red), and + * the reading is printed as a warning annotation on every run, because a + * persistent NOT MEASURED is a defect: it means the refresh stopped writing + * ceilings. + * + * ## Where it runs, and why the CLI's slices are SUMMED + * + * In `Test Core` -- the aggregator job ci.yml runs after all six shards -- as + * the last step. It reads the per-shard captures `report-test-timings.mjs + * --capture` already writes (#16454): each capture carries every executed + * package's turbo execution window, read by `samplesFromSummary` -- the reader + * the drift step and the generator use, so there is no second reader of the + * turbo summary format here -- and every vitest file line with its duration, + * which is where "its slowest files" comes from. + * + * A file-sharded package (the CLI, cut into 3 slices on 3 shards whenever a + * run's own list makes it the serial floor) arrives as three partial windows + * on three shards. They are SUMMED across shards (foldCaptures, which marks a + * set missing a part as incomplete) and the sum is graded against the whole + * package's ceiling, because that is the quantity the ceiling was measured in: + * the generator records a sliced package as the per-run SUM of its slices, and + * takes the slowest of those sums. Comparing each slice to a third of the + * ceiling was measured and rejected: vitest cuts the file list by a hash of + * each path, so the three slices carry 0.81 / 1.17 / 1.02 of an even third + * (PR #22456), and the heaviest slice against a third of `max x 1.25` keeps + * only ~7% headroom over the window maximum instead of 25% -- and a new or + * renamed file can move a slow file between slices without the suite getting + * one second slower. A set missing a part is NOT MEASURED for that package, + * never a part graded as a whole. + * + * ## The density the ceilings assume + * + * A package's window is CONTENDED wall clock: up to four suites share a 4-vCPU + * runner (`--concurrency=4`), so its window grows with how much its shard + * carries. Round 2 of the CLI split packed ~2641 s of predicted windows onto + * three shards (1.49x the density cap) and those shards' windows read 1.5-1.9x; + * on the one full run under it (37902633826), two packages read over this + * ratchet's ceilings. The ceilings are measured at the density the dataset's + * own split runs at, which the live split holds under its density cap + * (`densityCap()` in partition-test-shards.mjs). A future split that raises + * the density will read red here on packages it did not touch; that red is + * the split making suites slower, and the remedy is the split, not a raise. + * + * ## Exit codes + * + * 0 OK, or NOT MEASURED (no table yet; nothing executed; no captures) + * 1 a package over its ceiling, a package absent from the dataset, or a + * refusal (a malformed ceiling table or raise record) + */ + +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import process from 'node:process'; + +import { isEntrypoint } from './invoked-as.mjs'; +import { escapeWorkflowCommandMessage } from './partition-test-shards.mjs'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const DATASET_PATH = path.join(REPO_ROOT, 'scripts', 'test-shard-timings.json'); +const DATASET_REL = 'scripts/test-shard-timings.json'; + +/** How many of a red package's slowest test files the verdict names. */ +export const SLOWEST_FILES_NAMED = 5; + +// ── The ruled raises ────────────────────────────────────────────────────── +// +// A ceiling rises ONLY here, and only on a ruling. One entry per package: +// +// '': { seconds: , ruling: '' } +// +// `ruling` must be the URL of the issue or pull-request comment that records +// the ruling (`.../issues/#issuecomment-` or `.../pull/#...`): the +// raise is reviewable only if the words that authorised it can be read. The +// effective ceiling is the larger of the generated one and the raise, so a raise +// below the generated ceiling changes nothing and is reported as stale. An entry +// for a package absent from the dataset gives a new package its first ceiling -- +// the only way a package that has never landed can get one, since a refresh only +// measures what runs on `main`. +// +// ⛔ Never raise a ceiling by editing `ceilings` in scripts/test-shard-timings.json: +// that file is generated, and the next refresh holds whatever it finds. +export const RULED_CEILING_RAISES = Object.freeze({}); + +const RULING_URL = /^https:\/\/github\.com\/objectstack-ai\/[A-Za-z0-9._-]+\/(?:issues|pull)\/\d+#issuecomment-\d+$/; + +/** Problems with a raise record; empty when every entry is well-formed. */ +export function raiseProblems(raises) { + const problems = []; + if (!raises || typeof raises !== 'object' || Array.isArray(raises)) { + return [`the ruled-raise record is ${JSON.stringify(raises)}, not a map of package -> { seconds, ruling }`]; + } + for (const [name, entry] of Object.entries(raises)) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + problems.push(`${name}: the raise is ${JSON.stringify(entry)}, not { seconds, ruling }`); + continue; + } + if (typeof entry.seconds !== 'number' || !Number.isFinite(entry.seconds) || !(entry.seconds > 0)) { + problems.push(`${name}: the raised ceiling is ${JSON.stringify(entry.seconds)}, not a positive number of seconds`); + } + if (typeof entry.ruling !== 'string' || !RULING_URL.test(entry.ruling)) { + problems.push( + `${name}: the raise names no ruling (${JSON.stringify(entry.ruling)}). A ceiling rises only by a ruling, ` + + 'so the entry carries the URL of the comment that records it.' + ); + } + } + return problems; +} + +// ── Reading the dataset's ceiling table ─────────────────────────────────── + +/** + * The parts of the dataset this check reads, or `{ table: null }` when the + * dataset predates the ceilings (no `ceilings` key) -- the NOT MEASURED + * bootstrap reading. A dataset that HAS the key but carries a malformed table + * is a refusal: it was generated, so a malformed one is a defect to fix, and + * grading against a guess would be a pass nobody earned. + */ +export function readCeilingTable(dataset) { + if (!dataset || typeof dataset !== 'object') { + throw new Error(`${DATASET_REL} is not a JSON object`); + } + const packages = dataset.packages; + if (!packages || typeof packages !== 'object' || Array.isArray(packages)) { + throw new Error(`${DATASET_REL}: expected {packages:{"":}}`); + } + if (!Object.hasOwn(dataset, 'ceilings')) return { table: null }; + const { ceilings, uncapped = {}, provisional = [] } = dataset; + if (!ceilings || typeof ceilings !== 'object' || Array.isArray(ceilings)) { + throw new Error(`${DATASET_REL}: \`ceilings\` is ${JSON.stringify(ceilings)}, not a map of package -> seconds`); + } + for (const [name, seconds] of Object.entries(ceilings)) { + if (typeof seconds !== 'number' || !Number.isFinite(seconds) || !(seconds > 0)) { + throw new Error(`${DATASET_REL}: the ceiling for ${name} is ${JSON.stringify(seconds)}, not a positive duration`); + } + } + if (!uncapped || typeof uncapped !== 'object' || Array.isArray(uncapped)) { + throw new Error(`${DATASET_REL}: \`uncapped\` is ${JSON.stringify(uncapped)}, not a map of package -> reason`); + } + if (!Array.isArray(provisional)) { + throw new Error(`${DATASET_REL}: \`provisional\` is ${JSON.stringify(provisional)}, not a list of package names`); + } + // Every package the dataset weighs is either capped or uncapped with a reason: + // the generator guarantees it, so a package in neither map is a table that + // was not written by it, and is refused rather than read as "no ceiling". + const uncovered = Object.keys(packages).filter((n) => !Object.hasOwn(ceilings, n) && !Object.hasOwn(uncapped, n)); + if (uncovered.length > 0) { + throw new Error( + `${DATASET_REL}: ${uncovered.length} package(s) carry a weight but neither a ceiling nor an \`uncapped\` ` + + `reason (${uncovered.join(', ')}). The generator writes one or the other for every package; regenerate ` + + 'the dataset rather than editing it.' + ); + } + return { + table: { + packages, + ceilings, + uncapped, + provisional: new Set(provisional), + headroom: dataset.provenance?.ceilingHeadroom ?? null, + measuredAt: dataset.provenance?.measuredAt ?? null, + }, + }; +} + +// ── Grading ─────────────────────────────────────────────────────────────── + +/** + * Grade one run. `merged` is foldCaptures()' result: every executed package's + * window, a sliced package already summed across shards with `complete` saying + * whether every part turned up, and every file line. + * + * Returns `{ table, rows, problems }`; each row is one executed package with a + * `status`: + * + * over graded, over its effective ceiling RED + * absent not in the dataset, and no ruled raise RED + * ok graded, within its effective ceiling + * provisional in the dataset's `provisional` list no ceiling, not red + * uncapped `uncapped` with a reason (`carried`) no ceiling, not red + * partial a sliced package missing a part this run NOT MEASURED + * + * `problems` are refusals (a malformed raise record) and red the run on their own. + */ +export function gradeCeilings({ merged, dataset, raises = RULED_CEILING_RAISES }) { + const problems = raiseProblems(raises); + const { table } = readCeilingTable(dataset); + const rows = []; + if (table === null) return { table, rows, problems, staleRaises: [] }; + + const staleRaises = []; + for (const [name, raise] of Object.entries(problems.length ? {} : raises)) { + const generated = table.ceilings[name]; + if (generated !== undefined && raise.seconds <= generated) { + staleRaises.push(`${name} (raised to ${raise.seconds}s, generated ${generated}s)`); + } + } + + for (const pkg of merged.packages) { + const name = pkg.pkg; + const seconds = pkg.seconds; + const slices = pkg.sliceCount === null ? null : { count: pkg.sliceCount, parts: pkg.parts }; + const files = merged.files + .filter((f) => f.pkg === name) + .sort((a, b) => b.ms - a.ms) + .slice(0, SLOWEST_FILES_NAMED); + const row = { name, seconds, slices, files, ceiling: null, generated: null, raise: null }; + rows.push(row); + if (!pkg.complete) { + row.status = 'partial'; + continue; + } + const raise = problems.length === 0 && Object.hasOwn(raises, name) ? raises[name] : null; + const inDataset = Object.hasOwn(table.packages, name); + if (inDataset && table.provisional.has(name)) { + row.status = 'provisional'; + continue; + } + const generated = inDataset && Object.hasOwn(table.ceilings, name) ? table.ceilings[name] : null; + row.generated = generated; + row.raise = raise; + if (generated === null && raise === null) { + if (!inDataset) { + row.status = 'absent'; + } else { + row.status = 'uncapped'; + row.reason = table.uncapped[name]; + } + continue; + } + row.ceiling = Math.max(generated ?? 0, raise?.seconds ?? 0); + // Strict `>`: a run AT its ceiling is within it, like the drift bound. + row.status = seconds > row.ceiling ? 'over' : 'ok'; + } + rows.sort((a, b) => a.name.localeCompare(b.name, 'en')); + return { table, rows, problems, staleRaises }; +} + +// ── Rendering ───────────────────────────────────────────────────────────── + +const RED_STATUSES = new Set(['over', 'absent']); + +function sliceNote(row) { + if (!row.slices) return ''; + return row.slices.parts.length === row.slices.count + ? ` (all ${row.slices.count} slices, summed across shards)` + : ` (slices ${row.slices.parts.join('+') || 'none'} of ${row.slices.count})`; +} + +function filesText(row) { + if (row.files.length === 0) return ' (no vitest file line was captured for it)'; + return row.files + .map((f) => ` ${(f.ms / 1000).toFixed(2)}s ${f.file}${f.project ? ` [${f.project}]` : ''}`) + .join('\n'); +} + +function ceilingText(row) { + const parts = []; + if (row.generated !== null) parts.push(`generated ${row.generated.toFixed(2)}s`); + if (row.raise) parts.push(`ruled raise ${row.raise.seconds.toFixed(2)}s, ${row.raise.ruling}`); + return parts.join('; '); +} + +/** + * The verdict a runner receives, rendered without printing it so the + * self-test can read it: `out` is stdout (workflow-command annotations), + * `err` the human log. + */ +export function renderCeilingVerdict(report, label) { + const head = `suite-ceiling`; + if (report.problems.length > 0) { + return { + verdict: 'REFUSED', + exitCode: 1, + out: [`::error title=Suite-duration ceilings::${escapeWorkflowCommandMessage(`${label}: the ruled-raise record is malformed: ${report.problems.join('; ')}`)}`], + err: [ + `${head}: REFUSED -- ${label}: RULED_CEILING_RAISES in scripts/check-test-suite-ceilings.mjs is malformed:\n` + + report.problems.map((p) => ` - ${p}`).join('\n'), + ], + }; + } + + if (report.table === null) { + const text = + `NOT MEASURED -- ${label}: no ceiling table yet. ${DATASET_REL} carries no \`ceilings\`, because it was ` + + 'written before the suite-duration ceilings existed; the first run of .github/workflows/shard-timings-refresh.yml ' + + 'after this check landed writes them. Nothing was graded and nothing failed. If this still prints after that ' + + 'refresh has merged, the refresh is not writing ceilings: that is a defect, file it.'; + return { + verdict: 'NOT MEASURED', + exitCode: 0, + out: [`::warning title=Suite-duration ceilings::${escapeWorkflowCommandMessage(text)}`], + err: [`${head}: ${text}`], + }; + } + + const graded = report.rows.filter((r) => r.status === 'ok' || r.status === 'over'); + const red = report.rows.filter((r) => RED_STATUSES.has(r.status)); + const lines = report.rows.map((r) => { + switch (r.status) { + case 'ok': + case 'over': + return ( + ` ${r.status === 'over' ? 'OVER' : 'ok '} ${r.name}: ${r.seconds.toFixed(2)}s against ${r.ceiling.toFixed(2)}s ` + + `(${(r.seconds / r.ceiling).toFixed(2)}x)${sliceNote(r)}` + ); + case 'provisional': + return ` -- ${r.name}: ${r.seconds.toFixed(2)}s -- no ceiling: provisional`; + case 'uncapped': + return ` -- ${r.name}: ${r.seconds.toFixed(2)}s -- no ceiling: ${r.reason}`; + case 'partial': + return ` -- ${r.name}: NOT MEASURED -- a part of its suite is missing from this run${sliceNote(r)}; a part is never graded as the whole`; + case 'absent': + return ` NEW ${r.name}: ${r.seconds.toFixed(2)}s -- no ceiling: not in ${DATASET_REL} (a new package)`; + default: + throw new Error(`unknown row status ${r.status}`); + } + }); + const stale = report.staleRaises.length + ? `\n Ruled raises at or below the generated ceiling, so changing nothing (delete them): ${report.staleRaises.join(', ')}.` + : ''; + const basis = + `ceilings from ${DATASET_REL}${report.table.measuredAt ? ` (refreshed ${report.table.measuredAt})` : ''}: ` + + `each package's slowest executed run x ${report.table.headroom ?? '?'} when first set, held since`; + + if (red.length === 0) { + const verdict = graded.length === 0 ? 'NOT MEASURED' : 'OK'; + const summaryLine = + graded.length === 0 + ? `${head}: NOT MEASURED -- ${label}: no executed package carried a ceiling to grade against (${report.rows.length} executed).` + : `${head}: OK -- ${label}: ${graded.length} package(s) graded, every one within its ceiling.`; + return { + verdict, + exitCode: 0, + out: [], + err: [`${summaryLine}\n ${basis}.${lines.length ? `\n${lines.join('\n')}` : ''}${stale}`], + }; + } + + const detail = red + .map((r) => + r.status === 'over' + ? ` ${r.name}: ${r.seconds.toFixed(2)}s measured against its ${r.ceiling.toFixed(2)}s ceiling ` + + `(${(r.seconds / r.ceiling).toFixed(2)}x; ${ceilingText(r)})${sliceNote(r)}. Its slowest test files:\n${filesText(r)}` + : ` ${r.name}: ${r.seconds.toFixed(2)}s and no ceiling -- it is not in ${DATASET_REL}, so it is a new package. ` + + 'It stays red until a ruled raise gives it a first ceiling (a refresh only measures what runs on main). ' + + `Its slowest test files:\n${filesText(r)}` + ) + .join('\n'); + return { + verdict: 'OVER', + exitCode: 1, + out: red.map( + (r) => + `::error title=Suite-duration ceiling::${escapeWorkflowCommandMessage( + r.status === 'over' + ? `${r.name}: ${r.seconds.toFixed(2)}s, over its ${r.ceiling.toFixed(2)}s ceiling. Slowest file: ` + + `${r.files[0] ? `${r.files[0].file} ${(r.files[0].ms / 1000).toFixed(2)}s` : '(none captured)'}.` + : `${r.name}: ${r.seconds.toFixed(2)}s and no ceiling -- a package absent from ${DATASET_REL}.` + )}` + ), + err: [ + `${head}: OVER -- ${label}: ${red.length} package(s) over their suite-duration ceiling or without one.\n` + + `${detail}\n` + + ` ${basis}.\n` + + ' Make the suite faster, or take the raise to a ruling: a ceiling rises only by an entry in\n' + + ' RULED_CEILING_RAISES (scripts/check-test-suite-ceilings.mjs) naming the ruling comment. ⛔ Never\n' + + ` hand-edit \`ceilings\` in ${DATASET_REL} -- it is generated, and the next refresh holds what it finds.\n` + + ` Every executed package this run:\n${lines.join('\n')}${stale}`, + ], + }; +} + +// ── Reading the shard captures ──────────────────────────────────────────── + +/** The capture schema `report-test-timings.mjs --capture` writes; any other is refused. */ +export const CAPTURE_SCHEMA = 'test-timing-capture/1'; + +/** + * Fold the per-shard captures into one run: every executed package's window, + * a file-sliced package SUMMED across the shards that ran its parts and marked + * `complete` only when every part turned up, and every file line. + * + * The same fold `report-test-timings.mjs` makes for its table, kept here rather + * than imported: that module is a gate file only through its own self-test, and + * a gate family importing it would silently stop inheriting its declared + * populations in the dispatch-gates derivation (its promotion invariant reds on + * exactly that). The windows themselves are not re-read here -- each capture's + * package rows are `samplesFromSummary`'s, taken on the shard. + */ +export function foldCaptures(captures) { + const files = []; + const byName = new Map(); + const problems = []; + const shards = []; + for (const capture of captures) { + if (capture?.schema !== CAPTURE_SCHEMA) { + problems.push(`a capture declares schema ${JSON.stringify(capture?.schema ?? null)}, not ${CAPTURE_SCHEMA}`); + continue; + } + if (capture.shard) shards.push(capture.shard); + files.push(...(capture.files ?? [])); + for (const p of capture.problems ?? []) problems.push(p); + for (const row of capture.packages ?? []) { + const acc = byName.get(row.pkg) ?? { pkg: row.pkg, seconds: 0, sliceCount: null, parts: [] }; + acc.seconds += row.seconds; + if (row.sliceCount) { + acc.sliceCount = row.sliceCount; + acc.parts.push(...(row.parts ?? [])); + } + byName.set(row.pkg, acc); + } + } + const packages = [...byName.values()].map((row) => { + const parts = [...new Set(row.parts)].sort((a, b) => a - b); + return { ...row, parts, complete: row.sliceCount === null || parts.length === row.sliceCount }; + }); + return { files, packages, problems, shards: shards.sort() }; +} + + +/** Every capture JSON under `dir`, recursively; unreadable files become problems. */ +export function readCaptures(dir) { + const captures = []; + const problems = []; + const walk = (d) => { + let entries; + try { + entries = readdirSync(d); + } catch (err) { + problems.push(`capture directory unreadable at ${d}: ${err?.message ?? String(err)}`); + return; + } + for (const entry of entries.sort()) { + const p = path.join(d, entry); + let s; + try { + s = statSync(p); + } catch { + continue; + } + if (s.isDirectory()) { + walk(p); + continue; + } + if (!entry.endsWith('.json')) continue; + try { + captures.push(JSON.parse(readFileSync(p, 'utf8'))); + } catch (err) { + problems.push(`${entry}: ${err?.message ?? String(err)}`); + } + } + }; + walk(dir); + return { captures, problems }; +} + +// ── Self-test ───────────────────────────────────────────────────────────── + +// Set by `selfTest()` only after its verdict is printed, and read at the +// dispatch: a `return` above that line prints nothing and still exits 0 -- a +// self-test that never finished, reported as one that passed. +let selfTestReachedVerdict = false; + +// The roster: battery name -> minimum case count. A FLOOR, not an equality -- +// adding cases is ordinary work. A battery below its floor means cases stopped +// running; the remedy is to find what stopped registering, never to lower it. +const SELF_TEST_BATTERIES = Object.freeze({ + 'over the ceiling is red, naming the package and its slowest files': 6, + 'provisional prints "no ceiling: provisional" and is not red': 3, + 'a package absent from the dataset is red; an uncapped one is not': 4, + 'no ceiling table yet is NOT MEASURED, never red': 5, + 'file-sliced packages are summed across shards': 6, + 'a ruled raise lifts a ceiling, and only a well-formed one does': 7, + 'a malformed ceiling table is refused, never read as a pass': 4, +}); + +// Deleting an entry silences that battery's floor as effectively as zeroing +// it, so the roster's own size is pinned too. +const SELF_TEST_BATTERY_FLOOR = 7; + +const UNATTRIBUTED_BATTERY = '(no battery open)'; + +function selfTest() { + const batterySeen = new Map(); + let openBattery = null; + const battery = (name) => { + openBattery = name; + }; + const check = (fn) => { + const b = openBattery ?? UNATTRIBUTED_BATTERY; + batterySeen.set(b, (batterySeen.get(b) ?? 0) + 1); + fn(); + }; + const threw = (fn) => { + try { + fn(); + return false; + } catch { + return true; + } + }; + + // Fixtures in the shapes the live inputs have: a capture as + // `report-test-timings.mjs --capture` writes it, a dataset as the generator does. + const capture = (shard, packages, files = []) => ({ + schema: 'test-timing-capture/1', + shard, + files, + packages: packages.map(([pkg, seconds, slice = null]) => ({ + pkg, + seconds, + sliceCount: slice ? slice[1] : null, + parts: slice ? [slice[0]] : [], + })), + cached: [], + problems: [], + coverage: { declaredFiles: files.length, unattributed: 0, summariesRead: 1 }, + }); + const file = (pkg, name, ms) => ({ pkg, file: name, project: null, tests: 1, ms, state: 'pass', replayed: false }); + const dataset = (extra = {}) => ({ + packages: { a: 100, b: 50, prov: 10, carried: 20, cli: 1500 }, + provisional: ['prov'], + ceilings: { a: 150, b: 80, cli: 2000 }, + uncapped: { prov: 'provisional', carried: 'carried' }, + provenance: { measuredAt: '2026-10-12', ceilingHeadroom: 1.25 }, + ...extra, + }); + const run = (captures, ds = dataset(), raises = {}) => { + const report = gradeCeilings({ merged: foldCaptures(captures), dataset: ds, raises }); + return { report, ...renderCeilingVerdict(report, 'Test Core') }; + }; + const status = (r, name) => r.report.rows.find((row) => row.name === name)?.status; + const raiseUrl = 'https://github.com/objectstack-ai/objectstack/issues/1#issuecomment-2'; + + battery('over the ceiling is red, naming the package and its slowest files'); + const over = run([ + capture('1/6', [['a', 151]], [file('a', 'fast.test.ts', 1_000), file('a', 'slow.test.ts', 90_000), file('a', 'mid.test.ts', 30_000)]), + ]); + check(() => { + if (over.exitCode !== 1 || over.verdict !== 'OVER') throw new Error(`over: exit ${over.exitCode}, verdict ${over.verdict}`); + }); + check(() => { + if (!over.err[0].includes('a: 151.00s measured against its 150.00s ceiling')) throw new Error(`over: the package is not named:\n${over.err[0]}`); + }); + check(() => { + // The slowest files, slowest first. + const text = over.err[0]; + const [i, j, k] = ['slow.test.ts', 'mid.test.ts', 'fast.test.ts'].map((n) => text.indexOf(n)); + if (!(i >= 0 && i < j && j < k)) throw new Error(`over: the slowest files are not named slowest first:\n${text}`); + }); + check(() => { + if (!over.out.some((l) => l.startsWith('::error title=Suite-duration ceiling::a: 151.00s'))) { + throw new Error(`over: no error annotation names the package: ${JSON.stringify(over.out)}`); + } + }); + check(() => { + // AT the ceiling is within it (strict `>`), and under is OK. + const at = run([capture('1/6', [['a', 150], ['b', 10]])]); + if (at.exitCode !== 0 || at.verdict !== 'OK' || status(at, 'a') !== 'ok') throw new Error(`over: a run AT its ceiling read ${at.verdict}`); + }); + check(() => { + // Only the files of the red package are named, at most SLOWEST_FILES_NAMED. + const many = Array.from({ length: 9 }, (_, i) => file('a', `f${i}.test.ts`, 1000 * (i + 1))); + const r = run([capture('1/6', [['a', 999], ['b', 1]], [...many, file('b', 'other.test.ts', 999_999)])]); + const row = r.report.rows.find((x) => x.name === 'a'); + if (row.files.length !== SLOWEST_FILES_NAMED || row.files[0].file !== 'f8.test.ts' || r.err[0].includes('other.test.ts ')) { + throw new Error(`over: the named files are wrong: ${JSON.stringify(row.files.map((f) => f.file))}`); + } + }); + + battery('provisional prints "no ceiling: provisional" and is not red'); + const prov = run([capture('1/6', [['prov', 9999], ['a', 10]])]); + check(() => { + if (prov.exitCode !== 0 || status(prov, 'prov') !== 'provisional') throw new Error(`provisional: exit ${prov.exitCode}, ${status(prov, 'prov')}`); + }); + check(() => { + if (!prov.err[0].includes('prov: 9999.00s -- no ceiling: provisional')) throw new Error(`provisional: the line is wrong:\n${prov.err[0]}`); + }); + check(() => { + // Provisional wins even over a ceiling the table still holds for it. + const held = run([capture('1/6', [['prov', 9999]])], dataset({ ceilings: { a: 150, b: 80, cli: 2000, prov: 12 }, uncapped: { carried: 'carried' } })); + if (held.exitCode !== 0 || status(held, 'prov') !== 'provisional') throw new Error(`provisional: a held ceiling overrode the provisional exemption (${held.verdict})`); + }); + + battery('a package absent from the dataset is red; an uncapped one is not'); + const fresh = run([capture('2/6', [['brand-new', 5], ['a', 10]], [file('brand-new', 'n.test.ts', 4_000)])]); + check(() => { + if (fresh.exitCode !== 1 || status(fresh, 'brand-new') !== 'absent') throw new Error(`absent: exit ${fresh.exitCode}, ${status(fresh, 'brand-new')}`); + }); + check(() => { + if (!fresh.err[0].includes('brand-new: 5.00s and no ceiling') || !fresh.err[0].includes('n.test.ts')) { + throw new Error(`absent: the package or its files are not named:\n${fresh.err[0]}`); + } + }); + check(() => { + const carried = run([capture('2/6', [['carried', 9999]])]); + if (carried.exitCode !== 0 || status(carried, 'carried') !== 'uncapped') throw new Error(`uncapped: a carried package read ${carried.verdict}`); + }); + check(() => { + const carried = run([capture('2/6', [['carried', 9999], ['a', 1]])]); + if (!carried.err[0].includes('carried: 9999.00s -- no ceiling: carried')) throw new Error(`uncapped: the reason is not named:\n${carried.err[0]}`); + }); + + battery('no ceiling table yet is NOT MEASURED, never red'); + const { ceilings: _c, uncapped: _u, ...preCeilings } = dataset(); + check(() => { + // Even a package absent from the dataset is not red before the table exists. + const none = run([capture('1/6', [['a', 9999], ['brand-new', 5]])], preCeilings); + if (none.exitCode !== 0 || none.verdict !== 'NOT MEASURED') throw new Error(`no table: exit ${none.exitCode}, verdict ${none.verdict}`); + }); + check(() => { + const none = run([capture('1/6', [['a', 9999]])], preCeilings); + if (!none.err[0].includes('no ceiling table yet')) throw new Error(`no table: the reason is not named:\n${none.err[0]}`); + }); + check(() => { + // Said as a warning annotation, so a persistent one is seen. + const none = run([capture('1/6', [['a', 9999]])], preCeilings); + if (!none.out.some((l) => l.startsWith('::warning title=Suite-duration ceilings::NOT MEASURED'))) { + throw new Error(`no table: no warning annotation: ${JSON.stringify(none.out)}`); + } + }); + check(() => { + // No capture at all, and only cached packages, grade nothing: NOT MEASURED. + const empty = run([]); + if (empty.exitCode !== 0 || empty.verdict !== 'NOT MEASURED') throw new Error(`no captures: verdict ${empty.verdict}`); + }); + check(() => { + const onlyProv = run([capture('1/6', [['prov', 1]])]); + if (onlyProv.exitCode !== 0 || onlyProv.verdict !== 'NOT MEASURED') throw new Error(`nothing graded: verdict ${onlyProv.verdict}`); + }); + + battery('file-sliced packages are summed across shards'); + check(() => { + // Three parts on three shards: summed, graded against the WHOLE ceiling. + const r = run([capture('2/6', [['cli', 600, [1, 3]]]), capture('3/6', [['cli', 700, [2, 3]]]), capture('4/6', [['cli', 650, [3, 3]]])]); + const row = r.report.rows.find((x) => x.name === 'cli'); + if (row.seconds !== 1950 || row.status !== 'ok' || r.exitCode !== 0) throw new Error(`slices: read ${row.seconds}s, ${row.status}`); + }); + check(() => { + // THE SKEW CASE: the heavy slice is past a third of the ceiling (700 > + // 2000/3 = 666.67), but the suite is within it -- not red. + const r = run([capture('2/6', [['cli', 560, [1, 3]]]), capture('3/6', [['cli', 700, [2, 3]]]), capture('4/6', [['cli', 600, [3, 3]]])]); + if (r.exitCode !== 0 || status(r, 'cli') !== 'ok') throw new Error(`slices: a skewed slice redded a suite within its ceiling (${r.verdict})`); + }); + check(() => { + // The sum over the whole ceiling is red, said as a summed set. + const r = run([capture('2/6', [['cli', 700, [1, 3]]]), capture('3/6', [['cli', 700, [2, 3]]]), capture('4/6', [['cli', 700, [3, 3]]])]); + if (r.exitCode !== 1 || !r.err[0].includes('cli: 2100.00s measured against its 2000.00s ceiling') || !r.err[0].includes('all 3 slices, summed across shards')) { + throw new Error(`slices: an over-ceiling sum was not red, or not said as a sum:\n${r.err[0]}`); + } + }); + check(() => { + // A part missing: NOT MEASURED for the package, never a part graded as the whole. + const r = run([capture('2/6', [['cli', 9999, [1, 3]]]), capture('3/6', [['cli', 9999, [2, 3]]])]); + if (r.exitCode !== 0 || status(r, 'cli') !== 'partial') throw new Error(`slices: an incomplete set read ${status(r, 'cli')} / ${r.verdict}`); + }); + check(() => { + const r = run([capture('2/6', [['cli', 10, [1, 3]]]), capture('3/6', [['a', 1]])]); + if (!r.err[0].includes('cli: NOT MEASURED') || !r.err[0].includes('slices 1 of 3')) throw new Error(`slices: the partial line is wrong:\n${r.err[0]}`); + }); + check(() => { + // A capture in a shape this fold does not know contributes nothing and is + // named, never read as a set of windows. + const folded = foldCaptures([{ ...capture('2/6', [['cli', 600, [1, 3]]]), schema: 'test-timing-capture/2' }, capture('3/6', [['a', 1]])]); + if (folded.packages.some((p) => p.pkg === 'cli') || !folded.problems.some((p) => p.includes('test-timing-capture/2'))) { + throw new Error(`slices: an unknown capture schema was folded in: ${JSON.stringify(folded)}`); + } + }); + + battery('a ruled raise lifts a ceiling, and only a well-formed one does'); + check(() => { + // Over the generated 150, within the ruled 200: not red, and the ruling is named. + const r = run([capture('1/6', [['a', 180]])], dataset(), { a: { seconds: 200, ruling: raiseUrl } }); + const row = r.report.rows.find((x) => x.name === 'a'); + if (r.exitCode !== 0 || row.status !== 'ok' || row.ceiling !== 200) throw new Error(`raise: ${row.status} against ${row.ceiling}`); + }); + check(() => { + const r = run([capture('1/6', [['a', 201]])], dataset(), { a: { seconds: 200, ruling: raiseUrl } }); + if (r.exitCode !== 1 || !r.err[0].includes(raiseUrl)) throw new Error(`raise: over the raise was not red, or the ruling not named:\n${r.err[0]}`); + }); + check(() => { + // A new package's first ceiling, before any refresh could give it one. + const r = run([capture('1/6', [['brand-new', 5]])], dataset(), { 'brand-new': { seconds: 30, ruling: raiseUrl } }); + if (r.exitCode !== 0 || status(r, 'brand-new') !== 'ok') throw new Error(`raise: a ruled first ceiling did not lift the new-package red (${status(r, 'brand-new')})`); + }); + check(() => { + // A raise at or under the generated ceiling changes nothing, and says so. + const r = run([capture('1/6', [['a', 151]])], dataset(), { a: { seconds: 120, ruling: raiseUrl } }); + if (r.exitCode !== 1 || !r.err[0].includes('changing nothing')) throw new Error(`raise: a stale raise lowered or silently stayed:\n${r.err[0]}`); + }); + check(() => { + // No ruling, no raise: the record is refused and the run is red. + const r = run([capture('1/6', [['a', 180]])], dataset(), { a: { seconds: 200, ruling: 'the maintainer said so' } }); + if (r.exitCode !== 1 || r.verdict !== 'REFUSED') throw new Error(`raise: a raise naming no ruling was applied (${r.verdict})`); + }); + check(() => { + if (raiseProblems({ a: { seconds: -5, ruling: raiseUrl } }).length !== 1) throw new Error('raise: a negative raise was accepted'); + if (raiseProblems([]).length !== 1) throw new Error('raise: a list was read as the record'); + }); + check(() => { + // The live record is well-formed. + const live = raiseProblems(RULED_CEILING_RAISES); + if (live.length) throw new Error(`raise: RULED_CEILING_RAISES is malformed: ${live.join('; ')}`); + }); + + battery('a malformed ceiling table is refused, never read as a pass'); + check(() => { + if (!threw(() => run([capture('1/6', [['a', 1]])], dataset({ ceilings: [150] })))) throw new Error('table: a list was read as `ceilings`'); + }); + check(() => { + if (!threw(() => run([capture('1/6', [['a', 1]])], dataset({ ceilings: { a: 0, b: 80, cli: 2000 } })))) throw new Error('table: a zero ceiling was graded against'); + }); + check(() => { + // A weighed package in neither map is a table the generator did not write. + if (!threw(() => run([capture('1/6', [['a', 1]])], dataset({ uncapped: { prov: 'provisional' } })))) { + throw new Error('table: a package with neither a ceiling nor a reason was read as uncapped'); + } + }); + check(() => { + if (!threw(() => readCeilingTable({ ceilings: {} }))) throw new Error('table: a dataset with no `packages` was accepted'); + }); + + // The floor: every declared battery ran, and ran its cases. + const floorFailures = []; + const declared = Object.keys(SELF_TEST_BATTERIES); + if (declared.length < SELF_TEST_BATTERY_FLOOR) { + floorFailures.push(`SELF_TEST_BATTERIES declares ${declared.length} batteries, below the pinned ${SELF_TEST_BATTERY_FLOOR}`); + } + for (const [name, count] of batterySeen) { + if (!declared.includes(name)) floorFailures.push(`battery "${name}" registered ${count} case(s) but is not declared`); + } + for (const name of declared) { + const count = batterySeen.get(name) ?? 0; + if (count < SELF_TEST_BATTERIES[name]) { + floorFailures.push( + count === 0 + ? `battery "${name}" DID NOT RUN -- 0 cases registered, ${SELF_TEST_BATTERIES[name]} pinned` + : `battery "${name}" registered ${count} case(s), below its floor of ${SELF_TEST_BATTERIES[name]}` + ); + } + } + if (floorFailures.length > 0) { + throw new Error( + `check-test-suite-ceilings self-test floor (${floorFailures.length} breach(es)):\n` + + floorFailures.map((f) => ` - ${f}`).join('\n') + + '\n A battery below its floor means cases STOPPED RUNNING; find what stopped registering and restore it.' + ); + } + + const total = [...batterySeen.values()].reduce((a, b) => a + b, 0); + console.log(`check-test-suite-ceilings: self-test OK (${total} cases across ${declared.length} batteries)`); + selfTestReachedVerdict = true; +} + +// ── Entry point ─────────────────────────────────────────────────────────── + +function main() { + const argv = process.argv.slice(2); + if (argv.includes('--self-test')) { + selfTest(); + if (!selfTestReachedVerdict) { + console.error( + '\n✗ check-test-suite-ceilings self-test: selfTest() returned without reaching its verdict,\n' + + 'so no success line was printed. Exiting 0 here would report a self-test\n' + + 'that never finished as a self-test that passed.\n' + ); + process.exit(1); + } + return; + } + + let capturesDir = null; + let label = 'Test Core'; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--captures') capturesDir = argv[++i]; + else if (argv[i] === '--label') label = argv[++i]; + else throw new Error(`unrecognized argument: ${argv[i]}`); + } + if (!capturesDir) { + console.error('usage: check-test-suite-ceilings.mjs --captures [--label ] | --self-test'); + process.exit(1); + } + + const { captures, problems: readProblems } = readCaptures(capturesDir); + const merged = foldCaptures(captures); + const dataset = JSON.parse(readFileSync(DATASET_PATH, 'utf8')); + const report = gradeCeilings({ merged, dataset }); + const rendered = renderCeilingVerdict(report, label); + for (const line of rendered.out) console.log(line); + for (const line of rendered.err) console.error(line); + // Not a verdict of their own: a shard capture that could not be read leaves + // its packages ungraded, which the counts above already show. Said, so an + // OK over five of six shards never reads as an OK over six. + const notes = [...readProblems, ...merged.problems]; + const shards = merged.shards.length; + console.error( + `suite-ceiling: read ${captures.length} shard capture(s) (${shards ? merged.shards.join(', ') : 'none'})` + + `${notes.length ? `; ${notes.length} could not be fully read, so their packages are ungraded: ${notes.join('; ')}` : ''}.` + ); + if (rendered.exitCode !== 0) process.exit(rendered.exitCode); +} + +// Exports bindings, so an import for those exports alone must run nothing. +if (isEntrypoint(import.meta.url)) { + main(); +} diff --git a/scripts/measure-test-shard-timings.mjs b/scripts/measure-test-shard-timings.mjs index 43b32dd6bb1..4c08f3d8463 100644 --- a/scripts/measure-test-shard-timings.mjs +++ b/scripts/measure-test-shard-timings.mjs @@ -155,6 +155,92 @@ export function median(values) { // lane exists to end. export const MINIMUM_EXECUTED_RUNS = 3; +// The headroom of the per-package suite-duration ceiling (#16468). A ruled +// number, not a tunable: the maintainer's ruling put it at 25% over "the +// refreshed measurement", and the ruling that followed read that measurement +// as the package's SLOWEST executed run in the refresh window -- not its +// median, which is this file's weight. Measured on the 21-run refresh that +// landed as 040184752c: 31 of the 71 measured packages had a window maximum +// above median x 1.25, so a ceiling built on the median would have been red on +// the dataset's own runs, the standing-red shape a brake is learned around. +// +// The ceiling travels with the dataset as its own field (`ceilings`), beside +// the weights rather than in place of them: the partitioner and the drift gate +// keep reading `packages` as name -> median, unchanged. +export const CEILING_HEADROOM = 1.25; + +// One package's ceiling from its executed per-run readings in this refresh: +// the slowest of them times the headroom, rounded UP to the hundredth, so the +// rounding can never put the window's own slowest run over its own line. +// A file-sharded package's readings are already per-run SUMS of its slices +// (collectSamples), so this is the slowest whole-suite run, never one slice. +export function ceilingOf(values, headroom = CEILING_HEADROOM) { + if (!Array.isArray(values) || values.length === 0) throw new Error('ceilingOf: no executed readings'); + return Math.ceil(Math.max(...values) * headroom * 100) / 100; +} + +// The per-package ceilings a refresh writes (#16468), and the packages that +// get none, with the reason. Every package in `packages` lands in exactly one +// of the two. +// +// HELD a package the prior dataset already gave a ceiling keeps it, +// byte for byte. A refresh NEVER raises a ceiling -- the ruling +// is "a ceiling rises only by a ruling", and a refresh that +// re-derived it from a slower window would raise it by the very +// growth the ratchet exists to stop (the line ratchets' rule, +// which the card cites). A raise is a ruled record in +// scripts/check-test-suite-ceilings.mjs, never an edit here. +// Nor does a refresh LOWER one: a quieter window is variance, not +// a faster suite, and a ceiling ratcheted down on variance is red +// on the next ordinary run. +// NEW a package with no prior ceiling, not provisional, and executed +// in this refresh: its slowest executed run x the headroom. +// uncapped `provisional` -- fewer than `minimumRuns` executed runs is not a +// baseline (#22014), so no ceiling and, per the ruling, no red; +// `carried` -- re-confirmed by a cache HIT but executed in no run +// of this window, so there is no slowest run to read. The next +// refresh that executes it gives it one. +// +// A prior ceiling for a package no longer in `packages` is dropped with it. +export function suiteCeilings({ bySample, packages, provisional, prior = null, headroom = CEILING_HEADROOM }) { + if (prior !== null) { + if (typeof prior !== 'object' || Array.isArray(prior)) { + throw new Error( + `the prior dataset's \`ceilings\` is ${JSON.stringify(prior)}, not a map of package -> seconds. ` + + 'Refusing to guess which ceilings to hold.' + ); + } + for (const [name, seconds] of Object.entries(prior)) { + if (typeof seconds !== 'number' || !Number.isFinite(seconds) || !(seconds > 0)) { + throw new Error( + `the prior dataset's ceiling for ${name} is ${JSON.stringify(seconds)}, not a positive duration. ` + + 'Refusing to hold or replace it by guessing.' + ); + } + } + } + const short = new Set(provisional); + const ceilings = {}; + const uncapped = {}; + for (const name of Object.keys(packages).sort((a, b) => a.localeCompare(b, 'en'))) { + if (prior !== null && Object.hasOwn(prior, name)) { + ceilings[name] = prior[name]; + continue; + } + if (short.has(name)) { + uncapped[name] = 'provisional'; + continue; + } + const values = bySample.get(name); + if (!values || values.length === 0) { + uncapped[name] = 'carried'; + continue; + } + ceilings[name] = ceilingOf(values, headroom); + } + return { ceilings, uncapped }; +} + // Which file-level slice a summary's tasks were run as, or null for a whole // package (#16173). // @@ -569,6 +655,9 @@ export function buildDataset({ provenance, carryFrom = null, carryProvisional = null, + // The prior dataset's `ceilings`, or null when it has none -- a dataset + // written before the ceilings existed, read as "nothing to hold" (#16468). + carryCeilings = null, minimumRuns = MINIMUM_EXECUTED_RUNS, }) { const { bySample, cachedNames, incompleteSlices } = collectSamples(perSummary); @@ -628,6 +717,8 @@ export function buildDataset({ if (!Number.isInteger(minimumRuns) || minimumRuns < 1) { throw new Error(`minimumRuns must be a positive integer, got ${JSON.stringify(minimumRuns)}`); } + const provisional = provisionalPackages({ bySample, carriedOver, carryProvisional, minimumRuns }); + const { ceilings, uncapped } = suiteCeilings({ bySample, packages, provisional, prior: carryCeilings }); return { note: 'GENERATED by scripts/measure-test-shard-timings.mjs -- do not hand-edit. Per-package ' + @@ -636,10 +727,15 @@ export function buildDataset({ 'Every weight is a real measurement: the ones in `carriedOver` were measured by an earlier ' + 'refresh and re-confirmed unchanged by a turbo cache HIT in this one. A weight is the median ' + 'of the executed runs that measured it; the ones in `provisional` rest on fewer than ' + - '`provenance.minimumRuns` of them -- fit to balance on, NOT a baseline to set a ceiling from.', + '`provenance.minimumRuns` of them -- fit to balance on, NOT a baseline to set a ceiling from. ' + + '`ceilings` is each package\'s suite-duration ceiling, graded by scripts/check-test-suite-ceilings.mjs: ' + + 'its slowest executed run in the refresh window that first set it, times `provenance.ceilingHeadroom`, ' + + 'then held by every later refresh -- raised only by a ruling, never by a refresh. `uncapped` names ' + + 'the packages with none, and why.', // The bar `provisional` was judged against travels with the list, so a // reader never has to know which version of this script wrote the file. - provenance: { ...provenance, minimumRuns }, + // The ceiling headroom travels the same way (#16468). + provenance: { ...provenance, minimumRuns, ceilingHeadroom: CEILING_HEADROOM }, secondsPerTestFileFallback: fallbackRate(measured, fileCounts), packages, skippedAsCached: [...cachedNames].sort((a, b) => a.localeCompare(b, 'en')), @@ -657,14 +753,22 @@ export function buildDataset({ // the drift gate read `packages` as name -> number. Always present -- an // empty list and an absent key must not read the same way, since an absent // key is exactly how a dataset that predates this rule is recognised. - provisional: provisionalPackages({ bySample, carriedOver, carryProvisional, minimumRuns }), + provisional, + // #16468. The per-package suite-duration ceilings and the packages with + // none (see suiteCeilings). Beside the weights, never in place of them -- + // the same reason as `carriedOver` and `provisional`. Always present, for + // the same reason as `provisional`: an ABSENT `ceilings` is how the check + // recognises a dataset written before the ceilings existed, which it reads + // as NOT MEASURED rather than as a table with no rows. + ceilings, + uncapped, }; } // The run-to-run spread of every package, as the Markdown table the refresh // PR's body carries (#22014). One row per package in the dataset: // -// package | runs | min | median | max | max/min +// package | runs | min | median | max | max/min | ceiling // // where `runs` counts the executed runs behind the weight and the three // seconds columns are those runs' readings. A carried package has no reading in @@ -672,6 +776,13 @@ export function buildDataset({ // one is marked in its own column. Rows sort by max/min, widest first, so the // noisiest suites are the ones a reviewer meets first. // +// `ceiling` (#16468) is the dataset's own `ceilings` entry, `—` for an +// uncapped package with the reason in the last column. A HELD ceiling that this +// window would have set differently says so -- `held; this window: N` -- so the +// reviewer sees the growth a refresh deliberately did not absorb (a raise is a +// ruling), and a window reading already OVER its held ceiling is called out +// rather than left to be inferred from two columns. +// // ⛔ Kept OUT of the dataset file. Nothing machine-reads a spread -- the // partitioner and the drift gate read the weights, and the one machine fact a // baseline needs (did this weight meet the minimum?) is `provisional` -- so the @@ -679,12 +790,30 @@ export function buildDataset({ export function renderSpread({ bySample, dataset }) { const fmt = (n) => n.toFixed(2); const provisional = new Set(dataset.provisional); + const ceilings = dataset.ceilings ?? {}; + const uncapped = dataset.uncapped ?? {}; + const headroom = dataset.provenance?.ceilingHeadroom ?? CEILING_HEADROOM; + const ceilingCells = (name, values) => { + const flags = []; + if (provisional.has(name)) flags.push('provisional'); + if (!Object.hasOwn(ceilings, name)) { + if (uncapped[name] === 'carried') flags.push('carried: no ceiling'); + return ['—', flags.join('; ')]; + } + const ceiling = ceilings[name]; + if (values && !provisional.has(name)) { + const fresh = ceilingOf(values, headroom); + if (fresh !== ceiling) flags.push(`held; this window: ${fmt(fresh)}`); + if (Math.max(...values) > ceiling) flags.push('⚠ a window run is OVER its held ceiling'); + } + return [fmt(ceiling), flags.join('; ')]; + }; const rows = []; for (const name of Object.keys(dataset.packages)) { const values = bySample.get(name); - const flag = provisional.has(name) ? 'provisional' : ''; + const [ceilingCell, flag] = ceilingCells(name, values); if (!values) { - rows.push({ ratio: -1, name, cells: [`\`${name}\``, '0', 'carried', 'carried', 'carried', '—', flag] }); + rows.push({ ratio: -1, name, cells: [`\`${name}\``, '0', 'carried', 'carried', 'carried', '—', ceilingCell, flag] }); continue; } const lo = Math.min(...values); @@ -700,6 +829,7 @@ export function renderSpread({ bySample, dataset }) { fmt(Math.round(median(values) * 100) / 100), fmt(hi), Number.isFinite(ratio) ? `${ratio.toFixed(2)}×` : '∞', + ceilingCell, flag, ], }); @@ -707,10 +837,12 @@ export function renderSpread({ bySample, dataset }) { rows.sort((a, b) => b.ratio - a.ratio || a.name.localeCompare(b.name, 'en')); const minimumRuns = dataset.provenance?.minimumRuns; const lines = [ - `Each package's executed runs in this refresh (minimum ${minimumRuns}), and the spread of its readings in seconds.`, + `Each package's executed runs in this refresh (minimum ${minimumRuns}), the spread of its readings in seconds, ` + + `and the suite-duration ceiling \`Test Core\` grades it against: its slowest executed run x ${headroom} when ` + + 'first set, then held by every refresh -- a ceiling is raised only by a ruling.', '', - '| package | runs | min s | median s | max s | max/min | |', - '|---|--:|--:|--:|--:|--:|---|', + '| package | runs | min s | median s | max s | max/min | ceiling s | |', + '|---|--:|--:|--:|--:|--:|--:|---|', ...rows.map((r) => `| ${r.cells.join(' | ')} |`), ]; return `${lines.join('\n')}\n`; @@ -748,11 +880,13 @@ const SELF_TEST_BATTERIES = Object.freeze({ 'env-carried slices (#19278)': 11, // #22014: the per-package minimum of executed runs, and the spread table. 'minimum executed runs per package': 14, + // #16468: the per-package suite-duration ceilings the refresh writes. + 'suite-duration ceilings (#16468)': 14, }); // DELETING an entry silences that battery's floor exactly as effectively as // zeroing it, so the roster's own size is pinned too. -const SELF_TEST_BATTERY_FLOOR = 3; +const SELF_TEST_BATTERY_FLOOR = 4; // The key an assertion is filed under when no battery is open. It is not a // declared battery, so it reds by the same set difference rather than silently @@ -1664,14 +1798,116 @@ function selfTest() { const table = renderSpread({ bySample: collectSamples(runs).bySample, dataset: ds }); const body = table.split('\n').filter((l) => l.startsWith('| `')); if (body.length !== 3) throw new Error(`spread: expected one row per package, got ${body.length}:\n${table}`); - if (body[0] !== '| `a` | 3 | 10.00 | 20.00 | 30.00 | 3.00× | |') throw new Error(`spread: the widest row is wrong: ${body[0]}`); - if (!body.some((l) => l.startsWith('| `b` | 1 | 20.00 | 20.00 | 20.00 | 1.00× | provisional |'))) { + if (body[0] !== '| `a` | 3 | 10.00 | 20.00 | 30.00 | 3.00× | 37.50 | |') throw new Error(`spread: the widest row is wrong: ${body[0]}`); + if (!body.some((l) => l.startsWith('| `b` | 1 | 20.00 | 20.00 | 20.00 | 1.00× | — | provisional |'))) { throw new Error(`spread: the provisional row is wrong:\n${table}`); } if (!body.some((l) => l.startsWith('| `cached` | 0 | carried |'))) throw new Error(`spread: the carried row is wrong:\n${table}`); if (!table.includes(`minimum ${MINIMUM_EXECUTED_RUNS}`)) throw new Error('spread: the table does not state the minimum'); }); + // -- The suite-duration ceilings (#16468) ------------------------------- + // + // The ruled reading: a package's ceiling is its SLOWEST executed run in the + // refresh window x 1.25 (not its median, which is the weight), a provisional + // package gets none, and a refresh never moves a ceiling it already set. + battery('suite-duration ceilings (#16468)'); + const windowRuns = [ + runOf('r1', [testTask('a', 0, 10_000), testTask('b', 0, 20_000), testTask('cached', 0, 90, 'HIT')]), + runOf('r2', [testTask('a', 0, 30_000), testTask('cached', 0, 90, 'HIT')]), + runOf('r3', [testTask('a', 0, 20_000), testTask('cached', 0, 90, 'HIT')]), + ]; + const ceilingSet = (extra = {}) => depthSet(windowRuns, { carryFrom: { cached: 500 }, carryProvisional: [], ...extra }); + check(() => { + // The ruled headroom. Changing it is a ruling, not a tune. + if (CEILING_HEADROOM !== 1.25) throw new Error(`ceilings: CEILING_HEADROOM is ${CEILING_HEADROOM}, not the ruled 1.25`); + }); + check(() => { + // THE RULED READING: the window MAXIMUM (30s) x 1.25 = 37.5s, not the + // median (20s) x 1.25 = 25s -- which the window's own slowest run is over. + const first = ceilingSet(); + if (first.ceilings.a !== 37.5) throw new Error(`ceilings: a read ${first.ceilings.a}, not its slowest run x 1.25 (37.5)`); + }); + check(() => { + // The weights are untouched: `a` still weighs its median. + if (ceilingSet().packages.a !== 20) throw new Error(`ceilings: the median weight moved (${ceilingSet().packages.a})`); + }); + check(() => { + // Provisional: one executed run is not a baseline -- no ceiling, named why. + const first = ceilingSet(); + if (Object.hasOwn(first.ceilings, 'b') || first.uncapped.b !== 'provisional') { + throw new Error(`ceilings: a provisional package got ${JSON.stringify(first.ceilings.b)} / ${JSON.stringify(first.uncapped.b)}`); + } + }); + check(() => { + // Carried and not provisional, with nothing to hold: no slowest run exists + // in this window, so no ceiling, and the reason is named. + const first = ceilingSet(); + if (Object.hasOwn(first.ceilings, 'cached') || first.uncapped.cached !== 'carried') { + throw new Error(`ceilings: a carried package got ${JSON.stringify(first.ceilings.cached)} / ${JSON.stringify(first.uncapped.cached)}`); + } + }); + check(() => { + // Every package lands in exactly one of the two maps. + const first = ceilingSet(); + const names = Object.keys(first.packages); + const both = names.filter((n) => Object.hasOwn(first.ceilings, n) === Object.hasOwn(first.uncapped, n)); + if (both.length) throw new Error(`ceilings: ${both.join(', ')} is in both maps or in neither`); + }); + check(() => { + // HELD, never raised: a prior 30s stays 30s although this window gives 37.5. + const held = ceilingSet({ carryCeilings: { a: 30 } }); + if (held.ceilings.a !== 30) throw new Error(`ceilings: a refresh raised a held ceiling to ${held.ceilings.a}`); + }); + check(() => { + // HELD, never lowered: a quieter window is variance, not a faster suite. + const held = ceilingSet({ carryCeilings: { a: 100 } }); + if (held.ceilings.a !== 100) throw new Error(`ceilings: a refresh lowered a held ceiling to ${held.ceilings.a}`); + }); + check(() => { + // A carried package keeps the ceiling an earlier window gave it, and a + // ceiling for a package gone from the dataset goes with it. + const held = ceilingSet({ carryCeilings: { cached: 640, gone: 9 } }); + if (held.ceilings.cached !== 640 || Object.hasOwn(held.uncapped, 'cached')) { + throw new Error(`ceilings: a carried package lost its held ceiling (${JSON.stringify(held.ceilings)})`); + } + if (Object.hasOwn(held.ceilings, 'gone')) throw new Error('ceilings: a ceiling outlived its package'); + }); + check(() => { + // A malformed prior table is refused, never guessed around. + if (!threw(() => ceilingSet({ carryCeilings: [30] }))) throw new Error('ceilings: a prior list was read as a map'); + if (!threw(() => ceilingSet({ carryCeilings: { a: -1 } }))) throw new Error('ceilings: a negative prior ceiling was held'); + }); + check(() => { + // A file-sharded package's ceiling is from its per-run SUMS: the slowest + // whole run (650s), never the slowest slice (330s). + const sliceRun = (run, x, y) => [ + runOf(run, [slicedTask('cli', 0, x, 1, 2)]), + runOf(run, [slicedTask('cli', 0, y, 2, 2)]), + ]; + const ds = depthSet([...sliceRun('r1', 300_000, 300_000), ...sliceRun('r2', 310_000, 310_000), ...sliceRun('r3', 320_000, 330_000)]); + if (ds.ceilings.cli !== 812.5) throw new Error(`ceilings: a sliced package read ${ds.ceilings.cli}, not its slowest whole run x 1.25 (812.5)`); + }); + check(() => { + // The headroom travels with the table, like the minimum. + if (ceilingSet().provenance.ceilingHeadroom !== CEILING_HEADROOM) throw new Error('ceilings: provenance does not record the headroom'); + }); + check(() => { + // Rounding is UP, so the window's slowest run is never over its own line. + if (ceilingOf([1.0001]) < 1.0001 * CEILING_HEADROOM) throw new Error(`ceilings: rounding put the line under max x headroom (${ceilingOf([1.0001])})`); + if (!threw(() => ceilingOf([]))) throw new Error('ceilings: an empty reading set produced a ceiling'); + }); + check(() => { + // The spread table shows a held ceiling this window would have set + // differently, and a run already over it. + const held = ceilingSet({ carryCeilings: { a: 25 } }); + const table = renderSpread({ bySample: collectSamples(windowRuns).bySample, dataset: held }); + const row = table.split('\n').find((l) => l.startsWith('| `a` |')); + if (row !== '| `a` | 3 | 10.00 | 20.00 | 30.00 | 3.00× | 25.00 | held; this window: 37.50; ⚠ a window run is OVER its held ceiling |') { + throw new Error(`spread: the held row is wrong: ${row}`); + } + }); + // -- The floor: every declared battery RAN, and ran its cases (#13489) ---- // // Evaluated after every battery has had its chance and BEFORE the verdict, so @@ -1840,6 +2076,7 @@ function main() { // from the numbers actually in the file rather than from a subset of them. let carryFrom = null; let carryProvisional = null; + let carryCeilings = null; if (mergeInto !== null) { const prior = JSON.parse(readFileSync(mergeInto, 'utf8')); if (!prior || typeof prior.packages !== 'object' || prior.packages === null) { @@ -1853,6 +2090,10 @@ function main() { // dataset with no `provisional` key never recorded one, which buildDataset // reads as "not shown to meet the minimum" -- null, deliberately not []. carryProvisional = Object.hasOwn(prior, 'provisional') ? prior.provisional : null; + // The ceilings it set are HELD (#16468): a refresh never raises or lowers + // one. A prior dataset with no `ceilings` key predates them -- nothing to + // hold, so this refresh sets the first table. + carryCeilings = Object.hasOwn(prior, 'ceilings') ? prior.ceilings : null; } const fileCounts = new Map(); @@ -1870,6 +2111,7 @@ function main() { fileCounts, carryFrom, carryProvisional, + carryCeilings, provenance: { measuredAt: new Date().toISOString().slice(0, 10), summaries: inputs.map((i) => path.basename(i.file)), @@ -1926,6 +2168,26 @@ function main() { 'read them as balancing inputs only.' ); } + // The ceilings (#16468), on the same stderr: how many a package now has, which + // packages have none and why, and -- the line a reviewer most needs -- any + // HELD ceiling a run of this very window already went over, which on `main` + // is the brake having fired, not a number to absorb. + const uncappedNames = Object.entries(dataset.uncapped); + const bySampleNow = collectSamples(perSummary).bySample; + const over = Object.entries(dataset.ceilings) + .filter(([name, ceiling]) => !dataset.provisional.includes(name) && Math.max(...(bySampleNow.get(name) ?? [0])) > ceiling) + .map(([name, ceiling]) => `${name} (${Math.max(...bySampleNow.get(name)).toFixed(2)}s > ${ceiling}s)`); + console.error( + `measure-test-shard-timings: ${Object.keys(dataset.ceilings).length} suite-duration ceiling(s)` + + `${carryCeilings === null ? ' -- the FIRST table: every one is its slowest executed run x ' + dataset.provenance.ceilingHeadroom : ', the prior ones held'}` + + `; ${uncappedNames.length} uncapped${uncappedNames.length ? ` (${uncappedNames.map(([n, why]) => `${n}: ${why}`).join(', ')})` : ''}.` + ); + if (over.length > 0) { + console.error( + `measure-test-shard-timings: ⚠ ${over.length} package(s) ran OVER their held ceiling in this window: ` + + `${over.join(', ')}. A refresh never raises a ceiling; that takes a ruling.` + ); + } if (dataset.skippedIncompleteSlices.length > 0) { console.error( `measure-test-shard-timings: ⚠ ${dataset.skippedIncompleteSlices.length} package(s) reported an ` +