bench: one-pass release evidence with lower-noise engine comparisons

.gitignore

@@ -26,6 +26,8 @@ playwright-report/
 bench/results/candidates/
 bench/results/render-fresh.json
 bench/results/render-fresh.md
+# Release-capture rehearsals; never published evidence
+bench/results/smoke/
 .formula-evidence/
 
 # Editor / OS

bench/README.md

@@ -81,19 +81,32 @@ passed timing regression gate. The legacy shared source keys name the WASM
 role; their hashes use the selected engine. The outer artifact records the
 physical paths and binds each engine to its captured WASM hash.
 
-Build the packages first. Run the capture from `bench/` with one pinned runner:
+Build the packages first. Run one capture at a time from `bench/` on an
+otherwise idle machine, pinned to one CPU. Do not run concurrent builds or
+timing captures: competing work adds scheduler and CPU noise. For example,
+on Linux with CPU 4:
 
 ```sh
-flock -x /home/vvin/learn/sheetwrite-wt/build.lock flock -x /home/vvin/learn/sheetwrite-wt/quiet.lock taskset -c 4 bun run src/full-engine-bench.ts
+taskset -c 4 bun run src/full-engine-bench.ts
 ```
 
 `bench:formula:matched` checks the eight shared rows that were slower in the
-first sequential capture. It runs 51 default/full pairs per row. Each timed
-sample uses a fresh process and one untimed warmup. The artifact retains all
-samples and checksums, median paired ratios, and the p10 to p90 ratio spread.
-The single-sample runner is `bun run src/formula-bench.ts --sample ID SIZE`.
+first sequential capture. It runs 25 default/full pairs per row (`--rounds`
+changes that count, minimum 2). Each timed sample uses a fresh Bun process,
+runs three untimed warm-up fixtures, then repeats the fixture until at least
+50 ms of measured time and records the per-iteration mean. Repeating a short
+operation is what makes the paired ratio usable: a single cold run of a 2 ms
+workload drifted by more than 4x between fresh processes, while the aggregated
+sample stays inside a few percent. The artifact retains all samples and
+checksums, every sample's iteration count and total, median paired ratios, and
+the p10 to p90 ratio spread; the evidence page marks any row whose spread is
+too wide to support a ratio claim.
+The single-sample runner is
+`bun run src/formula-bench.ts --sample ID SIZE [--aggregate-ms 50] [--warmup-fixtures 3]`.
 It supports the matched workload set. Use the same pinned-runner command as
-above with `src/matched-engine-bench.ts`.
+above with `src/matched-engine-bench.ts`. `--results-dir <dir>` redirects every
+artifact write of the engine captures, so a rehearsal never overwrites tracked
+evidence.
 
 The suite covers independent parse/load and first recompute, safe-depth linear
 chains, 100K fan-out, diamonds, shared/distinct ranges, cross-sheet ranges,
@@ -206,8 +219,48 @@ bun run bench:render:smoke -- --engine handsontable
 
 # Fail-closed schema, completeness, and byte-stable Markdown validation:
 bun run bench:render:validate
+
+# Core write-path evidence (packed block writes, CSV import, large restore,
+# paged residency); writes results/core-paths-results.json
+bun run bench:core-paths
 ```
 
+The evidence page at
+`docs/src/content/docs/guides/performance-resources.md` publishes every
+artifact above. One command captures that whole set, in order, from a clean
+tree. Use one pinned CPU on an otherwise idle machine, with no concurrent
+builds or timing captures. The example below uses CPU 4 on Linux:
+
+```sh
+# Build first: every capture needs the built packages and WASM binaries.
+bun run build:packages
+
+# Then, from bench/, with the prepared XLSX comparison checkout:
+taskset -c 4 bun run bench:release --xlsx-baseline-root /path/to/sheetwrite-xlsx-baseline
+```
+
+`bench:release` refuses a dirty tree, runs the captures in page order, stops at
+the first failure, and rejects an artifact that does not stamp the current
+clean commit. The XLSX capture compares the current tree against the
+pre-0.5.0 codec, so the run needs a prepared checkout of that commit
+(`git worktree add ../sheetwrite-xlsx-baseline 87fadb72`, then
+`bun install --frozen-lockfile && bun run build:packages` inside it) passed as
+`--xlsx-baseline-root <checkout path>`. `bench:release:smoke`
+rehearses the same order with the smoke
+matrices of the suites that have one, writing to `results/smoke/` (ignored by
+git) so a rehearsal never overwrites published evidence. Regenerate the page
+after a release capture with `bun run docs:generate`.
+
+A change to a harness source file, to the sampling flags, or to the protocol
+version invalidates the committed render baseline: `bench:check` compares the
+harness hashes, the declared sampling, and the runner (OS, CPU, Bun, Node,
+Chromium, power mode, concurrency) of the fresh capture against
+`results/render-baseline.json` and fails closed on any mismatch. `bench:release`
+therefore ends by re-recording it on the frozen harness: ten controlled rounds
+into `results/render-baseline-raw.json`, then the promotion of that raw artifact
+over the committed baseline. Commit both in the release change; the promotion
+diff is baseline-only and reviewable on its own.
+
 ### Fail-closed gate policy
 
 Ordinary CI runs the deterministic gate, not a timing comparison:

bench/package.json

@@ -11,7 +11,10 @@
     "bench:range:smoke": "bun run src/range-bench.ts --smoke",
     "bench:paged": "bun run src/paged-bench.ts",
     "bench:paged:smoke": "bun run src/paged-bench.ts --smoke",
-    "bench:core-paths": "bun run src/core-paths-bench.ts",
+    "bench:core-paths": "bun run src/core-paths-bench.ts --output results/core-paths-results.json",
+    "bench:core-paths:smoke": "bun run src/core-paths-bench.ts --smoke",
+    "bench:release": "bun run src/release-capture.ts",
+    "bench:release:smoke": "bun run src/release-capture.ts --smoke",
     "bench:prefetch": "bun run src/datasource-prefetch-bench.ts",
     "bench:billion-cell": "bun run src/billion-cell-bench.ts",
     "bench:formula": "bun run src/formula-bench.ts",
@@ -33,7 +36,7 @@
     "bench:baseline:candidate": "bun run src/generate-baseline.ts",
     "test:gates": "bun test test",
     "typecheck": "tsc --noEmit -p tsconfig.json",
-    "bench:render:scale": "bun run src/render-driver.ts --rows 1000,10000,100000,1000000 --rounds 10 --compact-samples --output results/render-scale.json --markdown-output results/render-scale.md"
+    "bench:render:scale": "bun run src/render-driver.ts --rows 1000,10000,100000,1000000 --rounds 10 --samples 5 --warmups 2 --compact-samples --output results/render-scale.json --markdown-output results/render-scale.md"
   },
   "devDependencies": {
     "@happy-dom/global-registrator": "^20.14.5",

bench/src/formula-bench.ts

@@ -465,6 +465,126 @@ function summarizeAllocations(samples: readonly FormulaAllocationSample[]): Form
   };
 }
 
+/** Untimed fixture runs allowed inside one aggregated sample before it is abandoned. */
+export const AGGREGATED_SAMPLE_MAX_ITERATIONS = 1_000_000;
+/** Default aggregation floor (ms) of the single-sample runner used by the matched engine protocol. */
+export const SAMPLE_AGGREGATE_MS = 50;
+/** Default untimed warm-up fixture runs of the single-sample runner. */
+export const SAMPLE_WARMUP_FIXTURES = 3;
+
+/** Knobs for one aggregated timing sample: a wall-clock total over repeated fixture runs. */
+export interface AggregatedSampleOptions {
+  /** Untimed fixture runs before the timed loop; the first one also checks correctness. */
+  readonly warmupFixtures: number;
+  /** Timed fixture runs repeat until their total wall time reaches this floor. */
+  readonly minimumSampleDurationMs: number;
+}
+
+/**
+ * One aggregated sample. `samplesMs` holds the per-iteration mean so a sample
+ * stays comparable across workloads, while `totalMs` and `iterations` keep the
+ * raw duration and the repetition count auditable.
+ */
+export interface AggregatedWorkloadSample extends CompleteFormulaWorkloadResult {
+  readonly iterations: number;
+  readonly totalMs: number;
+  readonly warmupFixtures: number;
+}
+
+/**
+ * One timed fixture run: a fresh fixture, a monotonic duration around `run`,
+ * the post-run store allocation, and the workload's exact output check. The
+ * duration is proven finite before it can enter an aggregate.
+ */
+function timedFixtureRun(
+  id: string,
+  factory: FixtureFactory,
+  expected: FormulaOutput,
+): { readonly elapsedMs: number; readonly allocation: FormulaAllocationSample } {
+  const fixture = factory();
+  fixture.store.resetFormulaMatrixResourceStats();
+  const started = now();
+  fixture.run();
+  const elapsedMs = now() - started;
+  const allocation = allocationAt(fixture.store);
+  assert(outputsEqual(fixture.check(), expected), `${id} measured output`);
+  fixture.dispose();
+  assert(Number.isFinite(elapsedMs) && elapsedMs >= 0, `${id} measured duration`);
+  return { elapsedMs, allocation };
+}
+
+function warmUpFixtures(
+  id: string,
+  factory: FixtureFactory,
+  warmupFixtures: number,
+  expected: FormulaOutput,
+): FormulaAllocationSample {
+  if (!Number.isInteger(warmupFixtures) || warmupFixtures < 1) {
+    throw new RangeError("warmupFixtures must be a positive integer");
+  }
+  let allocation: FormulaAllocationSample | undefined;
+  for (let index = 0; index < warmupFixtures; index += 1) {
+    const fixture = factory();
+    fixture.store.resetFormulaMatrixResourceStats();
+    fixture.run();
+    assert(outputsEqual(fixture.check(), expected), `${id} warmup output`);
+    allocation = allocationAt(fixture.store);
+    fixture.dispose();
+  }
+  if (allocation === undefined) throw new RangeError("warmupFixtures must run at least once");
+  return allocation;
+}
+
+/**
+ * Collect one aggregated timing sample: untimed warm-up fixtures first, then
+ * repeated timed runs until the total wall time reaches `minimumSampleDurationMs`.
+ * A single run of a cheap workload sits close to timer and scheduler noise; the
+ * aggregate averages that noise away while keeping one auditable sample.
+ */
+export function collectAggregatedSample(
+  id: string,
+  size: number,
+  factory: FixtureFactory,
+  options: AggregatedSampleOptions,
+  expectedOutput?: FormulaOutput,
+): AggregatedWorkloadSample {
+  if (!Number.isFinite(options.minimumSampleDurationMs) || options.minimumSampleDurationMs <= 0) {
+    throw new RangeError("minimumSampleDurationMs must be a positive number");
+  }
+  const expected = expectedOutput ?? expectedFormulaOutput(id, size);
+  let allocation = warmUpFixtures(id, factory, options.warmupFixtures, expected);
+
+  // One forced GC brackets the whole sample: a per-iteration GC would dominate
+  // the cheap workloads the aggregate exists to measure.
+  forceGc();
+  let totalMs = 0;
+  let iterations = 0;
+  while (totalMs < options.minimumSampleDurationMs) {
+    if (iterations >= AGGREGATED_SAMPLE_MAX_ITERATIONS) {
+      throw new RangeError(
+        `${id} did not reach ${options.minimumSampleDurationMs} ms within ${AGGREGATED_SAMPLE_MAX_ITERATIONS} fixture runs`,
+      );
+    }
+    const run = timedFixtureRun(id, factory, expected);
+    totalMs += run.elapsedMs;
+    iterations += 1;
+    allocation = run.allocation;
+  }
+  const perIterationMs = totalMs / iterations;
+  return {
+    id,
+    size,
+    samplesMs: [perIterationMs],
+    stat: summarize([perIterationMs]),
+    allocationSamples: [allocation],
+    allocationStat: summarizeAllocations([allocation]),
+    output: expected,
+    iterations,
+    totalMs,
+    warmupFixtures: options.warmupFixtures,
+  };
+}
+
 export function collectFixture(
   id: string,
   size: number,
@@ -2416,8 +2536,26 @@ if (import.meta.main) {
     };
     const factory = id === undefined ? undefined : factories[id];
     assert(id !== undefined && factory, "unknown single-sample workload");
+    const aggregateIndex = process.argv.indexOf("--aggregate-ms");
+    const aggregateMs =
+      aggregateIndex < 0 ? SAMPLE_AGGREGATE_MS : Number(process.argv[aggregateIndex + 1]);
+    assert(Number.isFinite(aggregateMs) && aggregateMs > 0, "invalid sample aggregation floor");
+    const warmupIndex = process.argv.indexOf("--warmup-fixtures");
+    const warmupFixtures =
+      warmupIndex < 0 ? SAMPLE_WARMUP_FIXTURES : Number(process.argv[warmupIndex + 1]);
+    assert(
+      Number.isInteger(warmupFixtures) && warmupFixtures > 0,
+      "invalid sample warm-up fixture count",
+    );
     await initSheetwrite(undefined, engine);
-    process.stdout.write(`${JSON.stringify(collectFixture(id, size, factory, 1))}\n`);
+    process.stdout.write(
+      `${JSON.stringify(
+        collectAggregatedSample(id, size, factory, {
+          warmupFixtures,
+          minimumSampleDurationMs: aggregateMs,
+        }),
+      )}\n`,
+    );
   } else {
     const memoryIndex = process.argv.indexOf("--memory");
     if (memoryIndex >= 0) {

bench/src/full-engine-bench.ts

@@ -1,5 +1,6 @@
 import { createHash } from "node:crypto";
 import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
 import { brotliCompressSync, constants } from "node:zlib";
 import { initSheetwrite } from "@sheetwrite/core";
 import * as formulas from "@sheetwrite/formulas";
@@ -11,6 +12,7 @@ import {
 } from "./formula-bench.js";
 import { validateExactMatrix, validateRawStat } from "./gate-protocol.js";
 import { type ProtocolCaptureMeta, protocolCaptureMeta } from "./protocol-meta.js";
+import { resultsDirectory } from "./results-dir.js";
 import { type Stat, summarize } from "./stats.js";
 
 const ENGINE_IDS = ["default", "full"] as const;
@@ -260,6 +262,8 @@ function analysisFixture(formula: string) {
 }
 async function capture(): Promise<void> {
   const meta = protocolCaptureMeta();
+  const args = process.argv.slice(2).filter((argument) => argument !== "--");
+  const resultsDir = resultsDirectory(args);
   const engines: EngineCapture[] = [];
   for (const engine of ENGINE_IDS) {
     // Keep tracked results unchanged until both engines and all checks finish.
@@ -336,12 +340,12 @@ async function capture(): Promise<void> {
   validateFullEngineResult(result);
   for (const capture of engines) {
     await Bun.write(
-      new URL(`../results/formula-${capture.engine}-results.json`, import.meta.url),
+      resolve(resultsDir, `formula-${capture.engine}-results.json`),
       `${JSON.stringify(capture.shared, null, 2)}\n`,
     );
   }
   await Bun.write(
-    new URL("../results/full-engine-results.json", import.meta.url),
+    resolve(resultsDir, "full-engine-results.json"),
     `${JSON.stringify(result, null, 2)}\n`,
   );
 }

bench/src/generate-baseline.ts

@@ -12,6 +12,7 @@ import {
   computeHarnessFingerprint,
   MATRIX_IDS,
 } from "./gate-protocol.js";
+import { sourceTreeDirty } from "./protocol-meta.js";
 import type { RenderBenchmarkArtifact } from "./render-protocol.js";
 import { parseRenderArtifact } from "./render-protocol.js";
 
@@ -79,7 +80,7 @@ export async function generateBaseline(args: readonly string[]): Promise<string>
   if (diagnostic && writeBaseline) {
     throw new Error("--diagnostic can produce review candidates only, not an approved baseline");
   }
-  const dirty = gitOutput(["status", "--porcelain"]).length > 0;
+  const dirty = sourceTreeDirty(true);
   if (dirty && !diagnostic) {
     throw new Error(
       "baseline generation refuses a dirty tree; use --diagnostic for a non-approved candidate",