From 0d5d663d7a41a8b249a834c2255ffb0e44a3fc5c Mon Sep 17 00:00:00 2001 From: chh-ay Date: Wed, 7 Oct 2026 00:38:58 +0700 Subject: [PATCH 1/5] bench: measure current interaction metrics against release ceilings The performance showcase's decision record reads results/interaction-results.json, but that artifact was a one-off before/after pair of the 0.3.0 comparison with no producer: bench:release never refreshed it, and its improvement-ratio gates cannot be re-measured at a later release. bench:interaction now measures the current values of the four published metrics on the current build - packed inverse-index lookup latency and backing bytes, retained bytes after 100 distant edits, and the cold-route long task owned by Sheetwrite - in fresh worker processes and a fresh browser context, and records raw samples, their summary, runner provenance, and the method. Schema version 2 gates those metrics on absolute release ceilings, so no baseline build is re-measured and no old sample is carried forward. Cold-route ownership is reconstructed from a V8 CPU profile inside each long task and stated in the artifact. bench:release and its smoke rehearsal now capture the interaction and paged artifacts, and the bench and guide documentation describe that set. --- bench/README.md | 19 +- bench/package.json | 2 + bench/src/interaction-bench.ts | 577 ++++++++++++++++++ bench/src/interaction-gate.ts | 517 ++++++++++------ bench/src/paged-bench.ts | 12 + bench/src/release-capture.ts | 21 + bench/test/interaction-gate.test.ts | 337 +++++++++- bench/test/paged-gate.test.ts | 14 + .../docs/guides/performance-resources.md | 14 + 9 files changed, 1305 insertions(+), 208 deletions(-) create mode 100644 bench/src/interaction-bench.ts diff --git a/bench/README.md b/bench/README.md index 06543877..e87a367b 100644 --- a/bench/README.md +++ b/bench/README.md @@ -234,8 +234,11 @@ bun run bench:core-paths The evidence page at `docs/src/content/docs/guides/performance-resources.md` publishes every -artifact above. One command captures that whole set, in order, from a clean -tree. Use one pinned CPU on an otherwise idle machine, with no concurrent +artifact above. `bench:release` captures render-scale, the interaction +evidence, paged storage, data, core paths, formula, formula engines, matched +engines, and XLSX, then records the render regression baseline last. Every +capture stamps its artifact with the commit, tree state, and timestamp. +Use one pinned CPU on an otherwise idle machine, with no concurrent builds or timing captures. The example below uses CPU 4 on Linux: ```sh @@ -258,6 +261,18 @@ matrices of the suites that have one, writing to `results/smoke/` (ignored by git) so a rehearsal never overwrites published evidence. Regenerate the page after a release capture with `bun run docs:generate`. +The performance showcase reads `results/paged-results.json` and +`results/interaction-results.json`. `bench:paged` writes the paged capture; +`bench:interaction` measures the current values of the four interaction metrics +on the current build — packed inverse-index lookup latency, inverse-index +backing bytes, retained bytes after 100 distant edits, and the cold-route long +task owned by Sheetwrite — with raw samples, their summary, and the runner. +Those four metrics are held to the absolute release ceilings recorded in +`src/interaction-gate.ts`, not to a comparison with another build: the version 1 +artifact was a one-off before/after pair, and a release cannot re-measure a +baseline build. `bench:release` refreshes both captures, and +`bench:interaction:smoke` rehearses the interaction one. + A change to a harness source file, to the sampling flags, or to the protocol version invalidates the committed render baseline: `bench:check` compares the harness hashes, the declared sampling, and the runner (OS, CPU, Bun, Node, diff --git a/bench/package.json b/bench/package.json index f70f016a..33a7a05d 100644 --- a/bench/package.json +++ b/bench/package.json @@ -11,6 +11,8 @@ "bench:range:smoke": "bun run src/range-bench.ts --smoke", "bench:paged": "bun run src/paged-bench.ts", "bench:paged:smoke": "bun run src/paged-bench.ts --smoke", + "bench:interaction": "bun run src/interaction-bench.ts", + "bench:interaction:smoke": "bun run src/interaction-bench.ts --smoke", "bench:core-paths": "bun run src/core-paths-bench.ts --output results/core-paths-results.json", "bench:core-paths:smoke": "bun run src/core-paths-bench.ts --smoke", "bench:release": "bun run src/release-capture.ts", diff --git a/bench/src/interaction-bench.ts b/bench/src/interaction-bench.ts new file mode 100644 index 00000000..173a27c8 --- /dev/null +++ b/bench/src/interaction-bench.ts @@ -0,0 +1,577 @@ +import { cpus } from "node:os"; +import { resolve } from "node:path"; +import { type Browser, chromium, type Page } from "@playwright/test"; +import { + INTERACTION_CEILING_KEYS, + INTERACTION_CEILINGS, + INTERACTION_LOOKUPS_PER_RUN, + INTERACTION_MATRIX_ID, + INTERACTION_METRIC_LABELS, + INTERACTION_PROTOCOL_VERSION, + INTERACTION_ROWS, + INTERACTION_SAMPLE_COUNTS, + INTERACTION_SAMPLE_KEYS, + type InteractionMetric, + type InteractionPerformanceArtifact, + type InteractionPreviousCapture, + type InteractionSamples, + type InteractionValues, + validateInteractionArtifact, +} from "./interaction-gate.js"; +import { protocolCaptureMeta } from "./protocol-meta.js"; +import { summarize } from "./stats.js"; + +/** Fields of the showcase page this capture drives; installed by the route. */ +interface InteractionGridProbe { + readonly store: { + readonly getPagedStats?: (sheet: string) => { readonly loadedCells: number } | null | undefined; + }; +} + +declare global { + interface Window { + __sheetwriteScaleGrid?: InteractionGridProbe; + } +} + +const BENCH_ROOT = resolve(import.meta.dir, ".."); +const DOCS_ROOT = resolve(BENCH_ROOT, "../docs"); +const DEFAULT_OUTPUT = "results/interaction-results.json"; +const ROUTE_PATH = "/showcases/performance/"; +const DOCS_PORT = 5197; +const SERVER_READY_TIMEOUT_MS = 120_000; +const SERVER_RETRY_MS = 250; +const USABLE_TIMEOUT_MS = 60_000; +/** RAIL long-task threshold: a task at or above this block a frame. */ +const LONG_TASK_MS = 50; +const TRACE_MICROSECONDS_PER_MS = 1_000; +const TRACE_CATEGORIES = + "devtools.timeline,disabled-by-default-devtools.timeline,disabled-by-default-v8.cpu_profiler"; +/** Lookup batches the packed view-index worker runs per sample. */ +const LOOKUP_BATCHES = 128; +const LOOKUP_BATCH_SIZE = 4_096; +/** Distant edits the paged `dirty-100` probe applies per sample. */ +const DISTANT_EDITS = 100; +/** Sheetwrite package frames decide ownership; the application shell is other work. */ +const OWNED_PACKAGE_PATHS = /\/packages\//; + +/** The V8 CPU profile payload a trace `ProfileChunk` event carries. */ +export interface TraceProfile { + /** Trace-clock start of this profile, set by the opening `Profile` event. */ + readonly startTime: number | undefined; + readonly nodes: readonly { readonly id: number; readonly url: string | undefined }[]; + readonly samples: readonly number[]; + readonly timeDeltas: readonly number[]; +} + +/** One DevTools protocol trace event, as far as attribution reads it. */ +export interface TraceEvent { + readonly name: string; + readonly ph: string; + readonly ts: number; + readonly dur: number | undefined; + readonly pid: number; + readonly tid: number; + readonly url: string | undefined; + readonly profile: TraceProfile | undefined; +} + +interface ColdRouteSample { + readonly ownedMs: number; + readonly unattributedMs: number; + readonly usableMs: number; +} + +interface ProfileSample { + readonly ts: number; + readonly pid: number; + readonly tid: number; + readonly url: string | undefined; +} + +function record(value: unknown, path: string): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError(`${path} must be an object`); + } + return value as Record; +} + +function finite(value: unknown, path: string, minimum = Number.NEGATIVE_INFINITY): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < minimum) { + throw new TypeError(`${path} must be a finite number >= ${minimum}`); + } + return value; +} + +/** Run one isolated worker process and read the artifact it prints last. */ +function workerArtifact(command: readonly string[]): unknown { + const child = Bun.spawnSync([process.execPath, "run", ...command], { + cwd: BENCH_ROOT, + stdout: "pipe", + stderr: "pipe", + }); + if (child.exitCode !== 0) { + throw new Error( + `interaction worker ${command.join(" ")} exited ${child.exitCode}: ${child.stderr.toString().trim()}`, + ); + } + const lastLine = child.stdout.toString().trim().split("\n").at(-1); + if (lastLine === undefined) { + throw new Error(`interaction worker ${command.join(" ")} produced no artifact`); + } + return JSON.parse(lastLine) as unknown; +} + +/** + * One packed inverse-index lookup sample. The worker owns the lookup protocol: + * one million rows, {@link LOOKUP_BATCHES} batches of {@link LOOKUP_BATCH_SIZE} + * deterministic lookups, reported in nanoseconds per lookup. + */ +function lookupSample(): { medianNs: number; p95Ns: number; inverseIndexBytes: number } { + const value = record( + workerArtifact(["src/view-index-bench.ts", "--worker", "packed"]), + "view-index worker artifact", + ); + const inverseIndexBytes = finite( + value.actualBackingBytes, + "view-index worker artifact.actualBackingBytes", + 0, + ); + return { + medianNs: finite(value.lookupMedianNs, "view-index worker artifact.lookupMedianNs", 0), + p95Ns: finite(value.lookupP95Ns, "view-index worker artifact.lookupP95Ns", 0), + inverseIndexBytes, + }; +} + +/** + * One sparse dirty-state sample: the paged `dirty-100` probe in its own + * process, applying {@link DISTANT_EDITS} distant edits without loading a clean + * page, and reporting the clean plus sparse dirty allocation it retains. + */ +function dirtySample(): number { + const value = record( + workerArtifact([ + "src/paged-bench.ts", + "--probe", + "dirty-100", + "--rows", + String(INTERACTION_ROWS), + ]), + "paged dirty-100 probe", + ); + if (value.scenario !== "dirty-100") { + throw new TypeError("paged dirty-100 probe reported another scenario"); + } + if (value.dirtyCells !== DISTANT_EDITS) { + throw new TypeError(`paged dirty-100 probe retained ${String(value.dirtyCells)} dirty cells`); + } + return finite(value.retainedBytes, "paged dirty-100 probe.retainedBytes", 0); +} + +function traceEventOf(value: unknown): TraceEvent | null { + if (typeof value !== "object" || value === null) return null; + if (!("name" in value && "ph" in value && "ts" in value && "pid" in value && "tid" in value)) { + return null; + } + const { name, ph, ts, pid, tid } = value; + if ( + typeof name !== "string" || + typeof ph !== "string" || + typeof ts !== "number" || + typeof pid !== "number" || + typeof tid !== "number" + ) { + return null; + } + const dur = "dur" in value && typeof value.dur === "number" ? value.dur : undefined; + const data = traceEventData(value); + return { name, ph, ts, dur, pid, tid, url: urlOf(data), profile: profileOf(data) }; +} + +function traceEventData(value: object): Record | undefined { + if (!("args" in value) || typeof value.args !== "object" || value.args === null) return undefined; + if (!("data" in value.args)) return undefined; + const data = value.args.data; + return typeof data === "object" && data !== null ? (data as Record) : undefined; +} + +function urlOf(data: Record | undefined): string | undefined { + if (data === undefined || !("url" in data)) return undefined; + return typeof data.url === "string" ? data.url : undefined; +} + +function profileOf(data: Record | undefined): TraceProfile | undefined { + if (data === undefined) return undefined; + const startTime = typeof data.startTime === "number" ? data.startTime : undefined; + const raw = data.cpuProfile; + if (typeof raw !== "object" || raw === null) return undefined; + return { + startTime, + nodes: profileNodesOf(raw), + samples: numbersOf("samples" in raw ? raw.samples : undefined), + timeDeltas: numbersOf( + "timeDeltas" in raw && Array.isArray(raw.timeDeltas) ? raw.timeDeltas : data.timeDeltas, + ), + }; +} + +function profileNodesOf(profile: object): TraceProfile["nodes"] { + if (!("nodes" in profile) || !Array.isArray(profile.nodes)) return []; + const nodes: Array<{ id: number; url: string | undefined }> = []; + for (const node of profile.nodes) { + if (typeof node !== "object" || node === null || !("id" in node)) continue; + if (typeof node.id !== "number") continue; + const frame = "callFrame" in node && typeof node.callFrame === "object" ? node.callFrame : null; + const url = + frame !== null && "url" in frame && typeof frame.url === "string" ? frame.url : undefined; + nodes.push({ id: node.id, url }); + } + return nodes; +} + +function numbersOf(value: unknown): number[] { + if (!Array.isArray(value)) return []; + return value.filter((item): item is number => typeof item === "number"); +} + +/** One CPU profile sample per rendered frame, on the trace clock. */ +function profileSamplesOf(events: readonly TraceEvent[]): ProfileSample[] { + const urls = new Map(); + for (const event of events) { + for (const node of event.profile?.nodes ?? []) urls.set(node.id, node.url); + } + const samples: ProfileSample[] = []; + const clocks = new Map(); + for (const event of events) { + const profile = event.profile; + if (profile === undefined || profile.samples.length === 0) continue; + const thread = `${event.pid}:${event.tid}`; + let at = profile.startTime ?? clocks.get(thread) ?? event.ts; + for (const [index, nodeId] of profile.samples.entries()) { + at += profile.timeDeltas[index] ?? 0; + samples.push({ ts: at, pid: event.pid, tid: event.tid, url: urls.get(nodeId) }); + } + clocks.set(thread, at); + } + return samples; +} + +/** + * Attribute the longest cold-route tasks to Sheetwrite or to everyone else. + * + * The version 1 capture shipped no producer, so this reconstruction is stated + * in the artifact's `method.attribution`. A task is owned when it reaches the + * long-task threshold and Sheetwrite package frames account for at least half + * of the CPU profiled inside it; the whole task duration is then charged to + * Sheetwrite. Tasks below that share, tasks that only ran other code, and + * tasks with no profile coverage are reported as unattributed and are never + * counted as owned work. + */ +export function coldTaskDurations(events: readonly TraceEvent[]): { + ownedMs: number; + unattributedMs: number; +} { + const samples = profileSamplesOf(events); + const longTasks = events.filter( + (event) => + event.name === "RunTask" && + event.ph === "X" && + (event.dur ?? 0) >= LONG_TASK_MS * TRACE_MICROSECONDS_PER_MS, + ); + let ownedMs = 0; + let unattributedMs = 0; + for (const task of longTasks) { + const duration = task.dur ?? 0; + const end = task.ts + duration; + const inside = samples.filter( + (sample) => + sample.pid === task.pid && + sample.tid === task.tid && + sample.ts >= task.ts && + sample.ts < end, + ); + let profiled = 0; + let owned = 0; + for (const [index, sample] of inside.entries()) { + const next = inside[index + 1]; + const step = Math.min(next?.ts ?? end, end) - sample.ts; + if (step <= 0) continue; + profiled += step; + if (OWNED_PACKAGE_PATHS.test(sample.url ?? "")) owned += step; + } + const durationMs = duration / TRACE_MICROSECONDS_PER_MS; + if (profiled > 0 && owned * 2 >= profiled) ownedMs = Math.max(ownedMs, durationMs); + else unattributedMs = Math.max(unattributedMs, durationMs); + } + return { ownedMs, unattributedMs }; +} + +function traceEventsOf(events: TraceEvent[], value: unknown): void { + if (!Array.isArray(value)) return; + for (const entry of value) { + const event = traceEventOf(entry); + if (event !== null) events.push(event); + } +} + +async function coldRouteSample(browser: Browser, route: string): Promise { + const context = await browser.newContext(); + try { + const page = await context.newPage(); + const session = await context.newCDPSession(page); + const events: TraceEvent[] = []; + session.on("Tracing.dataCollected", ({ value }) => traceEventsOf(events, value)); + const tracingComplete = new Promise((resolveComplete) => + session.once("Tracing.tracingComplete", () => resolveComplete()), + ); + await session.send("Tracing.start", { + categories: TRACE_CATEGORIES, + transferMode: "ReportEvents", + }); + const usableMs = await loadUsableRoute(page, route); + await session.send("Tracing.end"); + await tracingComplete; + return { ...coldTaskDurations(events), usableMs }; + } finally { + await context.close(); + } +} + +/** + * One untraced load before the samples. The development server transforms the + * route graph on first request; warming it keeps the samples on the browser's + * own cold work, and every sample still starts from an empty context, cache, + * and code cache. + */ +async function warmRoute(browser: Browser, route: string): Promise { + const context = await browser.newContext(); + try { + await loadUsableRoute(await context.newPage(), route); + } finally { + await context.close(); + } +} + +/** Navigate, prove the route renders, and wait until its first page is resident. */ +async function loadUsableRoute(page: Page, route: string): Promise { + const response = await page.goto(route); + if (response === null || !response.ok()) { + throw new Error( + `cold route answered ${response?.status() ?? "no response"}; the showcase route must render before it can be measured`, + ); + } + // The first rectangular page of the feed must be resident in the public + // paged store: the same readiness signal the showcase browser test uses. + await page.waitForFunction( + () => (window.__sheetwriteScaleGrid?.store.getPagedStats?.("scale")?.loadedCells ?? 0) > 0, + undefined, + { timeout: USABLE_TIMEOUT_MS }, + ); + return page.evaluate(() => performance.now()); +} + +async function waitForRoute(route: string): Promise { + const deadline = Date.now() + SERVER_READY_TIMEOUT_MS; + let refusal: string | undefined; + while (Date.now() < deadline) { + try { + const response = await fetch(route); + if (response.ok) return; + throw new Error( + `cold route answered ${response.status} before the first sample; the showcase route does not render on this tree`, + ); + } catch (error) { + if (!(error instanceof TypeError)) throw error; + // Connection refused while Vite starts; the deadline reports a real failure. + refusal = error.message; + } + await Bun.sleep(SERVER_RETRY_MS); + } + throw new Error(`interaction docs server did not become ready on ${route}: ${refusal ?? ""}`); +} + +/** The capture already at this path, when it is a capture of this schema. */ +async function readPreviousCapture(path: string): Promise { + const file = Bun.file(path); + if (!(await file.exists())) return null; + try { + return previousCaptureOf(await file.json()); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write( + `interaction capture: the artifact at ${path} is not a comparable capture (${message}); recording no previous values\n`, + ); + return null; + } +} + +/** + * Read the medians of a capture of this schema. The predecessor is not gated + * again here: a capture that breached a ceiling is still the honest reference + * for how the current numbers moved, and only its raw samples are read. + */ +function previousCaptureOf(value: unknown): InteractionPreviousCapture { + const input = record(value, "previous capture"); + if ( + input.protocolVersion !== INTERACTION_PROTOCOL_VERSION || + input.matrixId !== INTERACTION_MATRIX_ID + ) { + throw new TypeError(`expected ${INTERACTION_MATRIX_ID}`); + } + const mode = input.mode; + if (mode !== "full" && mode !== "smoke") throw new TypeError("previous capture mode is unknown"); + const inputSamples = record(input.samples, "previous capture.samples"); + const expected = INTERACTION_SAMPLE_COUNTS[mode]; + const samples = {} as Record; + for (const key of INTERACTION_SAMPLE_KEYS) { + const raw = inputSamples[key]; + if (!Array.isArray(raw) || raw.length !== expected) { + throw new TypeError(`previous capture.samples.${key} needs ${expected} raw samples`); + } + samples[key] = raw.map((sample, index) => + finite(sample, `previous capture.samples.${key}[${index}]`, 0), + ); + } + const commit = input.metadata; + if (typeof commit !== "object" || commit === null || !("commit" in commit)) { + throw new TypeError("previous capture.metadata.commit is missing"); + } + if (typeof commit.commit !== "string" || commit.commit.length === 0) { + throw new TypeError("previous capture.metadata.commit must be a non-empty string"); + } + const timestamp = "timestamp" in commit ? commit.timestamp : undefined; + if (typeof timestamp !== "string" || !Number.isFinite(Date.parse(timestamp))) { + throw new TypeError("previous capture.metadata.timestamp must be an ISO timestamp"); + } + return { + matrixId: INTERACTION_MATRIX_ID, + mode, + commit: commit.commit, + timestamp, + values: summarizeSamples(samples), + }; +} + +function summarizeSamples(samples: InteractionSamples): InteractionValues { + return { + lookupMedianNs: summarize(samples.lookupMedianNs), + lookupP95Ns: summarize(samples.lookupP95Ns), + inverseIndexBytes: summarize(samples.inverseIndexBytes), + dirty100Bytes: summarize(samples.dirty100Bytes), + ownedColdLongTaskMs: summarize(samples.ownedColdLongTaskMs), + unattributedColdLongTaskMs: summarize(samples.unattributedColdLongTaskMs), + usableMs: summarize(samples.usableMs), + }; +} + +function methodNotes(): InteractionPerformanceArtifact["method"] { + return { + lookup: `Fresh packed view-index worker process per sample: ${INTERACTION_ROWS.toLocaleString("en-US")} rows of one column, ${LOOKUP_BATCHES} batches of ${LOOKUP_BATCH_SIZE.toLocaleString("en-US")} deterministic lookups (${INTERACTION_LOOKUPS_PER_RUN.toLocaleString("en-US")} lookups), nanoseconds per lookup measured over the batches. inverseIndexBytes is the inverse view-index backing store of that worker.`, + dirty: `Fresh paged dirty-100 probe process per sample: ${INTERACTION_ROWS.toLocaleString("en-US")} rows, ${DISTANT_EDITS} distant edits, no clean page loaded. dirty100Bytes is allocated plus sparse dirty bytes retained.`, + coldRoute: `Current Vite development route ${ROUTE_PATH}, rendered once before sampling so the development server's transform cache is warm, then loaded in a fresh browser context per sample with no shared cache, code cache, or service worker state. Server-side rendering answers the route before the first sample, and a sample measures the browser, not the first server transform; this is a development-server measurement, not a production bundle. usableMs is the page clock when the first rectangular page of feed cells became resident in the public paged store, an upper bound that includes the readiness poll.`, + attribution: `Chrome DevTools protocol tracing (${TRACE_CATEGORIES}) from before navigation until the page is usable, in the renderer process: ${LONG_TASK_MS} ms or longer RunTask events are long tasks, and the V8 CPU profile on the same clock says what ran inside them. A task is owned by Sheetwrite when frames served from packages account for at least half of the CPU profiled inside it, and its complete duration is then recorded as the owned long task. Tasks below that share, tasks running only other code, and tasks the profile does not cover are reported as unattributed, so a partly owned task is never counted as owned work. A zero sample means no owned long task was observed.`, + previous: `Same-schema comparison only. When the artifact already at this path is a capture of ${INTERACTION_MATRIX_ID}, previous records that capture's identity and its per-metric medians, so a reader can state the change without inventing a baseline. A first capture, or an unreadable predecessor, records null.`, + }; +} + +async function main(): Promise { + const args = process.argv.slice(2).filter((argument) => argument !== "--"); + const mode = args.includes("--smoke") ? "smoke" : "full"; + const outputIndex = args.indexOf("--output"); + const outputArgument = outputIndex >= 0 ? args[outputIndex + 1] : undefined; + if (outputIndex >= 0 && (outputArgument === undefined || outputArgument.startsWith("--"))) { + throw new Error("interaction capture needs a path after --output"); + } + const outputPath = resolve(BENCH_ROOT, outputArgument ?? DEFAULT_OUTPUT); + const route = `http://127.0.0.1:${DOCS_PORT}${ROUTE_PATH}`; + const samples = INTERACTION_SAMPLE_COUNTS[mode]; + + const previous = await readPreviousCapture(outputPath); + const server = Bun.spawn( + [ + process.execPath, + "run", + "dev", + "--host", + "127.0.0.1", + "--port", + String(DOCS_PORT), + "--strictPort", + ], + { cwd: DOCS_ROOT, stdout: "ignore", stderr: "inherit" }, + ); + const browser = await chromium.launch(); + const collected: Record = { + lookupMedianNs: [], + lookupP95Ns: [], + inverseIndexBytes: [], + dirty100Bytes: [], + ownedColdLongTaskMs: [], + unattributedColdLongTaskMs: [], + usableMs: [], + }; + try { + await waitForRoute(route); + await warmRoute(browser, route); + for (let sample = 0; sample < samples; sample++) { + const lookup = lookupSample(); + collected.lookupMedianNs.push(lookup.medianNs); + collected.lookupP95Ns.push(lookup.p95Ns); + collected.inverseIndexBytes.push(lookup.inverseIndexBytes); + collected.dirty100Bytes.push(dirtySample()); + const cold = await coldRouteSample(browser, route); + collected.ownedColdLongTaskMs.push(cold.ownedMs); + collected.unattributedColdLongTaskMs.push(cold.unattributedMs); + collected.usableMs.push(cold.usableMs); + process.stderr.write( + `sample ${sample + 1}/${samples}: lookup ${lookup.medianNs.toFixed(1)} ns, inverse ${lookup.inverseIndexBytes} B, dirty100 ${collected.dirty100Bytes[sample]} B, owned cold long task ${cold.ownedMs} ms, usable ${cold.usableMs.toFixed(1)} ms\n`, + ); + } + } finally { + await browser.close(); + server.kill(); + await server.exited; + } + + const values = summarizeSamples(collected); + const artifact: InteractionPerformanceArtifact = { + protocolVersion: INTERACTION_PROTOCOL_VERSION, + matrixId: INTERACTION_MATRIX_ID, + mode, + metadata: protocolCaptureMeta(), + runner: { + runtime: `Bun ${Bun.version} on ${process.platform} ${process.arch}`, + browser: browser.version(), + cpu: cpus()[0]?.model ?? "unknown", + }, + method: methodNotes(), + rows: INTERACTION_ROWS, + lookupsPerRun: INTERACTION_LOOKUPS_PER_RUN, + samples: collected, + values, + previous, + ceilings: INTERACTION_CEILINGS, + }; + validateInteractionArtifact(artifact); + await Bun.write(outputPath, `${JSON.stringify(artifact, null, 2)}\n`); + process.stderr.write( + [ + "", + `interaction ${mode} capture written to ${outputPath} over ${samples} samples; every release ceiling passed`, + ...INTERACTION_SAMPLE_KEYS.map((key) => { + const ceiling = INTERACTION_CEILING_KEYS.find((candidate) => candidate === key); + const limit = + ceiling === undefined ? "" : ` (release ceiling ${INTERACTION_CEILINGS[ceiling]})`; + return ` ${INTERACTION_METRIC_LABELS[key]}: median ${values[key].median}${limit}`; + }), + previous === null + ? " previous capture: none comparable at this path" + : ` previous capture: ${previous.matrixId} ${previous.mode} at ${previous.commit.slice(0, 12)} (${previous.timestamp})`, + "", + ].join("\n"), + ); + process.stdout.write(`${JSON.stringify(artifact)}\n`); +} + +if (import.meta.main) await main(); diff --git a/bench/src/interaction-gate.ts b/bench/src/interaction-gate.ts index b2288636..03c97232 100644 --- a/bench/src/interaction-gate.ts +++ b/bench/src/interaction-gate.ts @@ -1,206 +1,359 @@ +import type { BenchmarkMode } from "./gate-protocol.js"; +import { validateRawStat, validateStat } from "./gate-protocol.js"; +import type { ProtocolCaptureMeta } from "./protocol-meta.js"; +import type { Stat } from "./stats.js"; + +/** + * Schema of the performance showcase's interaction evidence. + * + * Version 1 recorded a before/after pair from a one-off comparison capture and + * gated the improvement ratios of that pair. A ratio gate cannot be refreshed + * by a release: re-measuring the "before" build is a separate protocol, and a + * copied forward old sample would be a fabricated input. Version 2 therefore + * records the current values of the four published metrics with their raw + * samples and provenance, and gates them on absolute release ceilings. + */ +export const INTERACTION_PROTOCOL_VERSION = 2 as const; +export const INTERACTION_MATRIX_ID = "interaction-current-v2" as const; +export const INTERACTION_ROWS = 1_000_000; +export const INTERACTION_LOOKUPS_PER_RUN = 524_288; +/** Raw samples per matrix: the published matrix asks for five, a rehearsal two. */ +export const INTERACTION_SAMPLE_COUNTS: Readonly> = { + full: 5, + smoke: 2, +}; + +export const INTERACTION_SAMPLE_KEYS = [ + "lookupMedianNs", + "lookupP95Ns", + "inverseIndexBytes", + "dirty100Bytes", + "ownedColdLongTaskMs", + "unattributedColdLongTaskMs", + "usableMs", +] as const; +export type InteractionMetric = (typeof INTERACTION_SAMPLE_KEYS)[number]; +/** Metrics the release refuses to exceed; the rest are reported, never gated. */ +export const INTERACTION_CEILING_KEYS = [ + "lookupP95Ns", + "inverseIndexBytes", + "dirty100Bytes", + "ownedColdLongTaskMs", +] as const; +export type InteractionCeilingMetric = (typeof INTERACTION_CEILING_KEYS)[number]; + +export type InteractionSamples = { readonly [K in InteractionMetric]: readonly number[] }; +export type InteractionValues = { readonly [K in InteractionMetric]: Stat }; + +/** + * Absolute release ceilings, not comparisons against another build. The lookup + * ceiling applies to the median of the sample p95s, because one batch of a + * sub-microsecond measurement can always be disturbed; the byte and long-task + * ceilings apply to every sample. + * + * - `lookupP95Ns`: a packed inverse-index lookup at one million rows must stay + * under one microsecond at its 95th percentile. + * - `inverseIndexBytes`: the packed inverse index may use at most four bytes + * per data row, the Uint32 view-row-per-data-row bound the view-index + * protocol asserts. + * - `dirty100Bytes`: a hundred distant edits must stay inside the one mebibyte + * sparse overlay budget the paged protocol asserts for `dirty-100`. + * - `ownedColdLongTaskMs`: no long task owned by Sheetwrite on the cold route + * may reach the 50 ms long-task threshold, the freeze budget the version 1 + * capture stated. + */ +export const INTERACTION_CEILINGS: Readonly> = { + lookupP95Ns: 1_000, + inverseIndexBytes: INTERACTION_ROWS * 4, + dirty100Bytes: 1024 * 1024, + ownedColdLongTaskMs: 50, +}; + +/** Ceilings every raw sample must meet, as opposed to a summary of the samples. */ +export const INTERACTION_SAMPLE_CEILING_KEYS = [ + "inverseIndexBytes", + "dirty100Bytes", + "ownedColdLongTaskMs", +] as const satisfies readonly InteractionCeilingMetric[]; + +/** Runner identity of a capture: enough to judge whether a number is comparable. */ +export interface InteractionRunner { + readonly runtime: string; + readonly browser: string; + readonly cpu: string; +} + +/** + * How each metric was produced. Cold-route attribution is reconstructed rather + * than inherited: the version 1 capture shipped its numbers without a producer, + * so this record states exactly what the current numbers mean. + */ +export interface InteractionMethod { + readonly lookup: string; + readonly dirty: string; + readonly coldRoute: string; + readonly attribution: string; + readonly previous: string; +} + +/** The capture this artifact was written over, when one existed at its path. */ +export interface InteractionPreviousCapture { + readonly matrixId: typeof INTERACTION_MATRIX_ID; + readonly mode: BenchmarkMode; + readonly commit: string; + readonly timestamp: string; + readonly values: InteractionValues; +} + export interface InteractionPerformanceArtifact { - protocolVersion: number; - matrixId: string; - capturedAt: string; - source: { - commit: string; - workingTree: boolean; - runtime: string; - browser: string; - cpu: string; - samples: number; - warmup: string; - correctness: string; + readonly protocolVersion: typeof INTERACTION_PROTOCOL_VERSION; + readonly matrixId: typeof INTERACTION_MATRIX_ID; + readonly mode: BenchmarkMode; + readonly metadata: ProtocolCaptureMeta; + readonly runner: InteractionRunner; + readonly method: InteractionMethod; + readonly rows: typeof INTERACTION_ROWS; + readonly lookupsPerRun: typeof INTERACTION_LOOKUPS_PER_RUN; + readonly samples: InteractionSamples; + readonly values: InteractionValues; + readonly previous: InteractionPreviousCapture | null; + readonly ceilings: Readonly>; +} + +export const INTERACTION_METRIC_LABELS: Readonly> = { + lookupMedianNs: "packed inverse-index lookup, median (ns)", + lookupP95Ns: "packed inverse-index lookup, p95 (ns)", + inverseIndexBytes: "packed inverse-index backing store (bytes)", + dirty100Bytes: "retained bytes after 100 distant edits", + ownedColdLongTaskMs: "owned cold-route long task (ms)", + unattributedColdLongTaskMs: "reported unattributed cold-route long task (ms)", + usableMs: "cold route to first resolved feed cell (ms)", +}; + +function record(value: unknown, path: string): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError(`${path} must be an object`); + } + return value as Record; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new TypeError(`${path} must be a non-empty string`); + } + return value; +} + +function finite(value: unknown, path: string, minimum = Number.NEGATIVE_INFINITY): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < minimum) { + throw new TypeError(`${path} must be a finite number >= ${minimum}`); + } + return value; +} + +function bool(value: unknown, path: string): boolean { + if (typeof value !== "boolean") throw new TypeError(`${path} must be a boolean`); + return value; +} + +function modeOf(value: unknown, path: string): BenchmarkMode { + if (value !== "full" && value !== "smoke") { + throw new TypeError(`${path} must be full or smoke`); + } + return value; +} + +function timestampOf(value: unknown, path: string): string { + const timestamp = text(value, path); + if (!Number.isFinite(Date.parse(timestamp))) { + throw new TypeError(`${path} must be an ISO timestamp`); + } + return timestamp; +} + +function samplesOf(value: unknown, mode: BenchmarkMode): InteractionSamples { + const input = record(value, "interaction artifact.samples"); + const expected = INTERACTION_SAMPLE_COUNTS[mode]; + const samplesOfKey = (key: InteractionMetric): readonly number[] => { + const raw = input[key]; + if (!Array.isArray(raw) || raw.length !== expected) { + throw new TypeError(`interaction artifact.samples.${key} requires ${expected} raw samples`); + } + return raw.map((sample, index) => + finite(sample, `interaction artifact.samples.${key}[${index}]`, 0), + ); }; - directionalPrefetch: { - sourceLatencyMs: number; - frameBudgetMs: number; - residencyRatioSamples: number[]; - p95VisibleWaitMsSamples: number[]; - requestedRowMultiplierSamples: number[]; - medianResidencyRatio: number; - medianP95VisibleWaitMs: number; + return { + lookupMedianNs: samplesOfKey("lookupMedianNs"), + lookupP95Ns: samplesOfKey("lookupP95Ns"), + inverseIndexBytes: samplesOfKey("inverseIndexBytes"), + dirty100Bytes: samplesOfKey("dirty100Bytes"), + ownedColdLongTaskMs: samplesOfKey("ownedColdLongTaskMs"), + unattributedColdLongTaskMs: samplesOfKey("unattributedColdLongTaskMs"), + usableMs: samplesOfKey("usableMs"), }; - viewIndex: { - rows: number; - lookupsPerRun: number; - baseline: { - lookupMedianNsSamples: number[]; - lookupP95NsSamples: number[]; - retainedBytes: number; - }; - packed: { - lookupMedianNsSamples: number[]; - lookupP95NsSamples: number[]; - retainedBytes: number; - backingBytes: number; - }; - medianLookupImprovementRatio: number; - p95LookupRegressionRatio: number; - retainedHeapReductionRatio: number; +} + +function statOf(value: unknown, path: string): Stat { + const input = record(value, path); + const stat: Stat = { + median: finite(input.median, `${path}.median`, 0), + p95: finite(input.p95, `${path}.p95`, 0), + mean: finite(input.mean, `${path}.mean`, 0), + stddev: finite(input.stddev, `${path}.stddev`, 0), + min: finite(input.min, `${path}.min`, 0), + max: finite(input.max, `${path}.max`, 0), + iters: finite(input.iters, `${path}.iters`, 1), }; - sparseDirty: { - baseline100Bytes: number; - dirty100Bytes: number; - dirty10000Bytes: number; - dirty100ReductionRatio: number; - dirty100CleanChunks: number; - dirty10000CleanChunks: number; + if (!Number.isInteger(stat.iters)) throw new TypeError(`${path}.iters must be an integer`); + return stat; +} + +/** Every published value must be the summary of the raw samples beside it. */ +function valuesOf(value: unknown, samples: InteractionSamples): InteractionValues { + const input = record(value, "interaction artifact.values"); + const valueAt = (key: InteractionMetric): Stat => { + const stat = statOf(input[key], `interaction artifact.values.${key}`); + validateRawStat(samples[key], stat, `interaction artifact.values.${key}`); + return stat; }; - coldRoute: { - before: { usableMsSamples: number[]; sheetwriteLongTaskMsSamples: number[] }; - after: { - usableMsSamples: number[]; - sheetwriteLongTaskMsSamples: number[]; - routeCpuMsSamples: number[]; - gridBuildCpuMsSamples: number[]; - reportedUnattributedLongTaskMsSamples: number[]; - }; - medianUsableImprovementRatio: number; - sheetwriteLongTaskCeilingMs: number; - sheetwriteLongTaskGatePassed: boolean; + return { + lookupMedianNs: valueAt("lookupMedianNs"), + lookupP95Ns: valueAt("lookupP95Ns"), + inverseIndexBytes: valueAt("inverseIndexBytes"), + dirty100Bytes: valueAt("dirty100Bytes"), + ownedColdLongTaskMs: valueAt("ownedColdLongTaskMs"), + unattributedColdLongTaskMs: valueAt("unattributedColdLongTaskMs"), + usableMs: valueAt("usableMs"), }; } -function object(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); +function ceilingsOf(value: unknown): Readonly> { + const input = record(value, "interaction artifact.ceilings"); + const ceilingOf = (key: InteractionCeilingMetric): number => { + const ceiling = finite(input[key], `interaction artifact.ceilings.${key}`, 0); + if (ceiling !== INTERACTION_CEILINGS[key]) { + throw new TypeError( + `interaction artifact.ceilings.${key} is not the release ceiling ${INTERACTION_CEILINGS[key]}`, + ); + } + return ceiling; + }; + return { + lookupP95Ns: ceilingOf("lookupP95Ns"), + inverseIndexBytes: ceilingOf("inverseIndexBytes"), + dirty100Bytes: ceilingOf("dirty100Bytes"), + ownedColdLongTaskMs: ceilingOf("ownedColdLongTaskMs"), + }; } -function finiteSamples(value: unknown, count: number, path: string): asserts value is number[] { - if ( - !Array.isArray(value) || - value.length !== count || - value.some((item) => !Number.isFinite(item)) - ) { - throw new Error(`${path} requires ${count} finite raw samples`); - } +function metaOf(value: unknown): ProtocolCaptureMeta { + const input = record(value, "interaction artifact.metadata"); + return { + commit: text(input.commit, "interaction artifact.metadata.commit"), + dirty: bool(input.dirty, "interaction artifact.metadata.dirty"), + timestamp: timestampOf(input.timestamp, "interaction artifact.metadata.timestamp"), + }; } -function median(values: readonly number[]): number { - const sorted = [...values].sort((left, right) => left - right); - return sorted[Math.floor(sorted.length / 2)]!; +function runnerOf(value: unknown): InteractionRunner { + const input = record(value, "interaction artifact.runner"); + return { + runtime: text(input.runtime, "interaction artifact.runner.runtime"), + browser: text(input.browser, "interaction artifact.runner.browser"), + cpu: text(input.cpu, "interaction artifact.runner.cpu"), + }; } -function sameRatio(recorded: number, computed: number): boolean { - return Number.isFinite(recorded) && Math.abs(recorded - computed) <= 1e-12; +function methodOf(value: unknown): InteractionMethod { + const input = record(value, "interaction artifact.method"); + return { + lookup: text(input.lookup, "interaction artifact.method.lookup"), + dirty: text(input.dirty, "interaction artifact.method.dirty"), + coldRoute: text(input.coldRoute, "interaction artifact.method.coldRoute"), + attribution: text(input.attribution, "interaction artifact.method.attribution"), + previous: text(input.previous, "interaction artifact.method.previous"), + }; } -export function validateInteractionArtifact(value: unknown): InteractionPerformanceArtifact { - if (!object(value)) throw new Error("interaction artifact must be an object"); - const artifact = value as unknown as InteractionPerformanceArtifact; - if (artifact.protocolVersion !== 1 || artifact.matrixId !== "interaction-full-v1") { - throw new Error("interaction artifact protocol identity changed"); +function previousOf(value: unknown): InteractionPreviousCapture | null { + if (value === null) return null; + const input = record(value, "interaction artifact.previous"); + const matrixId = text(input.matrixId, "interaction artifact.previous.matrixId"); + if (matrixId !== INTERACTION_MATRIX_ID) { + throw new TypeError("interaction artifact.previous must be a capture of the same schema"); } - const samples = artifact.source?.samples; - if ( - samples !== 5 || - !artifact.source.commit || - !artifact.source.runtime || - !artifact.source.browser || - !artifact.source.cpu || - !artifact.source.warmup || - !artifact.capturedAt || - Number.isNaN(Date.parse(artifact.capturedAt)) - ) { - throw new Error("interaction artifact provenance is incomplete"); + const values = record(input.values, "interaction artifact.previous.values"); + const parsed = {} as Record; + for (const key of INTERACTION_SAMPLE_KEYS) { + // The predecessor's raw samples are not in this artifact, so its summary + // can only be re-checked for internal consistency, not re-derived. + const stat = statOf(values[key], `interaction artifact.previous.values.${key}`); + validateStat(stat, `interaction artifact.previous.values.${key}`); + parsed[key] = stat; } - if (!artifact.source.correctness) - throw new Error("interaction artifact lacks a correctness oracle"); - for (const [path, sample] of [ - [ - "directionalPrefetch.residencyRatioSamples", - artifact.directionalPrefetch?.residencyRatioSamples, - ], - [ - "directionalPrefetch.p95VisibleWaitMsSamples", - artifact.directionalPrefetch?.p95VisibleWaitMsSamples, - ], - [ - "directionalPrefetch.requestedRowMultiplierSamples", - artifact.directionalPrefetch?.requestedRowMultiplierSamples, - ], - [ - "viewIndex.baseline.lookupMedianNsSamples", - artifact.viewIndex?.baseline?.lookupMedianNsSamples, - ], - ["viewIndex.baseline.lookupP95NsSamples", artifact.viewIndex?.baseline?.lookupP95NsSamples], - ["viewIndex.packed.lookupMedianNsSamples", artifact.viewIndex?.packed?.lookupMedianNsSamples], - ["viewIndex.packed.lookupP95NsSamples", artifact.viewIndex?.packed?.lookupP95NsSamples], - ["coldRoute.before.usableMsSamples", artifact.coldRoute?.before?.usableMsSamples], - [ - "coldRoute.before.sheetwriteLongTaskMsSamples", - artifact.coldRoute?.before?.sheetwriteLongTaskMsSamples, - ], - ["coldRoute.after.usableMsSamples", artifact.coldRoute?.after?.usableMsSamples], - [ - "coldRoute.after.sheetwriteLongTaskMsSamples", - artifact.coldRoute?.after?.sheetwriteLongTaskMsSamples, - ], - ["coldRoute.after.routeCpuMsSamples", artifact.coldRoute?.after?.routeCpuMsSamples], - ["coldRoute.after.gridBuildCpuMsSamples", artifact.coldRoute?.after?.gridBuildCpuMsSamples], - [ - "coldRoute.after.reportedUnattributedLongTaskMsSamples", - artifact.coldRoute?.after?.reportedUnattributedLongTaskMsSamples, - ], - ] as const) { - finiteSamples(sample, samples, path); + return { + matrixId: INTERACTION_MATRIX_ID, + mode: modeOf(input.mode, "interaction artifact.previous.mode"), + commit: text(input.commit, "interaction artifact.previous.commit"), + timestamp: timestampOf(input.timestamp, "interaction artifact.previous.timestamp"), + values: parsed as InteractionValues, + }; +} + +/** + * Validate a published interaction artifact and return its typed value. + * Rejects a stale schema, incomplete provenance, a weakened matrix, a value + * that is not derived from its samples, or a sample above a release ceiling. + */ +export function validateInteractionArtifact(value: unknown): InteractionPerformanceArtifact { + const input = record(value, "interaction artifact"); + const protocolVersion = finite(input.protocolVersion, "interaction artifact.protocolVersion", 0); + if (protocolVersion !== INTERACTION_PROTOCOL_VERSION) { + throw new TypeError( + `interaction artifact stale protocol: expected ${INTERACTION_PROTOCOL_VERSION}, observed ${protocolVersion}`, + ); } - const residencyMedian = median(artifact.directionalPrefetch.residencyRatioSamples); - const waitMedian = median(artifact.directionalPrefetch.p95VisibleWaitMsSamples); - if ( - artifact.directionalPrefetch.sourceLatencyMs !== 90 || - !sameRatio(artifact.directionalPrefetch.medianResidencyRatio, residencyMedian) || - !sameRatio(artifact.directionalPrefetch.medianP95VisibleWaitMs, waitMedian) || - residencyMedian < 0.95 || - waitMedian >= 16.7 - ) { - throw new Error("directional prefetch gate failed"); + const matrixId = text(input.matrixId, "interaction artifact.matrixId"); + if (matrixId !== INTERACTION_MATRIX_ID) { + throw new TypeError( + `interaction artifact stale matrix: expected ${INTERACTION_MATRIX_ID}, observed ${matrixId}`, + ); } - const baselineMedian = median(artifact.viewIndex.baseline.lookupMedianNsSamples); - const packedMedian = median(artifact.viewIndex.packed.lookupMedianNsSamples); - const baselineP95 = median(artifact.viewIndex.baseline.lookupP95NsSamples); - const packedP95 = median(artifact.viewIndex.packed.lookupP95NsSamples); - const medianImprovement = 1 - packedMedian / baselineMedian; - const p95Regression = packedP95 / baselineP95 - 1; - const heapReduction = - 1 - artifact.viewIndex.packed.retainedBytes / artifact.viewIndex.baseline.retainedBytes; - if ( - artifact.viewIndex.rows !== 1_000_000 || - artifact.viewIndex.packed.backingBytes > artifact.viewIndex.rows * 4 || - !sameRatio(artifact.viewIndex.retainedHeapReductionRatio, heapReduction) || - !sameRatio(artifact.viewIndex.medianLookupImprovementRatio, medianImprovement) || - !sameRatio(artifact.viewIndex.p95LookupRegressionRatio, p95Regression) || - heapReduction < 0.5 || - medianImprovement < 0.2 || - p95Regression > 0.1 - ) { - throw new Error("packed view-index gate failed"); + const mode = modeOf(input.mode, "interaction artifact.mode"); + const rows = finite(input.rows, "interaction artifact.rows", 1); + const lookupsPerRun = finite(input.lookupsPerRun, "interaction artifact.lookupsPerRun", 1); + if (rows !== INTERACTION_ROWS || lookupsPerRun !== INTERACTION_LOOKUPS_PER_RUN) { + throw new TypeError("interaction artifact workload changed"); } - const dirtyReduction = - 1 - artifact.sparseDirty.dirty100Bytes / artifact.sparseDirty.baseline100Bytes; - if ( - artifact.sparseDirty.dirty100Bytes > 1024 * 1024 || - !sameRatio(artifact.sparseDirty.dirty100ReductionRatio, dirtyReduction) || - dirtyReduction < 0.8 || - artifact.sparseDirty.dirty100CleanChunks !== 0 || - artifact.sparseDirty.dirty10000CleanChunks !== 0 - ) { - throw new Error("sparse dirty-state gate failed"); + const samples = samplesOf(input.samples, mode); + const values = valuesOf(input.values, samples); + const ceilings = ceilingsOf(input.ceilings); + if (values.lookupP95Ns.median > ceilings.lookupP95Ns) { + throw new TypeError( + `interaction artifact lookupP95Ns median ${values.lookupP95Ns.median} exceeds the release ceiling ${ceilings.lookupP95Ns}`, + ); } - const usableImprovement = - 1 - - median(artifact.coldRoute.after.usableMsSamples) / - median(artifact.coldRoute.before.usableMsSamples); - const coldPassed = artifact.coldRoute.after.sheetwriteLongTaskMsSamples.every( - (duration) => duration <= artifact.coldRoute.sheetwriteLongTaskCeilingMs, - ); - if ( - artifact.coldRoute.sheetwriteLongTaskCeilingMs !== 50 || - artifact.coldRoute.sheetwriteLongTaskGatePassed !== coldPassed || - !sameRatio(artifact.coldRoute.medianUsableImprovementRatio, usableImprovement) || - !coldPassed - ) { - throw new Error("cold-route Sheetwrite long-task gate failed"); + for (const key of INTERACTION_SAMPLE_CEILING_KEYS) { + const breached = samples[key].find((sample) => sample > ceilings[key]); + if (breached !== undefined) { + throw new TypeError( + `interaction artifact ${key} sample ${breached} exceeds the release ceiling ${ceilings[key]}`, + ); + } } - return artifact; + return { + protocolVersion: INTERACTION_PROTOCOL_VERSION, + matrixId: INTERACTION_MATRIX_ID, + mode, + metadata: metaOf(input.metadata), + runner: runnerOf(input.runner), + method: methodOf(input.method), + rows: INTERACTION_ROWS, + lookupsPerRun: INTERACTION_LOOKUPS_PER_RUN, + samples, + values, + previous: previousOf(input.previous), + ceilings, + }; } diff --git a/bench/src/paged-bench.ts b/bench/src/paged-bench.ts index f287f57a..bf0a9823 100644 --- a/bench/src/paged-bench.ts +++ b/bench/src/paged-bench.ts @@ -18,6 +18,7 @@ import { validateExactMatrix, validateRawStat, } from "./gate-protocol.js"; +import { type ProtocolCaptureMeta, protocolCaptureMeta } from "./protocol-meta.js"; import { type Stat, summarize } from "./stats.js"; export const PAGED_FULL_ROWS = 1_000_000; @@ -114,6 +115,7 @@ export interface PagedTimingResult { } export interface PagedBenchmarkResult extends GateIdentity { + readonly meta: ProtocolCaptureMeta; readonly rows: number; readonly columns: number; readonly runs: number; @@ -170,6 +172,15 @@ export function validatePagedBenchmark( ) { throw new Error(`paged ${expectedMode} configuration does not match its declared protocol`); } + if ( + result.meta.commit.length === 0 || + typeof result.meta.dirty !== "boolean" || + !Number.isFinite(Date.parse(result.meta.timestamp)) + ) { + throw new Error( + "paged capture provenance is incomplete: commit, dirty, and timestamp required", + ); + } const observed = [ ...Object.keys(result.timings).map((workload) => @@ -727,6 +738,7 @@ async function runBenchmark( protocolVersion: PERFORMANCE_GATE_PROTOCOL_VERSION, mode, matrixId: MATRIX_IDS.paged[mode], + meta: protocolCaptureMeta(), rows: rowCount, columns: COLUMNS, runs, diff --git a/bench/src/release-capture.ts b/bench/src/release-capture.ts index 4e24de8a..0f134b77 100644 --- a/bench/src/release-capture.ts +++ b/bench/src/release-capture.ts @@ -47,6 +47,27 @@ const STEPS: readonly CaptureStep[] = [ ], ], }, + { + name: "interaction", + artifacts: ["interaction-results.json"], + full: [["bench:interaction"]], + smoke: [ + [ + "src/interaction-bench.ts", + "--smoke", + "--output", + `${SCRATCH_DIRECTORY}/interaction-results.json`, + ], + ], + }, + { + name: "paged", + artifacts: ["paged-results.json"], + full: [["bench:paged"]], + smoke: [ + ["src/paged-bench.ts", "--smoke", "--output", `${SCRATCH_DIRECTORY}/paged-results.json`], + ], + }, { name: "data", artifacts: ["data-results.json"], diff --git a/bench/test/interaction-gate.test.ts b/bench/test/interaction-gate.test.ts index f0212bf2..9a64ed35 100644 --- a/bench/test/interaction-gate.test.ts +++ b/bench/test/interaction-gate.test.ts @@ -1,31 +1,320 @@ -import { describe, expect, it } from "bun:test"; -import artifact from "../results/interaction-results.json" with { type: "json" }; -import { validateInteractionArtifact } from "../src/interaction-gate.js"; +import { describe, expect, test } from "bun:test"; +import { coldTaskDurations, type TraceEvent } from "../src/interaction-bench.js"; +import { + INTERACTION_CEILING_KEYS, + INTERACTION_CEILINGS, + INTERACTION_LOOKUPS_PER_RUN, + INTERACTION_MATRIX_ID, + INTERACTION_PROTOCOL_VERSION, + INTERACTION_ROWS, + INTERACTION_SAMPLE_CEILING_KEYS, + type InteractionCeilingMetric, + type InteractionSamples, + type InteractionValues, + validateInteractionArtifact, +} from "../src/interaction-gate.js"; +import { summarize } from "../src/stats.js"; -function clone(): typeof artifact { - return structuredClone(artifact); +const TEST_COMMIT = "1".repeat(40); +const TEST_TIMESTAMP = "2026-10-07T00:00:00.000Z"; + +const FULL_SAMPLES: InteractionSamples = { + lookupMedianNs: [30, 31, 32, 33, 34], + lookupP95Ns: [60, 61, 62, 63, 64], + inverseIndexBytes: [4_000_000, 4_000_000, 4_000_000, 4_000_000, 4_000_000], + dirty100Bytes: [5_679, 5_680, 5_678, 5_681, 5_679], + ownedColdLongTaskMs: [0, 12, 0, 0, 8], + unattributedColdLongTaskMs: [54, 0, 0, 0, 0], + usableMs: [210, 215, 220, 225, 230], +}; + +const SMOKE_SAMPLES: InteractionSamples = { + lookupMedianNs: [31, 32], + lookupP95Ns: [61, 62], + inverseIndexBytes: [4_000_000, 4_000_000], + dirty100Bytes: [5_679, 5_680], + ownedColdLongTaskMs: [0, 9], + unattributedColdLongTaskMs: [54, 0], + usableMs: [212, 218], +}; + +function valuesOf(samples: InteractionSamples): InteractionValues { + return { + lookupMedianNs: summarize(samples.lookupMedianNs), + lookupP95Ns: summarize(samples.lookupP95Ns), + inverseIndexBytes: summarize(samples.inverseIndexBytes), + dirty100Bytes: summarize(samples.dirty100Bytes), + ownedColdLongTaskMs: summarize(samples.ownedColdLongTaskMs), + unattributedColdLongTaskMs: summarize(samples.unattributedColdLongTaskMs), + usableMs: summarize(samples.usableMs), + }; +} + +const TEST_RUNNER = { + runtime: "Bun test on linux x64", + browser: "Chromium test", + cpu: "test cpu", +}; + +const TEST_METHOD = { + lookup: "test lookup method", + dirty: "test dirty method", + coldRoute: "test cold-route method", + attribution: "test attribution method", + previous: "test previous-capture method", +}; + +interface FixtureEdits { + readonly protocolVersion?: number; + readonly matrixId?: string; + readonly mode?: string; + readonly rows?: number; + readonly metadata?: { readonly commit?: string; readonly dirty?: unknown }; + readonly samples?: Partial; + readonly values?: InteractionValues; + readonly ceilings?: Partial>; + readonly previous?: unknown; +} + +/** + * A valid artifact with the requested hand-edit-style changes. Values derive + * from the samples of the same fixture, so only the edit under test can make a + * case invalid. + */ +function fixture(edits: FixtureEdits = {}): unknown { + const samples: InteractionSamples = { ...FULL_SAMPLES, ...edits.samples }; + return { + protocolVersion: edits.protocolVersion ?? INTERACTION_PROTOCOL_VERSION, + matrixId: edits.matrixId ?? INTERACTION_MATRIX_ID, + mode: edits.mode ?? "full", + metadata: { + commit: edits.metadata?.commit ?? TEST_COMMIT, + dirty: edits.metadata?.dirty ?? false, + timestamp: TEST_TIMESTAMP, + }, + runner: TEST_RUNNER, + method: TEST_METHOD, + rows: edits.rows ?? INTERACTION_ROWS, + lookupsPerRun: INTERACTION_LOOKUPS_PER_RUN, + samples, + values: edits.values ?? valuesOf(samples), + previous: edits.previous === undefined ? null : edits.previous, + ceilings: { ...INTERACTION_CEILINGS, ...edits.ceilings }, + }; +} + +/** The base samples of `key`, three of them just above the ceiling. */ +function aboveCeiling(key: InteractionCeilingMetric): Partial { + const samples = [...FULL_SAMPLES[key]]; + for (const index of [0, 1, 2]) samples[index] = INTERACTION_CEILINGS[key] + 1; + return { [key]: samples }; } describe("interaction performance artifact", () => { - it("rejects weakened speed, memory, sparse, and cold-route gates", () => { - for (const [mutate, message] of [ - [ - (value: typeof artifact) => (value.directionalPrefetch.medianResidencyRatio = 0.94), - "prefetch", - ], - [ - (value: typeof artifact) => (value.viewIndex.medianLookupImprovementRatio = 0.19), - "view-index", - ], - [(value: typeof artifact) => (value.sparseDirty.dirty100Bytes = 1024 * 1024 + 1), "sparse"], - [ - (value: typeof artifact) => (value.coldRoute.after.sheetwriteLongTaskMsSamples[0] = 51), - "cold-route", - ], - ] as const) { - const candidate = clone(); - mutate(candidate); - expect(() => validateInteractionArtifact(candidate)).toThrow(message); + test("accepts a full capture and a smoke rehearsal of the published schema", () => { + expect(() => validateInteractionArtifact(fixture())).not.toThrow(); + expect(() => + validateInteractionArtifact(fixture({ mode: "smoke", samples: SMOKE_SAMPLES })), + ).not.toThrow(); + const previous = fixture({ + previous: { + matrixId: INTERACTION_MATRIX_ID, + mode: "full", + commit: TEST_COMMIT, + timestamp: TEST_TIMESTAMP, + values: valuesOf(FULL_SAMPLES), + }, + }); + const artifact = validateInteractionArtifact(previous); + expect(artifact.previous?.commit).toBe(TEST_COMMIT); + expect(artifact.values.usableMs.median).toBe(220); + }); + + test("rejects a different protocol, matrix, mode, or workload", () => { + expect(() => validateInteractionArtifact(fixture({ protocolVersion: 3 }))).toThrow( + "stale protocol: expected 2", + ); + expect(() => validateInteractionArtifact(fixture({ matrixId: "interaction-full-v1" }))).toThrow( + "stale matrix", + ); + expect(() => validateInteractionArtifact(fixture({ mode: "rehearsal" }))).toThrow( + "must be full or smoke", + ); + expect(() => validateInteractionArtifact(fixture({ mode: "smoke" }))).toThrow( + "lookupMedianNs requires 2 raw samples", + ); + expect(() => + validateInteractionArtifact(fixture({ mode: "smoke", samples: SMOKE_SAMPLES, rows: 1_000 })), + ).toThrow("workload changed"); + }); + + test("rejects incomplete or dirty capture provenance", () => { + expect(() => validateInteractionArtifact(fixture({ metadata: { commit: "" } }))).toThrow( + "metadata.commit must be a non-empty string", + ); + expect(() => validateInteractionArtifact(fixture({ metadata: { dirty: "false" } }))).toThrow( + "metadata.dirty must be a boolean", + ); + }); + + test("rejects a value that is not the summary of its samples", () => { + const values = valuesOf(FULL_SAMPLES); + const notDerived = { + ...values, + // Internally consistent, but not the summary of the samples above. + lookupMedianNs: { median: 40, p95: 45, mean: 40, stddev: 2, min: 38, max: 45, iters: 5 }, + }; + expect(() => validateInteractionArtifact(fixture({ values: notDerived }))).toThrow( + "values.lookupMedianNs.stat.median does not match raw samples", + ); + }); + + test("rejects a sample above any release ceiling", () => { + for (const key of INTERACTION_SAMPLE_CEILING_KEYS) { + expect(() => validateInteractionArtifact(fixture({ samples: aboveCeiling(key) }))).toThrow( + `${key} sample ${INTERACTION_CEILINGS[key] + 1} exceeds the release ceiling`, + ); + } + expect(() => + validateInteractionArtifact(fixture({ samples: aboveCeiling("lookupP95Ns") })), + ).toThrow( + `lookupP95Ns median ${INTERACTION_CEILINGS.lookupP95Ns + 1} exceeds the release ceiling`, + ); + }); + + test("rejects a weakened release ceiling", () => { + for (const key of INTERACTION_CEILING_KEYS) { + expect(() => + validateInteractionArtifact( + fixture({ ceilings: { [key]: INTERACTION_CEILINGS[key] * 2 } }), + ), + ).toThrow(`ceilings.${key} is not the release ceiling`); } }); + + test("rejects a previous capture of another schema", () => { + expect(() => + validateInteractionArtifact( + fixture({ + previous: { + matrixId: "interaction-full-v1", + mode: "full", + commit: TEST_COMMIT, + timestamp: TEST_TIMESTAMP, + values: valuesOf(FULL_SAMPLES), + }, + }), + ), + ).toThrow("previous must be a capture of the same schema"); + }); +}); + +function task(ts: number, durationMs: number, overrides: Partial = {}): TraceEvent { + return { + name: "RunTask", + ph: "X", + ts, + dur: durationMs * 1_000, + pid: 1, + tid: 1, + url: undefined, + profile: undefined, + ...overrides, + }; +} + +const PACKAGE_URL = "http://127.0.0.1:5197/@fs/repo/packages/core/dist/canvas-paint.js"; +const SHELL_URL = "http://127.0.0.1:5197/node_modules/.vite/deps/react-dom_client.js"; +const PROFILE_INTERVAL_US = 1_000; + +function profileEvent(startUs: number, urls: readonly string[]): TraceEvent { + const nodes = [ + { id: 1, url: PACKAGE_URL }, + { id: 2, url: SHELL_URL }, + { id: 3, url: undefined }, + ]; + const nodeIdOf = (url: string): number => (url === PACKAGE_URL ? 1 : url === SHELL_URL ? 2 : 3); + return { + name: "ProfileChunk", + ph: "P", + ts: startUs, + dur: undefined, + pid: 1, + tid: 1, + url: undefined, + profile: { + startTime: startUs, + nodes, + samples: urls.map(nodeIdOf), + timeDeltas: urls.map((_, index) => (index === 0 ? 0 : PROFILE_INTERVAL_US)), + }, + }; +} + +/** `count` profiled samples of one URL, one millisecond apart from `ts`. */ +function scriptSamples( + ts: number, + url: string, + count: number, + overrides: Partial = {}, +): TraceEvent { + return { + ...profileEvent( + ts, + Array.from({ length: count }, () => url), + ), + ...overrides, + }; +} + +describe("cold-route long-task attribution", () => { + test("charges a long task that mostly runs package frames to Sheetwrite", () => { + const events = [ + task(0, 60), + scriptSamples(1_000, PACKAGE_URL, 50), + scriptSamples(51_000, SHELL_URL, 5), + ]; + expect(coldTaskDurations(events)).toEqual({ ownedMs: 60, unattributedMs: 0 }); + }); + + test("reports a long task that mostly runs other code as unattributed", () => { + const events = [ + task(0, 60), + scriptSamples(1_000, PACKAGE_URL, 20), + scriptSamples(21_000, SHELL_URL, 40), + ]; + expect(coldTaskDurations(events)).toEqual({ ownedMs: 0, unattributedMs: 60 }); + }); + + test("reports a long task the profile does not cover as unattributed", () => { + expect(coldTaskDurations([task(0, 60)])).toEqual({ ownedMs: 0, unattributedMs: 60 }); + }); + + test("ignores tasks below the long-task threshold", () => { + expect(coldTaskDurations([task(0, 49), scriptSamples(1_000, PACKAGE_URL, 60)])).toEqual({ + ownedMs: 0, + unattributedMs: 0, + }); + }); + + test("does not attribute samples of another thread or outside the task window", () => { + const otherThread = scriptSamples(1_000, PACKAGE_URL, 50, { tid: 2 }); + const afterTask = scriptSamples(90_000, PACKAGE_URL, 50); + expect(coldTaskDurations([task(0, 60), otherThread, afterTask])).toEqual({ + ownedMs: 0, + unattributedMs: 60, + }); + }); + + test("reports the longest owned and unattributed task of a sample", () => { + const events = [ + task(0, 55), + scriptSamples(1_000, PACKAGE_URL, 50), + task(200_000, 70), + scriptSamples(201_000, PACKAGE_URL, 65), + task(400_000, 80), + scriptSamples(401_000, SHELL_URL, 75), + ]; + expect(coldTaskDurations(events)).toEqual({ ownedMs: 70, unattributedMs: 80 }); + }); }); diff --git a/bench/test/paged-gate.test.ts b/bench/test/paged-gate.test.ts index 5cce805f..7ecffd04 100644 --- a/bench/test/paged-gate.test.ts +++ b/bench/test/paged-gate.test.ts @@ -70,6 +70,7 @@ function smokeFixture(): PagedBenchmarkResult { protocolVersion: PERFORMANCE_GATE_PROTOCOL_VERSION, mode: "smoke", matrixId: MATRIX_IDS.paged.smoke, + meta: { commit: "fixture", dirty: false, timestamp: "2026-10-07T00:00:00Z" }, rows: PAGED_SMOKE_ROWS, columns: 5, runs: 2, @@ -115,6 +116,7 @@ function fullFixture(): PagedBenchmarkResult { protocolVersion: PERFORMANCE_GATE_PROTOCOL_VERSION, mode: "full", matrixId: MATRIX_IDS.paged.full, + meta: { commit: "fixture", dirty: false, timestamp: "2026-10-07T00:00:00Z" }, rows: PAGED_FULL_ROWS, columns: 5, runs: 12, @@ -187,4 +189,16 @@ describe("paged benchmark exact matrix", () => { "rows=10000;probe=viewport.fullyLoaded", ); }); + + test("rejects a capture without a usable provenance stamp", () => { + const source = smokeFixture(); + expect(() => validatePagedBenchmark(source, "smoke")).not.toThrow(); + const unstamped = { + ...source, + meta: { ...source.meta, commit: "", timestamp: "not a timestamp" }, + }; + expect(() => validatePagedBenchmark(unstamped, "smoke")).toThrow( + "paged capture provenance is incomplete", + ); + }); }); diff --git a/docs/src/content/docs/guides/performance-resources.md b/docs/src/content/docs/guides/performance-resources.md index 70785a26..6322585f 100644 --- a/docs/src/content/docs/guides/performance-resources.md +++ b/docs/src/content/docs/guides/performance-resources.md @@ -6,6 +6,20 @@ Every number on this page comes from a validated local protocol artifact capture ## Capture +`bench:release` also captures the performance showcase's two captures: the +paged-storage evidence at `bench/results/paged-results.json` and the +interaction evidence at `bench/results/interaction-results.json`. Its smoke +rehearsal writes only to `bench/results/smoke/`. + +The interaction capture (`bench:interaction`, schema `interaction-current-v2`) +measures the current build, not a comparison: packed inverse-index lookup +latency in nanoseconds, inverse-index backing bytes, retained bytes after 100 +distant edits, and the cold-route long task owned by Sheetwrite. It records the +raw samples, the summary derived from them, the runner, and the method, and it +holds those four metrics to absolute release ceilings. Version 1 recorded a +one-off before/after pair whose baseline build cannot be re-measured at a later +release, so no old sample is carried forward into the current capture. +
Mixed capture vintages. The sections below were captured at different commits. Each section is internally consistent and each one names its own commit; the page as a whole is not a single point-in-time measurement.
| Section | Artifact | Commit | Captured | From 2679b8ded492dd8fdf1a9ff506af117c19320edd Mon Sep 17 00:00:00 2001 From: chh-ay Date: Wed, 7 Oct 2026 00:40:32 +0700 Subject: [PATCH 2/5] bench: publish the current interaction capture Replace the 0.3.0 before/after artifact with a capture of the current build at the harness commit: five samples of packed inverse-index latency and backing bytes, retained bytes after 100 distant edits, the Sheetwrite-owned cold-route long task, and the cold route's usable time, each with its summary and the runner that produced it. Every release ceiling passed, and the version 1 pair now lives in git history only. --- bench/results/interaction-results.json | 199 +++++++++++++++++-------- 1 file changed, 138 insertions(+), 61 deletions(-) diff --git a/bench/results/interaction-results.json b/bench/results/interaction-results.json index f6215522..057ffca0 100644 --- a/bench/results/interaction-results.json +++ b/bench/results/interaction-results.json @@ -1,70 +1,147 @@ { - "protocolVersion": 1, - "matrixId": "interaction-full-v1", - "capturedAt": "2026-07-22T03:02:46Z", - "source": { - "commit": "77c41983b53f09c393c6f8b82a4033283c0f9830", - "workingTree": true, - "runtime": "Bun 1.3.14 on Linux x64", - "browser": "Chrome 140.0.7339.127 via Playwright 1.61.1", - "cpu": "12th Gen Intel Core i9-12900H", - "samples": 5, - "warmup": "none; each cold-route sample used a new browser context", - "correctness": "first real feed cell resolved to 1 before each cold sample completed" + "protocolVersion": 2, + "matrixId": "interaction-current-v2", + "mode": "full", + "metadata": { + "commit": "5b5f8b4d2b296475cceef6cc658437e8f85f2f6a", + "dirty": false, + "timestamp": "2026-10-06T17:40:23.096Z" }, - "directionalPrefetch": { - "sourceLatencyMs": 90, - "frameBudgetMs": 16.7, - "residencyRatioSamples": [1, 1, 1, 1, 1], - "p95VisibleWaitMsSamples": [0, 0, 0, 0, 0], - "requestedRowMultiplierSamples": [1.1162790697674418, 1.1162790697674418, 1.1162790697674418, 1.1162790697674418, 1.1162790697674418], - "medianResidencyRatio": 1, - "medianP95VisibleWaitMs": 0 + "runner": { + "runtime": "Bun 1.4.2 on linux x64", + "browser": "153.0.8010.12", + "cpu": "12th Gen Intel(R) Core(TM) i9-12900H" }, - "viewIndex": { - "rows": 1000000, - "lookupsPerRun": 524288, - "baseline": { - "lookupMedianNsSamples": [37.88488769531501, 39.244018554689816, 44.43444824218934, 45.83947753906622, 44.521118164060624], - "lookupP95NsSamples": [45.98823242187404, 54.03151855468516, 101.12755126953121, 62.81248779296663, 83.76065673827702], - "retainedBytes": 67125641, - "coldBuildMs": 109.215509 - }, - "packed": { - "lookupMedianNsSamples": [25.080932617187276, 29.146606445313097, 25.03100585937463, 31.878540039062454, 32.43225097656309], - "lookupP95NsSamples": [45.79450683593725, 73.24847412109399, 66.49718017578127, 68.38371582031252, 81.93341064453101], - "retainedBytes": 4043043, - "backingBytes": 4000000, - "coldBuildMs": 4.693290000000001 - }, - "medianLookupImprovementRatio": 0.3440538231407775, - "p95LookupRegressionRatio": 0.08869618483682662, - "retainedHeapReductionRatio": 0.9397690220939566 + "method": { + "lookup": "Fresh packed view-index worker process per sample: 1,000,000 rows of one column, 128 batches of 4,096 deterministic lookups (524,288 lookups), nanoseconds per lookup measured over the batches. inverseIndexBytes is the inverse view-index backing store of that worker.", + "dirty": "Fresh paged dirty-100 probe process per sample: 1,000,000 rows, 100 distant edits, no clean page loaded. dirty100Bytes is allocated plus sparse dirty bytes retained.", + "coldRoute": "Current Vite development route /showcases/performance/, rendered once before sampling so the development server's transform cache is warm, then loaded in a fresh browser context per sample with no shared cache, code cache, or service worker state. Server-side rendering answers the route before the first sample, and a sample measures the browser, not the first server transform; this is a development-server measurement, not a production bundle. usableMs is the page clock when the first rectangular page of feed cells became resident in the public paged store, an upper bound that includes the readiness poll.", + "attribution": "Chrome DevTools protocol tracing (devtools.timeline,disabled-by-default-devtools.timeline,disabled-by-default-v8.cpu_profiler) from before navigation until the page is usable, in the renderer process: 50 ms or longer RunTask events are long tasks, and the V8 CPU profile on the same clock says what ran inside them. A task is owned by Sheetwrite when frames served from packages account for at least half of the CPU profiled inside it, and its complete duration is then recorded as the owned long task. Tasks below that share, tasks running only other code, and tasks the profile does not cover are reported as unattributed, so a partly owned task is never counted as owned work. A zero sample means no owned long task was observed.", + "previous": "Same-schema comparison only. When the artifact already at this path is a capture of interaction-current-v2, previous records that capture's identity and its per-metric medians, so a reader can state the change without inventing a baseline. A first capture, or an unreadable predecessor, records null." }, - "sparseDirty": { - "baseline100Bytes": 5427200, - "dirty100Bytes": 5679, - "dirty10000Bytes": 718911, - "dirty100ReductionRatio": 0.9989536040683962, - "dirty100CleanChunks": 0, - "dirty10000CleanChunks": 0 + "rows": 1000000, + "lookupsPerRun": 524288, + "samples": { + "lookupMedianNs": [ + 31.459106445311942, + 26.614868164063534, + 22.76940917968727, + 23.46667480468759, + 24.449707031250558 + ], + "lookupP95Ns": [ + 442.6306884765626, + 412.91258544921925, + 307.31452636718694, + 406.04642333984384, + 366.7959350585941 + ], + "inverseIndexBytes": [ + 4000000, + 4000000, + 4000000, + 4000000, + 4000000 + ], + "dirty100Bytes": [ + 5679, + 5679, + 5679, + 5679, + 5679 + ], + "ownedColdLongTaskMs": [ + 0, + 0, + 0, + 0, + 0 + ], + "unattributedColdLongTaskMs": [ + 127.079, + 118.366, + 129.117, + 108.983, + 112.155 + ], + "usableMs": [ + 1507.2000000001863, + 1351.5999999996275, + 1345.5999999996275, + 1369.5, + 1297.5 + ] }, - "coldRoute": { - "route": "/showcases/performance/", - "before": { - "usableMsSamples": [291.3, 351.2, 242.3, 265.5, 250.1], - "sheetwriteLongTaskMsSamples": [64.2, 65.8, 56.4, 65, 63.9], - "dominantOwner": "eager 100,000-row revenue scenario imported only for four theme fields" + "values": { + "lookupMedianNs": { + "median": 24.449707031250558, + "p95": 30.49025878906226, + "mean": 25.751953125000178, + "stddev": 3.134649315693444, + "min": 22.76940917968727, + "max": 31.459106445311942, + "iters": 5 + }, + "lookupP95Ns": { + "median": 406.04642333984384, + "p95": 436.687067871094, + "mean": 387.1400317382814, + "stddev": 46.66236755863049, + "min": 307.31452636718694, + "max": 442.6306884765626, + "iters": 5 + }, + "inverseIndexBytes": { + "median": 4000000, + "p95": 4000000, + "mean": 4000000, + "stddev": 0, + "min": 4000000, + "max": 4000000, + "iters": 5 }, - "after": { - "usableMsSamples": [231.5, 216.6, 201.3, 215.8, 186.6], - "sheetwriteLongTaskMsSamples": [0, 0, 0, 0, 0], - "routeCpuMsSamples": [9.209, 9.037, 9.064, 8.057, 7.202], - "gridBuildCpuMsSamples": [6.859, 6.268, 6.241, 5.343, 6.407], - "reportedUnattributedLongTaskMsSamples": [54, 54, 0, 0, 0] + "dirty100Bytes": { + "median": 5679, + "p95": 5679, + "mean": 5679, + "stddev": 0, + "min": 5679, + "max": 5679, + "iters": 5 }, - "medianUsableImprovementRatio": 0.1871939736346515, - "sheetwriteLongTaskCeilingMs": 50, - "sheetwriteLongTaskGatePassed": true + "ownedColdLongTaskMs": { + "median": 0, + "p95": 0, + "mean": 0, + "stddev": 0, + "min": 0, + "max": 0, + "iters": 5 + }, + "unattributedColdLongTaskMs": { + "median": 118.366, + "p95": 128.7094, + "mean": 119.14000000000001, + "stddev": 7.938793359195081, + "min": 108.983, + "max": 129.117, + "iters": 5 + }, + "usableMs": { + "median": 1351.5999999996275, + "p95": 1479.660000000149, + "mean": 1374.2799999998883, + "stddev": 70.60122945117568, + "min": 1297.5, + "max": 1507.2000000001863, + "iters": 5 + } + }, + "previous": null, + "ceilings": { + "lookupP95Ns": 1000, + "inverseIndexBytes": 4000000, + "dirty100Bytes": 1048576, + "ownedColdLongTaskMs": 50 } } From 76e79ffd1ad11f2896c19377cb92dff8e4b5bf06 Mon Sep 17 00:00:00 2001 From: chh-ay Date: Wed, 7 Oct 2026 00:42:56 +0700 Subject: [PATCH 3/5] docs: generate the release capture set into the evidence page The performance evidence page is generated, so the capture set it names has to come from scripts/docs.ts: the page now states that bench:release refreshes render-scale, the showcase's paged storage, the interaction evidence, data, core paths, formula, the engine captures, and XLSX, and what the interaction capture measures. --- .../docs/guides/performance-resources.md | 18 ++++-------------- scripts/docs.ts | 4 ++++ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/docs/src/content/docs/guides/performance-resources.md b/docs/src/content/docs/guides/performance-resources.md index 6322585f..b749ae9d 100644 --- a/docs/src/content/docs/guides/performance-resources.md +++ b/docs/src/content/docs/guides/performance-resources.md @@ -6,20 +6,6 @@ Every number on this page comes from a validated local protocol artifact capture ## Capture -`bench:release` also captures the performance showcase's two captures: the -paged-storage evidence at `bench/results/paged-results.json` and the -interaction evidence at `bench/results/interaction-results.json`. Its smoke -rehearsal writes only to `bench/results/smoke/`. - -The interaction capture (`bench:interaction`, schema `interaction-current-v2`) -measures the current build, not a comparison: packed inverse-index lookup -latency in nanoseconds, inverse-index backing bytes, retained bytes after 100 -distant edits, and the cold-route long task owned by Sheetwrite. It records the -raw samples, the summary derived from them, the runner, and the method, and it -holds those four metrics to absolute release ceilings. Version 1 recorded a -one-off before/after pair whose baseline build cannot be re-measured at a later -release, so no old sample is carried forward into the current capture. -
Mixed capture vintages. The sections below were captured at different commits. Each section is internally consistent and each one names its own commit; the page as a whole is not a single point-in-time measurement.
| Section | Artifact | Commit | Captured | @@ -29,6 +15,10 @@ release, so no old sample is carried forward into the current capture. | Formula engine benchmark | `bench/results/formula-results.json` | ed66b6d7c95c | 2026-10-02 18:09 UTC | | Formula engines (analysis capture) | `bench/results/full-engine-results.json` | a50a897e9f93 | 2026-10-04 11:20 UTC | +`bench:release` captures render-scale, the performance showcase's paged storage at `bench/results/paged-results.json`, the interaction evidence at `bench/results/interaction-results.json`, data, core paths, formula, formula engines, matched engines, and XLSX, then re-records the render regression baseline last. Its smoke rehearsal writes only to `bench/results/smoke/`. + +The interaction capture (`bench:interaction`, schema `interaction-current-v2`) measures the current build, not a comparison: packed inverse-index lookup latency in nanoseconds, inverse-index backing bytes, retained bytes after 100 distant edits, and the cold-route long task owned by Sheetwrite. It records the raw samples, the summary derived from them, the runner, and the method, and it holds those four metrics to absolute release ceilings. Version 1 recorded a one-off before/after pair whose baseline build cannot be re-measured at a later release, so no old sample is carried forward into the current capture. + ## Matched local regression check Timing comparisons run deliberately on a controlled local machine, not as a required CI job. Capture ten fresh matched rounds, retain every raw sample, and compare the fresh artifact with the committed baseline. Any unapproved slowdown fails the local command. diff --git a/scripts/docs.ts b/scripts/docs.ts index cc633f20..873d478d 100644 --- a/scripts/docs.ts +++ b/scripts/docs.ts @@ -2464,6 +2464,10 @@ export async function renderEvidencePage(sizeHistoryOverride?: SizeHistory): Pro "Every number on this page comes from a validated local protocol artifact captured on a clean tree; nothing is published from an unvalidated or protocol-mismatched artifact. Every expected cell carries either a validated timing or its recorded failure - a run that did not complete is shown as a failure, never converted into a timing.", "", ...renderCaptureHeader(captures), + "`bench:release` captures render-scale, the performance showcase's paged storage at `bench/results/paged-results.json`, the interaction evidence at `bench/results/interaction-results.json`, data, core paths, formula, formula engines, matched engines, and XLSX, then re-records the render regression baseline last. Its smoke rehearsal writes only to `bench/results/smoke/`.", + "", + "The interaction capture (`bench:interaction`, schema `interaction-current-v2`) measures the current build, not a comparison: packed inverse-index lookup latency in nanoseconds, inverse-index backing bytes, retained bytes after 100 distant edits, and the cold-route long task owned by Sheetwrite. It records the raw samples, the summary derived from them, the runner, and the method, and it holds those four metrics to absolute release ceilings. Version 1 recorded a one-off before/after pair whose baseline build cannot be re-measured at a later release, so no old sample is carried forward into the current capture.", + "", "## Matched local regression check", "", "Timing comparisons run deliberately on a controlled local machine, not as a required CI job. Capture ten fresh matched rounds, retain every raw sample, and compare the fresh artifact with the committed baseline. Any unapproved slowdown fails the local command.", From a4ebf2107e40353c42af812ccb758b2a23c5db4e Mon Sep 17 00:00:00 2001 From: chh-ay Date: Wed, 7 Oct 2026 00:51:02 +0700 Subject: [PATCH 4/5] bench: name the browser in the interaction provenance The capture wrote the raw Chromium build number, so the artifact's runner provenance did not say which browser it was. Record `Chromium `, as the other browser captures do; the next capture picks the string up. --- bench/src/interaction-bench.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/src/interaction-bench.ts b/bench/src/interaction-bench.ts index 173a27c8..51906b9d 100644 --- a/bench/src/interaction-bench.ts +++ b/bench/src/interaction-bench.ts @@ -542,7 +542,7 @@ async function main(): Promise { metadata: protocolCaptureMeta(), runner: { runtime: `Bun ${Bun.version} on ${process.platform} ${process.arch}`, - browser: browser.version(), + browser: `Chromium ${browser.version()}`, cpu: cpus()[0]?.model ?? "unknown", }, method: methodNotes(), From dbad27ef611844047c60fc7aac2ab76c299f57fa Mon Sep 17 00:00:00 2001 From: chh-ay Date: Wed, 7 Oct 2026 00:51:02 +0700 Subject: [PATCH 5/5] docs: read the current interaction capture in the performance showcase The showcase now reads the interaction-current-v2 artifact: the four cards show the current value of their metric and its release ceiling, and they show the change only against a recorded predecessor capture, so a first capture states the current value instead of an invented before. The scenario module annotates the artifact shape it consumes, which keeps the predecessor optional. --- docs/src/routes/showcases.performance.tsx | 102 ++++++++++++++------ docs/src/showcases/scenarios/scale.ts | 110 ++++++++++++++++------ 2 files changed, 156 insertions(+), 56 deletions(-) diff --git a/docs/src/routes/showcases.performance.tsx b/docs/src/routes/showcases.performance.tsx index fcc31cf4..67f400d5 100644 --- a/docs/src/routes/showcases.performance.tsx +++ b/docs/src/routes/showcases.performance.tsx @@ -75,6 +75,40 @@ const LANDMARKS = [ { label: "100%", ratio: 1 }, ] as const; const SCALE_OVERSCAN = 4; +/** Smallest visible bar so a capture that is far below its predecessor still reads. */ +const MINIMUM_BAR_PERCENT = 0.5; + +/** + * Width of the current-value bar as a percent of the previous same-schema + * capture. Without a previous capture the bar states the current value alone. + */ +function previousShareBar(current: number, previous: number | undefined): number { + if (previous === undefined || previous === 0 || !Number.isFinite(previous)) return 100; + return Math.min(Math.max((current / previous) * 100, MINIMUM_BAR_PERCENT), 100); +} + +/** Share of the owned long-task ceiling the current capture uses. */ +function ledgerPercent(current: number, ceiling: number): number { + if (!Number.isFinite(ceiling) || ceiling <= 0) return 0; + return Math.min(Math.max((current / ceiling) * 100, 0), 100); +} + +/** + * The value line under a card: the change since the previous capture of the + * same schema when there is one, otherwise what the current value means. + */ +function interactionNote( + current: number, + previous: number | undefined, + format: (value: number) => string, + description: string, +): string { + if (previous === undefined || previous === 0 || !Number.isFinite(previous)) return description; + const change = ((previous - current) / previous) * 100; + return change >= 0 + ? `from ${format(previous)} · ${change.toFixed(1)}% lower` + : `from ${format(previous)} · ${(-change).toFixed(1)}% higher`; +} const COMPARISON_LOG_MAX_EXPONENT = COMPARISON_EVIDENCE.available ? Math.max( @@ -519,8 +553,9 @@ function PerformanceRoute() { (owner) => owner.logicalBytes !== 0 || owner.allocatedBytes !== 0 || owner.entries !== 0, ) ?? []; const latestCrossing = crossings[0]; - const interactionBefore = INTERACTION_EVIDENCE.before; - const interactionAfter = INTERACTION_EVIDENCE.after; + const interaction = INTERACTION_EVIDENCE.current; + const interactionPrevious = INTERACTION_EVIDENCE.previous; + const interactionCeilings = INTERACTION_EVIDENCE.ceilings; const selectedFormula = gridRef.current?.store.getFormula(selectedAddress) ?? null; return ( @@ -1311,72 +1346,85 @@ function PerformanceRoute() {
Lookup latency - {interactionAfter.lookupMedianNs.toFixed(2)} ns + {interaction.lookupMedianNs.toFixed(2)} ns

- from {interactionBefore.lookupMedianNs.toFixed(2)} ns ·{" "} - {( - (1 - interactionAfter.lookupMedianNs / interactionBefore.lookupMedianNs) * - 100 - ).toFixed(1)} - % lower + {interactionNote( + interaction.lookupMedianNs, + interactionPrevious?.lookupMedianNs, + (value) => `${value.toFixed(2)} ns`, + `p95 ${interaction.lookupP95Ns.toFixed(0)} ns · ceiling ${interactionCeilings.lookupP95Ns} ns`, + )}

Inverse index - {formatBytes(interactionAfter.viewIndexBytes)} + {formatBytes(interaction.viewIndexBytes)}

- from {formatBytes(interactionBefore.viewIndexBytes)} ·{" "} - {( - (1 - interactionAfter.viewIndexBytes / interactionBefore.viewIndexBytes) * - 100 - ).toFixed(1)} - % lower + {interactionNote( + interaction.viewIndexBytes, + interactionPrevious?.viewIndexBytes, + formatBytes, + `ceiling ${formatBytes(interactionCeilings.inverseIndexBytes)}`, + )}

100 distant edits - {formatBytes(interactionAfter.dirty100Bytes)} + {formatBytes(interaction.dirty100Bytes)}

- from {formatBytes(interactionBefore.dirty100Bytes)} · sparse cells stay sparse + {interactionNote( + interaction.dirty100Bytes, + interactionPrevious?.dirty100Bytes, + formatBytes, + `sparse cells stay sparse · ceiling ${formatBytes(interactionCeilings.dirty100Bytes)}`, + )}

Owned cold long task - {interactionAfter.coldOwnedLongTaskMs.toFixed(1)} ms + {interaction.coldOwnedLongTaskMs.toFixed(1)} ms -

from {interactionBefore.coldOwnedLongTaskMs.toFixed(1)} ms median

+

+ {interactionNote( + interaction.coldOwnedLongTaskMs, + interactionPrevious?.coldOwnedLongTaskMs, + (value) => `${value.toFixed(1)} ms`, + `no Sheetwrite-owned long task · ${interaction.coldUnattributedLongTaskMs.toFixed(0)} ms unattributed nearby`, + )} +

diff --git a/docs/src/showcases/scenarios/scale.ts b/docs/src/showcases/scenarios/scale.ts index 50c715d0..34c2c5d8 100644 --- a/docs/src/showcases/scenarios/scale.ts +++ b/docs/src/showcases/scenarios/scale.ts @@ -601,41 +601,93 @@ export const PAGED_EVIDENCE = { probes: pagedResults.probes, } as const; -function sampleMedian(values: readonly number[]): number { - const sorted = [...values].sort((left, right) => left - right); - return sorted[Math.floor(sorted.length / 2)]!; +/** + * The interaction artifact published by `bench/src/interaction-gate.ts`, as the + * showcase consumes it. The artifact is annotated here because the page reads a + * predecessor that is absent from the first capture of the schema. + */ +interface InteractionArtifactPrevious { + readonly mode: string; + readonly commit: string; + readonly timestamp: string; + readonly values: { + readonly lookupMedianNs: { readonly median: number }; + readonly inverseIndexBytes: { readonly median: number }; + readonly dirty100Bytes: { readonly median: number }; + readonly ownedColdLongTaskMs: { readonly median: number }; + }; } -/** Committed five-sample interaction, memory, and cold-route evidence. */ +interface InteractionArtifact { + readonly matrixId: string; + readonly protocolVersion: number; + readonly mode: string; + readonly metadata: { + readonly commit: string; + readonly dirty: boolean; + readonly timestamp: string; + }; + readonly runner: { readonly runtime: string; readonly browser: string; readonly cpu: string }; + readonly samples: { readonly lookupMedianNs: readonly number[] }; + readonly values: { + readonly lookupMedianNs: { readonly median: number }; + readonly lookupP95Ns: { readonly median: number }; + readonly inverseIndexBytes: { readonly median: number }; + readonly dirty100Bytes: { readonly median: number }; + readonly ownedColdLongTaskMs: { readonly median: number }; + readonly unattributedColdLongTaskMs: { readonly median: number }; + readonly usableMs: { readonly median: number }; + }; + readonly ceilings: { + readonly lookupP95Ns: number; + readonly inverseIndexBytes: number; + readonly dirty100Bytes: number; + readonly ownedColdLongTaskMs: number; + }; + readonly previous: InteractionArtifactPrevious | null; +} + +const interactionArtifact: InteractionArtifact = interactionResults; + +/** + * Committed interaction evidence for the performance showcase: the current + * values of the four published metrics, their release ceilings, and the + * same-schema capture that was published before this one, when there was one. + */ export const INTERACTION_EVIDENCE = { source: "bench/results/interaction-results.json", - protocol: `${interactionResults.matrixId} (protocol v${interactionResults.protocolVersion})`, - capture: interactionResults.source, - prefetch: interactionResults.directionalPrefetch, - before: { - lookupMedianNs: sampleMedian(interactionResults.viewIndex.baseline.lookupMedianNsSamples), - viewIndexBytes: interactionResults.viewIndex.baseline.retainedBytes, - dirty100Bytes: interactionResults.sparseDirty.baseline100Bytes, - coldOwnedLongTaskMs: sampleMedian( - interactionResults.coldRoute.before.sheetwriteLongTaskMsSamples, - ), - }, - after: { - lookupMedianNs: sampleMedian(interactionResults.viewIndex.packed.lookupMedianNsSamples), - viewIndexBytes: interactionResults.viewIndex.packed.retainedBytes, - dirty100Bytes: interactionResults.sparseDirty.dirty100Bytes, - coldOwnedLongTaskMs: sampleMedian( - interactionResults.coldRoute.after.sheetwriteLongTaskMsSamples, - ), + protocol: `${interactionArtifact.matrixId} (protocol v${interactionArtifact.protocolVersion}, ${interactionArtifact.mode} mode)`, + capture: { + commit: interactionArtifact.metadata.commit, + dirty: interactionArtifact.metadata.dirty, + timestamp: interactionArtifact.metadata.timestamp, + runtime: interactionArtifact.runner.runtime, + browser: interactionArtifact.runner.browser, + cpu: interactionArtifact.runner.cpu, + samples: interactionArtifact.samples.lookupMedianNs.length, }, - gains: { - lookup: interactionResults.viewIndex.medianLookupImprovementRatio, - heap: interactionResults.viewIndex.retainedHeapReductionRatio, - sparse: interactionResults.sparseDirty.dirty100ReductionRatio, - coldUsable: interactionResults.coldRoute.medianUsableImprovementRatio, + ceilings: interactionArtifact.ceilings, + current: { + lookupMedianNs: interactionArtifact.values.lookupMedianNs.median, + lookupP95Ns: interactionArtifact.values.lookupP95Ns.median, + viewIndexBytes: interactionArtifact.values.inverseIndexBytes.median, + dirty100Bytes: interactionArtifact.values.dirty100Bytes.median, + coldOwnedLongTaskMs: interactionArtifact.values.ownedColdLongTaskMs.median, + coldUnattributedLongTaskMs: interactionArtifact.values.unattributedColdLongTaskMs.median, + usableMs: interactionArtifact.values.usableMs.median, }, - coldUnattributedLongTaskMs: - interactionResults.coldRoute.after.reportedUnattributedLongTaskMsSamples, + previous: + interactionArtifact.previous === null + ? null + : { + commit: interactionArtifact.previous.commit, + mode: interactionArtifact.previous.mode, + timestamp: interactionArtifact.previous.timestamp, + lookupMedianNs: interactionArtifact.previous.values.lookupMedianNs.median, + viewIndexBytes: interactionArtifact.previous.values.inverseIndexBytes.median, + dirty100Bytes: interactionArtifact.previous.values.dirty100Bytes.median, + coldOwnedLongTaskMs: interactionArtifact.previous.values.ownedColdLongTaskMs.median, + }, } as const; /**