From b09529e8c11533c31b67b686c95200f4758d7779 Mon Sep 17 00:00:00 2001 From: Robert DeLuca Date: Fri, 13 Nov 2026 00:34:45 -0600 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Remove=20historical=20mana?= =?UTF-8?q?ged=20benchmarks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the unused transport evaluator and machine-specific result snapshots. Preserve its useful hover, reduced-motion, and scroll checks in the real native browser suite, and trim obsolete CI history. --- benchmarks/managed/README.md | 17 - benchmarks/managed/cases.js | 249 ------- benchmarks/managed/direct.js | 8 - benchmarks/managed/fixture.js | 85 --- benchmarks/managed/operations.js | 86 --- .../managed/results/binding-chromium.json | 661 ------------------ .../managed/results/cross-browser-probes.json | 166 ----- .../managed/results/direct-chromium.json | 661 ------------------ benchmarks/managed/results/http-chromium.json | 661 ------------------ .../managed/results/vizzly-playwright.json | 122 ---- benchmarks/managed/run.js | 273 -------- docs/ci.md | 2 - tests/fixtures/native-app-server.js | 7 + tests/native/workflows.test.js | 10 + 14 files changed, 17 insertions(+), 2991 deletions(-) delete mode 100644 benchmarks/managed/README.md delete mode 100644 benchmarks/managed/cases.js delete mode 100644 benchmarks/managed/direct.js delete mode 100644 benchmarks/managed/fixture.js delete mode 100644 benchmarks/managed/operations.js delete mode 100644 benchmarks/managed/results/binding-chromium.json delete mode 100644 benchmarks/managed/results/cross-browser-probes.json delete mode 100644 benchmarks/managed/results/direct-chromium.json delete mode 100644 benchmarks/managed/results/http-chromium.json delete mode 100644 benchmarks/managed/results/vizzly-playwright.json delete mode 100644 benchmarks/managed/run.js diff --git a/benchmarks/managed/README.md b/benchmarks/managed/README.md deleted file mode 100644 index 2e7359e..0000000 --- a/benchmarks/managed/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Managed capability evaluation - -Run complete Vizzly comparisons with one worker and no shards. This smaller suite separately measures native-feature transport and verifies real browser outcomes. Run one benchmark at a time on an otherwise idle machine. - -```sh -node benchmarks/managed/run.js direct /private/tmp/dyno-managed-reference.json 20 chromium -``` - -The app persists saved notes over real HTTP. Interactor handles ordinary actions and convergence. Native cases check actual Tab traversal, Space activation, CSS hover, clipboard, browser credentials, app-only offline behavior, element pixels, and media settings. WebAuthn and offline reference probes use Chromium CDP; the clipboard reference is Chromium-specific too. These probes don't replace Vizzly's full authentication/security cases or establish native capability support in other browsers. - -One warmup precedes each scenario. Output keeps raw serial samples, median, p95, browser/runtime versions, setup time, and whole-workflow durations. The browser-local control runs before and after. Operation latency uses the caller's monotonic clock; it excludes setup and validation. Whole workflows include them. Don't subtract timestamps from different clocks or treat microbenchmarks as whole-suite speedups. - -Prototype adapters live in separate temporary folders. Pass an adapter's absolute path as the first argument. Its `createAdapter({ context, app, controller, fixture, perform, engine })` returns either `{ name, clock: 'node', invoke, dispose }` or `{ name, clock: 'browser', invokeName, dispose }`. Browser adapters register that named async function in the controller window. The evaluator owns both clocks and wraps the actual call; adapters return values, never reported durations. The direct reference uses the Node clock as a lower bound; browser candidates use the controller window clock, excluding the harness evaluation round trip. All variants use the same operation semantics and outcome checks. Missing required behavior fails the probe. - -This is historical evaluation code. Dyno now has an [explicit managed native API](../../docs/guide.md#native-browser-behavior); this evaluator compares transport candidates and does not replace product acceptance tests. - -Operations have a five-second deadline, scenarios thirty seconds, and the whole evaluator five minutes. `DYNO_EVAL_OPERATION_TIMEOUT` changes only the operation budget for failure probes. A failure stops the run, all cleanup is attempted with its own bounds, and the final JSON records the overall verdict and cleanup failures. Source fingerprints prevent comparing different evaluation suites. Saved records must increase exactly once per sample. Offline checks use fresh no-store responses and server counters. Tall offscreen captures verify every color band and catch temporary viewport enlargement through media queries and resize observations. diff --git a/benchmarks/managed/cases.js b/benchmarks/managed/cases.js deleted file mode 100644 index 6576a94..0000000 --- a/benchmarks/managed/cases.js +++ /dev/null @@ -1,249 +0,0 @@ -import assert from 'node:assert/strict'; - -let reset = async (app) => { - await app.bringToFront(); - await app.evaluate(async () => { - await window.evalI.click('Reset example'); - }); -}; -let rect = (app, selector) => - app.evaluate((selector) => { - let { x, y, width, height } = document.querySelector(selector).getBoundingClientRect(); - return { x: x + scrollX, y: y + scrollY, width, height }; - }, selector); - -export let cases = Object.freeze([ - { - name: 'native Tab and Space workflow', - async run({ app, fixture, invoke }) { - await reset(app); - let before = await (await fetch(fixture.origin + '/records', { cache: 'no-store' })).json(); - await app.evaluate(async () => { - await window.evalI.find('Name').then.focus(); - }); - await invoke({ kind: 'key', key: 'Tab' }); - await app.evaluate(async () => { - await window.evalI.assert( - () => document.activeElement.id === 'notes', - 'Tab must focus Notes', - ); - }); - await invoke({ kind: 'key', key: 'Tab' }); - await app.evaluate(async () => { - await window.evalI.assert( - () => document.activeElement.id === 'save', - 'Tab must focus Save', - ); - }); - await invoke({ kind: 'key', key: 'Space' }); - await app.evaluate(async () => { - await window.evalI.find('$(#saved)').text('Saved Robert'); - }); - let after = await (await fetch(fixture.origin + '/records', { cache: 'no-store' })).json(); - assert.equal( - after.length, - before.length + 1, - 'Native Space must persist exactly one new record', - ); - assert.equal(after.at(-1).name, 'Robert'); - }, - }, - { - name: 'real CSS hover and pointer exit', - async run({ app, invoke }) { - await reset(app); - let box = await rect(app, '#hover'); - await invoke({ kind: 'pointer', x: box.x + box.width / 2, y: box.y + box.height / 2 }); - await app.evaluate(async () => { - await window.evalI - .find('$(#hover)') - .then.assert( - ({ $ }) => - $.dataset.hovered === 'true' && - getComputedStyle($).backgroundColor === 'rgb(20, 160, 80)', - 'Native pointer must change CSS and event state', - ); - }); - await invoke({ kind: 'pointer', x: 0, y: 0 }); - await app.evaluate(async () => { - await window.evalI - .find('$(#hover)') - .then.assert( - ({ $ }) => - $.dataset.hovered === 'false' && - getComputedStyle($).backgroundColor === 'rgb(20, 60, 140)', - 'Pointer exit must restore CSS and event state', - ); - }); - }, - }, - { - name: 'real clipboard permission and copy', - chromium: true, - async run({ context, app, fixture, invoke }) { - await reset(app); - await context.clearPermissions(); - await invoke({ - kind: 'permissions', - permissions: ['clipboard-read', 'clipboard-write'], - origin: fixture.origin, - }); - await app.evaluate(async () => { - await window.evalI.click('Copy evaluation token'); - await window.evalI.find('$(#copied)').text('Copied'); - }); - assert.equal( - await app.evaluate(() => navigator.clipboard.readText()), - 'Dyno evaluation token', - ); - }, - }, - { - name: 'virtual WebAuthn with actual credential and assertion', - chromium: true, - async run({ app, invoke }) { - await reset(app); - await invoke({ kind: 'authenticator' }); - try { - let credential = await app.evaluate(async () => { - let created = await navigator.credentials.create({ - publicKey: { - challenge: crypto.getRandomValues(new Uint8Array(32)), - rp: { id: 'localhost', name: 'Dyno evaluation' }, - user: { - id: new Uint8Array([1, 2, 3, 4]), - name: 'evaluation@example.test', - displayName: 'Evaluation user', - }, - pubKeyCredParams: [{ type: 'public-key', alg: -7 }], - authenticatorSelection: { residentKey: 'required', userVerification: 'required' }, - attestation: 'none', - }, - }); - let signed = await navigator.credentials.get({ - publicKey: { - challenge: crypto.getRandomValues(new Uint8Array(32)), - rpId: 'localhost', - allowCredentials: [{ type: 'public-key', id: created.rawId }], - userVerification: 'required', - }, - }); - return { - created: created.id, - signed: signed.id, - signature: signed.response.signature.byteLength, - clientData: JSON.parse(new TextDecoder().decode(signed.response.clientDataJSON)), - }; - }); - assert.ok(credential.created); - assert.equal(credential.created, credential.signed); - assert.ok(credential.signature > 0); - assert.equal(credential.clientData.type, 'webauthn.get'); - } finally { - await invoke({ kind: 'remove-authenticator' }); - } - }, - }, - { - name: 'app offline and recovery with a connected controller', - chromium: true, - async run({ app, controller, fixture, invoke }) { - await reset(app); - let before = fixture.statusReads(); - await invoke({ kind: 'offline', offline: true }); - try { - assert.equal(await app.evaluate(() => navigator.onLine), false); - await app.evaluate(async () => { - await window.evalI.click('Load status'); - await window.evalI.find('$(#network)').text('Offline'); - }); - assert.equal( - fixture.statusReads(), - before, - 'Offline app request must not reach the server', - ); - let connection = await controller.evaluate( - async () => - await ( - await fetch('/status?probe=' + crypto.randomUUID(), { cache: 'no-store' }) - ).json(), - ); - assert.equal(connection.status, 'Connected'); - assert.equal(connection.revision, before + 1); - } finally { - await invoke({ kind: 'offline', offline: false }); - } - await app.evaluate(async () => { - await window.evalI.click('Load status'); - await window.evalI.find('$(#network)').text('Connected'); - }); - assert.equal(fixture.statusReads(), before + 2); - }, - }, - { - name: 'below-fold tall element pixels without viewport changes', - async run({ app, invoke }) { - await reset(app); - let before = await app.evaluate(() => { - window.observedResizes = []; - window.recordResize = () => window.observedResizes.push([innerWidth, innerHeight]); - addEventListener('resize', window.recordResize); - return { width: innerWidth, height: innerHeight, x: scrollX, y: scrollY }; - }); - try { - let box = await rect(app, '#tall'); - assert.ok(box.y > before.height && box.height > before.height); - let { value } = await invoke({ kind: 'capture', clip: box }); - assert.equal(value.width, 160); - assert.equal(value.height, 1500); - assert.deepEqual(value.bands, [ - [180, 40, 90, 255], - [20, 160, 80, 255], - [20, 60, 140, 255], - ]); - let after = await app.evaluate(async () => { - await new Promise((resolve) => - requestAnimationFrame(() => requestAnimationFrame(resolve)), - ); - return { - width: innerWidth, - height: innerHeight, - x: scrollX, - y: scrollY, - resizes: window.observedResizes, - }; - }); - assert.deepEqual( - { width: after.width, height: after.height, x: after.x, y: after.y }, - before, - ); - assert.ok( - after.resizes.every( - ([width, height]) => width === before.width && height === before.height, - ), - ); - } finally { - await app.evaluate(() => removeEventListener('resize', window.recordResize)); - } - }, - }, - { - name: 'real reduced-motion media setting', - async run({ app, invoke }) { - await reset(app); - await invoke({ kind: 'motion', value: 'reduce' }); - try { - await app.evaluate(async () => { - await window.evalI.assert( - () => - matchMedia('(prefers-reduced-motion: reduce)').matches && - getComputedStyle(document.querySelector('#motion')).animationName === 'none', - 'Real media emulation must disable CSS animation', - ); - }); - } finally { - await invoke({ kind: 'motion', value: 'no-preference' }); - } - }, - }, -]); diff --git a/benchmarks/managed/direct.js b/benchmarks/managed/direct.js deleted file mode 100644 index bc73393..0000000 --- a/benchmarks/managed/direct.js +++ /dev/null @@ -1,8 +0,0 @@ -export async function createAdapter({ perform }) { - return Object.freeze({ - name: 'direct-playwright', - clock: 'node', - invoke: perform, - dispose: async () => {}, - }); -} diff --git a/benchmarks/managed/fixture.js b/benchmarks/managed/fixture.js deleted file mode 100644 index 5a4339b..0000000 --- a/benchmarks/managed/fixture.js +++ /dev/null @@ -1,85 +0,0 @@ -import { createServer } from 'node:http'; -import { once } from 'node:events'; -import { build } from 'esbuild'; -import { fileURLToPath } from 'node:url'; - -let html = `Managed operation evaluation - -

Ready

Hover target
Capture target
Motion -
-`; - -export async function startFixture() { - let result = await build({ - stdin: { - contents: `import { Interactor } from './src/browser/interactor.js'; - let I=new Interactor({root:()=>document.body,assert:{timeout:2000}}); - window.evalI=I; - window.runLocalWorkflow=async()=>{ - await I.click('Reset example'); - await I.find('Name').then.fill('Robert'); - await I.find('Notes').then.fill('A saved note'); - await I.click('Save note'); - await I.find('$(#saved)').text('Saved Robert'); - };`, - resolveDir: fileURLToPath(new URL('../../', import.meta.url)), - }, - bundle: true, - write: false, - format: 'iife', - platform: 'browser', - target: 'es2022', - logLevel: 'silent', - }); - let records = []; - let statusReads = 0; - let server = createServer(async (request, response) => { - try { - response.setHeader('Cache-Control', 'no-store'); - let path = new URL(request.url, 'http://localhost').pathname; - if (path === '/local.js') { - response.setHeader('Content-Type', 'text/javascript'); - response.end(result.outputFiles[0].text); - } else if (path === '/controller') { - response.setHeader('Content-Type', 'text/html'); - response.end('Controller

Evaluation controller

'); - } else if (path === '/status') { - response.setHeader('Content-Type', 'application/json'); - response.end(JSON.stringify({ status: 'Connected', revision: ++statusReads })); - } else if (path === '/records') { - response.setHeader('Content-Type', 'application/json'); - if (request.method === 'POST') { - let chunks = []; - for await (let chunk of request) chunks.push(chunk); - let record = JSON.parse(Buffer.concat(chunks).toString()); - records = [...records, Object.freeze(record)]; - response.end(JSON.stringify(record)); - } else response.end(JSON.stringify(records)); - } else { - response.setHeader('Content-Type', 'text/html'); - response.end(html); - } - } catch (error) { - response.writeHead(500); - response.end(error.message); - } - }); - server.listen(0, '127.0.0.1'); - await once(server, 'listening'); - return Object.freeze({ - origin: `http://localhost:${server.address().port}`, - statusReads: () => statusReads, - close: () => - new Promise((resolve) => { - server.closeAllConnections(); - server.close(resolve); - }), - }); -} diff --git a/benchmarks/managed/operations.js b/benchmarks/managed/operations.js deleted file mode 100644 index 0a4e39d..0000000 --- a/benchmarks/managed/operations.js +++ /dev/null @@ -1,86 +0,0 @@ -import { PNG } from 'pngjs'; - -// This is the reference evaluator, not a shipped Dyno browser API. -export function createOperations({ context, app, engine }) { - let session; - let authenticator; - let cdp = async () => { - if (engine !== 'chromium') - throw Object.assign(new Error('This reference requires Chromium CDP.'), { - code: 'UNSUPPORTED', - }); - session ??= await context.newCDPSession(app); - return session; - }; - return async (input) => { - switch (input.kind) { - case 'key': - await app.keyboard.press(input.key); - return null; - case 'pointer': - await app.mouse.move(input.x, input.y); - return null; - case 'permissions': - await context.grantPermissions(input.permissions, { origin: input.origin }); - return null; - case 'authenticator': { - let client = await cdp(); - await client.send('WebAuthn.enable'); - ({ authenticatorId: authenticator } = await client.send( - 'WebAuthn.addVirtualAuthenticator', - { - options: { - protocol: 'ctap2', - transport: 'internal', - hasResidentKey: true, - hasUserVerification: true, - isUserVerified: true, - automaticPresenceSimulation: true, - }, - }, - )); - return authenticator; - } - case 'remove-authenticator': { - if (authenticator) - await (await cdp()).send('WebAuthn.removeVirtualAuthenticator', { - authenticatorId: authenticator, - }); - authenticator = undefined; - return null; - } - case 'offline': { - let client = await cdp(); - await client.send('Network.enable'); - await client.send('Network.emulateNetworkConditions', { - offline: input.offline, - latency: 0, - downloadThroughput: -1, - uploadThroughput: -1, - }); - return null; - } - case 'capture': { - let bytes = await app.screenshot({ - clip: input.clip, - fullPage: true, - animations: 'disabled', - caret: 'hide', - }); - let image = PNG.sync.read(bytes); - let pixel = (y) => [...image.data.subarray(y * image.width * 4, y * image.width * 4 + 4)]; - return { - width: image.width, - height: image.height, - pixel: pixel(0), - bands: image.height >= 1500 ? [pixel(10), pixel(600), pixel(1200)] : [], - }; - } - case 'motion': - await app.emulateMedia({ reducedMotion: input.value }); - return null; - default: - throw new Error(`Unknown evaluation operation: ${input.kind}`); - } - }; -} diff --git a/benchmarks/managed/results/binding-chromium.json b/benchmarks/managed/results/binding-chromium.json deleted file mode 100644 index 29779e1..0000000 --- a/benchmarks/managed/results/binding-chromium.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "schemaVersion": 1, - "status": "passed", - "adapter": "controller-playwright-binding", - "callerClock": "browser", - "suiteFingerprint": "fe7df6d39c22c4162f266a8a4b744971e237b41e188a457eaecd31f3957f4036", - "engine": "chromium", - "browserVersion": "156.0.8078.4", - "playwright": "1.64.1", - "node": "v24.22.1", - "platform": "darwin", - "arch": "arm64", - "cpu": "Apple M4 Max", - "samples": 20, - "setupDuration": 289.087667, - "totalDuration": 29942.912875, - "results": [ - { - "name": "browser-local Interactor control before", - "status": "passed", - "caller": "app-window-performance", - "workflow": { - "count": 20, - "median": 250.89999997615814, - "p95": 258.7999999523163, - "values": [ - 245.69999992847443, - 251.20000004768372, - 247.20000004768372, - 252.60000002384186, - 243.5, - 250, - 251, - 255.10000002384186, - 250.70000004768372, - 258.60000002384186, - 256, - 247.5, - 250.89999997615814, - 256.60000002384186, - 253.39999997615814, - 248.69999992847443, - 250.79999995231628, - 260.90000009536743, - 258.7999999523163, - 248.69999992847443 - ] - }, - "operations": [] - }, - { - "name": "native Tab and Space workflow", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 179.43912499999988, - "p95": 195.04899999999998, - "values": [ - 184.97345799999948, - 196.82175000000007, - 185.74837500000012, - 178.34108300000025, - 183.80554200000006, - 179.43912499999988, - 188.67091700000037, - 195.04899999999998, - 191.97212499999932, - 190.80162500000006, - 187.5704169999999, - 166.95437500000025, - 171.72341699999924, - 167.67075000000114, - 171.49820799999907, - 170.94412499999999, - 179.75704199999927, - 168.1431249999987, - 154.15979200000038, - 168.57300000000032 - ] - }, - "operations": [ - { - "kind": "key", - "count": 60, - "median": 2.1999999284744263, - "p95": 4.699999928474426, - "values": [ - 2.6999999284744263, - 1.2000000476837158, - 3.799999952316284, - 3.0999999046325684, - 2.600000023841858, - 1.3000000715255737, - 3.399999976158142, - 2.1999999284744263, - 2.299999952316284, - 2.200000047683716, - 1.600000023841858, - 2.8000000715255737, - 2, - 2, - 4.799999952316284, - 2.799999952316284, - 1.9000000953674316, - 2.700000047683716, - 2.5, - 3.299999952316284, - 5.299999952316284, - 4.699999928474426, - 4.599999904632568, - 2.700000047683716, - 4.199999928474426, - 3.1999999284744263, - 3.6999999284744263, - 5.299999952316284, - 2.5999999046325684, - 3.299999952316284, - 3.299999952316284, - 2.1999999284744263, - 2.3000000715255737, - 4.5, - 1.1999999284744263, - 1.5, - 0.8999999761581421, - 1.100000023841858, - 2, - 0.9000000953674316, - 1, - 1.5, - 1.1999999284744263, - 1.3000000715255737, - 1.5999999046325684, - 1.3000000715255737, - 1.1999999284744263, - 2.299999952316284, - 1.8000000715255737, - 1.7999999523162842, - 3.5999999046325684, - 1.2999999523162842, - 1.2000000476837158, - 2.1999999284744263, - 1.600000023841858, - 1.4000000953674316, - 3.5, - 1.100000023841858, - 0.8999999761581421, - 1.2000000476837158 - ] - } - ] - }, - { - "name": "real CSS hover and pointer exit", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 133.49812500000007, - "p95": 138.5555000000004, - "values": [ - 132.21700000000055, - 134.55175000000054, - 133.60508400000072, - 134.7969580000008, - 138.5555000000004, - 143.90320900000006, - 132.3892079999987, - 132.25116700000035, - 134.6829170000001, - 133.19845799999894, - 135.67324999999983, - 129.179583000001, - 138.32116700000006, - 133.49812500000007, - 132.05145900000025, - 130.98562499999935, - 131.99520800000028, - 134.40366600000016, - 136.33670800000073, - 131.13904100000036 - ] - }, - "operations": [ - { - "kind": "pointer", - "count": 40, - "median": 12.300000071525574, - "p95": 16.199999928474426, - "values": [ - 11.200000047683716, - 15.199999928474426, - 12.299999952316284, - 14.5, - 11.899999976158142, - 14.899999976158142, - 10.300000071525574, - 16.199999928474426, - 3.5, - 16.300000071525574, - 14.5, - 13.399999976158142, - 8.399999976158142, - 12, - 9.899999976158142, - 16.600000023841858, - 11.100000023841858, - 12.300000071525574, - 10.399999976158142, - 14.600000023841858, - 8.700000047683716, - 16, - 6.5, - 14.799999952316284, - 14.799999952316284, - 13.5, - 2.600000023841858, - 14.600000023841858, - 3.700000047683716, - 12.299999952316284, - 8.600000023841858, - 14.899999976158142, - 13, - 15.299999952316284, - 14.299999952316284, - 15.5, - 9.5, - 8.800000071525574, - 5.299999952316284, - 15.600000023841858 - ] - } - ] - }, - { - "name": "real clipboard permission and copy", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 115.87775000000147, - "p95": 126.21029100000123, - "values": [ - 128.63383299999987, - 121.5395840000001, - 109.35429199999999, - 113.89987499999916, - 107.49133299999994, - 110.09108399999968, - 118.43529099999978, - 119.58941700000105, - 126.21029100000123, - 115.87775000000147, - 118.40316699999858, - 105.66029100000014, - 113.39016699999956, - 122.82974999999897, - 120.85649999999987, - 119.64737500000047, - 116.59195799999907, - 107.78420799999913, - 111.15437500000007, - 111.57433400000082 - ] - }, - "operations": [ - { - "kind": "permissions", - "count": 20, - "median": 1.5, - "p95": 3, - "values": [ - 3.6999999284744263, - 2.8000000715255737, - 1.5, - 1.5, - 0.8000000715255737, - 1.2000000476837158, - 1.1999999284744263, - 2.299999952316284, - 2.8000000715255737, - 1.6999999284744263, - 2.799999952316284, - 0.7000000476837158, - 1.2999999523162842, - 1.5, - 1.7999999523162842, - 2.100000023841858, - 3, - 0.6999999284744263, - 0.8000000715255737, - 0.7999999523162842 - ] - } - ] - }, - { - "name": "virtual WebAuthn with actual credential and assertion", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 98.10404199999903, - "p95": 105.64354199999798, - "values": [ - 96.26520900000105, - 90.77279199999975, - 109.59354200000053, - 99.56554199999846, - 96.00454100000024, - 99.64854100000048, - 100.27512500000012, - 99.22041700000045, - 104.21245900000031, - 92.51854099999946, - 94.93537499999911, - 97.44695800000045, - 97.93224999999984, - 95.35737499999959, - 98.9903749999994, - 105.64354199999798, - 98.10404199999903, - 99.21241700000246, - 101.19404100000247, - 98.0525830000006 - ] - }, - "operations": [ - { - "kind": "authenticator", - "count": 20, - "median": 0.7999999523162842, - "p95": 1, - "values": [ - 0.9000000953674316, - 0.6000000238418579, - 0.6999999284744263, - 0.8000000715255737, - 0.6999999284744263, - 0.5999999046325684, - 0.7000000476837158, - 0.7000000476837158, - 1, - 0.9000000953674316, - 0.7000000476837158, - 0.8000000715255737, - 0.7000000476837158, - 0.7999999523162842, - 0.6999999284744263, - 1, - 0.7999999523162842, - 1, - 1, - 1.0999999046325684 - ] - }, - { - "kind": "remove-authenticator", - "count": 20, - "median": 0.7000000476837158, - "p95": 0.8999999761581421, - "values": [ - 0.6999999284744263, - 0.7000000476837158, - 1, - 0.6999999284744263, - 0.6999999284744263, - 0.8000000715255737, - 0.8999999761581421, - 0.6000000238418579, - 0.7000000476837158, - 0.5999999046325684, - 0.6000000238418579, - 0.7000000476837158, - 0.8000000715255737, - 0.6999999284744263, - 0.7000000476837158, - 0.7000000476837158, - 0.7999999523162842, - 0.8999999761581421, - 0.8000000715255737, - 0.7000000476837158 - ] - } - ] - }, - { - "name": "app offline and recovery with a connected controller", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 199.57887499999924, - "p95": 205.94820899999831, - "values": [ - 185.58845900000233, - 201.85183300000062, - 197.69724999999744, - 199.56154199999946, - 202.52337500000067, - 201.5985830000027, - 196.5444580000003, - 199.87791700000162, - 201.7516250000008, - 199.57887499999924, - 205.94820899999831, - 192.3492499999993, - 200.34904200000165, - 197.98562500000116, - 201.7070829999975, - 199.30525000000125, - 201.10420800000065, - 198.6389999999992, - 197.24491599999965, - 206.11158299999806 - ] - }, - "operations": [ - { - "kind": "offline", - "count": 40, - "median": 2.399999976158142, - "p95": 4, - "values": [ - 0.8000000715255737, - 3.100000023841858, - 2.899999976158142, - 2.100000023841858, - 3.299999952316284, - 2.8000000715255737, - 1.5, - 1.899999976158142, - 2.899999976158142, - 1.8000000715255737, - 3.100000023841858, - 2.399999976158142, - 2, - 2, - 3.4000000953674316, - 2.100000023841858, - 3.200000047683716, - 2.399999976158142, - 3.6999999284744263, - 2, - 4, - 2, - 1.5, - 1.899999976158142, - 3.5, - 1.899999976158142, - 1.899999976158142, - 2, - 3.899999976158142, - 2.100000023841858, - 3, - 2.299999952316284, - 2.200000047683716, - 4.399999976158142, - 3, - 2.600000023841858, - 2.700000047683716, - 3, - 4.100000023841858, - 2.899999976158142 - ] - } - ] - }, - { - "name": "below-fold tall element pixels without viewport changes", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 99.89879099999962, - "p95": 100.49887500000114, - "values": [ - 99.03604199999972, - 99.69370900000285, - 99.7567909999998, - 100.49887500000114, - 99.63508399999773, - 100.21674999999959, - 99.86025000000154, - 99.79662499999904, - 100.05233300000327, - 100.21262500000012, - 100.48300000000017, - 99.8615830000017, - 99.89879099999962, - 100.5458749999998, - 99.32341599999927, - 100.38666599999851, - 99.85200000000259, - 99.95308300000033, - 99.9957089999989, - 100.08437499999854 - ] - }, - "operations": [ - { - "kind": "capture", - "count": 20, - "median": 39.299999952316284, - "p95": 42.90000009536743, - "values": [ - 38.699999928474426, - 40.40000009536743, - 43.10000002384186, - 39.299999952316284, - 40.59999990463257, - 39.90000009536743, - 39.39999997615814, - 40.10000002384186, - 38.200000047683716, - 40.5, - 42.90000009536743, - 38.300000071525574, - 37.39999997615814, - 41.09999990463257, - 38.699999928474426, - 40.300000071525574, - 37.5, - 38.200000047683716, - 36.299999952316284, - 38.199999928474426 - ] - } - ] - }, - { - "name": "real reduced-motion media setting", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 59.59429200000159, - "p95": 64.71541699999943, - "values": [ - 59.452041999997164, - 59.36579200000051, - 61.130042000000685, - 63.65345900000102, - 57.71679199999926, - 56.88487499999974, - 54.746791999998095, - 61.217749999999796, - 58.20558300000266, - 60.491000000001804, - 56.3449580000015, - 57.91666600000099, - 60.95333400000163, - 64.71541699999943, - 64.59316700000272, - 61.446875000001455, - 59.33008300000074, - 64.45033400000102, - 59.59429200000159, - 67.04187500000262 - ] - }, - "operations": [ - { - "kind": "motion", - "count": 40, - "median": 1.2999999523162842, - "p95": 2.6999999284744263, - "values": [ - 1.100000023841858, - 1, - 1.3000000715255737, - 1, - 1.2999999523162842, - 1.2999999523162842, - 1.600000023841858, - 1.3000000715255737, - 0.7999999523162842, - 0.6999999284744263, - 0.7999999523162842, - 1.100000023841858, - 0.7999999523162842, - 1, - 1.2000000476837158, - 1.2000000476837158, - 1.2000000476837158, - 1.1999999284744263, - 1.2999999523162842, - 1.600000023841858, - 0.8999999761581421, - 0.8999999761581421, - 0.7999999523162842, - 1, - 1.8000000715255737, - 1.2999999523162842, - 2.799999952316284, - 1.600000023841858, - 2.6999999284744263, - 1.5, - 1.4000000953674316, - 1.899999976158142, - 1.2000000476837158, - 1.399999976158142, - 2.5, - 1.5, - 0.7999999523162842, - 1.600000023841858, - 2.8000000715255737, - 1.7000000476837158 - ] - } - ] - }, - { - "name": "browser-local Interactor control after", - "status": "passed", - "caller": "app-window-performance", - "workflow": { - "count": 20, - "median": 250.70000004768372, - "p95": 255.59999990463257, - "values": [ - 255.59999990463257, - 253.60000002384186, - 256.7000000476837, - 251, - 251.5, - 249.39999997615814, - 250.60000002384186, - 246, - 252.89999997615814, - 250.70000004768372, - 248.60000002384186, - 249.90000009536743, - 254.39999997615814, - 253.5, - 253, - 250.5, - 250.89999997615814, - 250.09999990463257, - 249.5, - 250.10000002384186 - ] - }, - "operations": [] - } - ], - "cleanup": [ - { - "name": "adapter", - "status": "passed" - }, - { - "name": "browser", - "status": "passed" - }, - { - "name": "fixture", - "status": "passed" - } - ] -} diff --git a/benchmarks/managed/results/cross-browser-probes.json b/benchmarks/managed/results/cross-browser-probes.json deleted file mode 100644 index 8a059e4..0000000 --- a/benchmarks/managed/results/cross-browser-probes.json +++ /dev/null @@ -1,166 +0,0 @@ -[ - { - "engine": "chromium", - "profileMode": "explicit disposable profile", - "version": "156.0.8078.4", - "initialBinding": { - "sameOrigin": true, - "title": "Managed operation evaluation" - }, - "syntheticSpaceRecords": 0, - "nativeSpaceRecords": 1, - "nativeTabFocus": "notes", - "cssHover": "rgb(20, 160, 80)", - "contextOffline": { - "app": { - "ok": false, - "error": "Failed to fetch", - "online": false - }, - "controller": { - "ok": false, - "error": "Failed to fetch", - "online": false - } - }, - "contextOnlineRestored": { - "app": { - "ok": true, - "value": { - "status": "Connected", - "revision": 1 - }, - "online": true - }, - "controller": { - "ok": true, - "value": { - "status": "Connected", - "revision": 2 - }, - "online": true - } - }, - "cdp": { - "supported": true - }, - "appOffline": { - "app": { - "ok": false, - "error": "Failed to fetch", - "online": false - }, - "controller": { - "ok": true, - "value": { - "status": "Connected", - "revision": 3 - }, - "online": false - } - }, - "restored": { - "ok": true, - "value": { - "status": "Connected", - "revision": 4 - }, - "online": true - }, - "virtualAuthenticator": "created and removed; credential flow belongs to fixed evaluator", - "clipboardPermissions": { - "supported": true - }, - "opaqueNavigationBinding": { - "accessible": false, - "name": "SecurityError" - }, - "opaqueTopLevelSubmission": { - "origin": "null", - "body": "login=test%40example.test&password=fixture-password" - }, - "sandboxSubmission": { - "origin": "null", - "body": "login=test%40example.test&password=fixture-password" - }, - "sandboxBinding": { - "sameOrigin": true, - "title": "Managed operation evaluation" - }, - "verdict": "passed" - }, - { - "engine": "firefox", - "profileMode": "SDK temporary persistent profile", - "verdict": "failed", - "error": "browserType.launchPersistentContext: Failed to launch the browser process.\nBrowser logs:\n\n /Users/robertdeluca/Library/Caches/ms-playwright/firefox-1555/firefox/Nightly.app/Contents/MacOS/firefox -no-remote -headless -profile /var/folders/44/ng7yt98j78z5kjhptpztzm240000gn/T/playwright_firefoxdev_profile-1njhdA -juggler-pipe about:blank\n pid=60383\n[pid=60383][err] *** You are running in headless mode.\n[pid=60383][err] Could not find profile folder.\n[pid=60383] \n[pid=60383] starting temporary directories cleanup\nCall log:\n\u001b[2m - /Users/robertdeluca/Library/Caches/ms-playwright/firefox-1555/firefox/Nightly.app/Contents/MacOS/firefox -no-remote -headless -profile /var/folders/44/ng7yt98j78z5kjhptpztzm240000gn/T/playwright_firefoxdev_profile-1njhdA -juggler-pipe about:blank\u001b[22m\n\u001b[2m - pid=60383\u001b[22m\n\u001b[2m - [pid=60383][err] *** You are running in headless mode.\u001b[22m\n\u001b[2m - [pid=60383][err] Could not find profile folder.\u001b[22m\n\u001b[2m - [pid=60383] \u001b[22m\n\u001b[2m - [pid=60383] starting temporary directories cleanup\u001b[22m\n\u001b[2m - [pid=60383] \u001b[22m\n\u001b[2m - [pid=60383] \u001b[22m\n\u001b[2m - [pid=60383] \u001b[22m\n\u001b[2m - [pid=60383] finished temporary directories cleanup\u001b[22m\n\u001b[2m - [pid=60383] \u001b[22m\n\n at async TestContext. (/private/tmp/dyno-native-spike-QyJmvk/adversarial/adversarial.js:33:17)" - }, - { - "engine": "webkit", - "profileMode": "explicit disposable profile", - "version": "27.2", - "initialBinding": { - "sameOrigin": true, - "title": "Managed operation evaluation" - }, - "syntheticSpaceRecords": 0, - "nativeSpaceRecords": 1, - "nativeTabFocus": "notes", - "cssHover": "rgb(20, 160, 80)", - "contextOffline": { - "app": { - "ok": false, - "error": "Load failed", - "online": false - }, - "controller": { - "ok": false, - "error": "Load failed", - "online": false - } - }, - "contextOnlineRestored": { - "app": { - "ok": true, - "value": { - "status": "Connected", - "revision": 1 - }, - "online": true - }, - "controller": { - "ok": true, - "value": { - "status": "Connected", - "revision": 2 - }, - "online": true - } - }, - "cdp": { - "supported": false, - "error": "browserContext.newCDPSession: CDP session is only available in Chromium" - }, - "clipboardPermissions": { - "supported": false, - "error": "browserContext.grantPermissions: Unknown permission: clipboard-write" - }, - "opaqueNavigationBinding": { - "accessible": false, - "name": "SecurityError" - }, - "opaqueTopLevelSubmission": { - "origin": "null", - "body": "login=test%40example.test&password=fixture-password" - }, - "sandboxSubmission": { - "origin": "null", - "body": "login=test%40example.test&password=fixture-password" - }, - "sandboxBinding": { - "sameOrigin": true, - "title": "Managed operation evaluation" - }, - "verdict": "passed" - } -] diff --git a/benchmarks/managed/results/direct-chromium.json b/benchmarks/managed/results/direct-chromium.json deleted file mode 100644 index e58a3aa..0000000 --- a/benchmarks/managed/results/direct-chromium.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "schemaVersion": 1, - "status": "passed", - "adapter": "direct-playwright", - "callerClock": "node", - "suiteFingerprint": "fe7df6d39c22c4162f266a8a4b744971e237b41e188a457eaecd31f3957f4036", - "engine": "chromium", - "browserVersion": "156.0.8078.4", - "playwright": "1.64.1", - "node": "v24.22.1", - "platform": "darwin", - "arch": "arm64", - "cpu": "Apple M4 Max", - "samples": 20, - "setupDuration": 577.598542, - "totalDuration": 29950.618084, - "results": [ - { - "name": "browser-local Interactor control before", - "status": "passed", - "caller": "app-window-performance", - "workflow": { - "count": 20, - "median": 252.70000004768372, - "p95": 259.2999999523163, - "values": [ - 247, - 247.20000004768372, - 256.7999999523163, - 254.20000004768372, - 254.5, - 251.19999992847443, - 252.70000004768372, - 246.39999997615814, - 248.20000004768372, - 254.10000002384186, - 253.70000004768372, - 248.10000002384186, - 256.3000000715256, - 251.5, - 262.10000002384186, - 259.2999999523163, - 251.79999995231628, - 255.30000007152557, - 253.70000004768372, - 252 - ] - }, - "operations": [] - }, - { - "name": "native Tab and Space workflow", - "status": "passed", - "caller": "node-performance", - "workflow": { - "count": 20, - "median": 185.2092909999992, - "p95": 192.51608300000044, - "values": [ - 163.0922500000006, - 187.89024999999947, - 191.14820800000052, - 171.3306250000005, - 178.08395800000017, - 181.8387499999999, - 184.23691700000018, - 187.68866699999944, - 179.00741600000038, - 189.87229200000002, - 173.58495800000037, - 178.8645000000006, - 192.79316599999947, - 188.05274999999892, - 192.51608300000044, - 185.93191699999988, - 187.9631249999984, - 167.86083300000064, - 185.2092909999992, - 185.59020899999996 - ] - }, - "operations": [ - { - "kind": "key", - "count": 60, - "median": 2.2269579999992857, - "p95": 4.48608299999978, - "values": [ - 3.203084000000672, - 2.037790999999743, - 4.574749999999767, - 2.6851669999996375, - 4.183334000000286, - 3.4178750000000946, - 2.937042000000474, - 2.0730419999999867, - 3.3354580000004717, - 0.9625419999993028, - 0.8322079999998095, - 1.8933750000005602, - 2.536000000000058, - 1.2020829999992202, - 2.6572079999996276, - 1.265374999999949, - 1.1323750000001382, - 3.8475420000004306, - 2.489708000000064, - 2.5364580000004935, - 1.7832909999997355, - 2.757832999999664, - 2.0895000000000437, - 2.22649999999976, - 1.7415410000003249, - 1.4298340000004828, - 1.8525419999996302, - 2.220041000000492, - 2.345542000000023, - 5.188125000000582, - 1.1945830000004207, - 1.2944170000000668, - 2.4246250000005602, - 2.0567080000000715, - 2.764415999999983, - 0.9812500000007276, - 2.834666999999172, - 2.6800830000011047, - 4.48608299999978, - 2.0774169999986043, - 2.5815419999998994, - 3.5030839999999444, - 2.2932919999984733, - 1.9862919999995938, - 2.371584000000439, - 2.627250000001368, - 1.6567500000001019, - 2.2269579999992857, - 2.103624999999738, - 1.9935829999994894, - 3.2327920000006998, - 2.0947919999998703, - 2.2021249999997963, - 6.302874999999403, - 2.156290999999328, - 2.7663329999995767, - 3.9274999999997817, - 2.19866699999875, - 2.7478330000012647, - 1.8063330000004498 - ] - } - ] - }, - { - "name": "real CSS hover and pointer exit", - "status": "passed", - "caller": "node-performance", - "workflow": { - "count": 20, - "median": 134.46095799999966, - "p95": 148.62395800000013, - "values": [ - 135.7430000000004, - 135.60004200000003, - 129.95800000000054, - 137.84170799999993, - 144.11645899999894, - 131.99816699999974, - 134.12729200000103, - 132.51541700000053, - 138.26241599999958, - 150.21637499999997, - 132.49208299999918, - 148.62395800000013, - 129.82087499999943, - 133.373834, - 133.61470799999915, - 134.0496660000008, - 138.76766700000007, - 148.02158299999974, - 134.46095799999966, - 147.2242499999993 - ] - }, - "operations": [ - { - "kind": "pointer", - "count": 40, - "median": 12.181415999999444, - "p95": 17.695292000000336, - "values": [ - 5.208457999999155, - 11.778417000001355, - 3.2746659999993426, - 16.586332999999286, - 5.752375000000029, - 16.63812500000131, - 2.6402500000003783, - 12.74029199999859, - 17.695292000000336, - 11.523417000000336, - 4.1996670000007725, - 12.181415999999444, - 10.310375000000931, - 14.962750000000597, - 6.210250000000087, - 13.569666999999754, - 9.545125000000553, - 15.878375000000233, - 18.031417000000147, - 11.975083999999697, - 2.692583999998533, - 16.341749999999593, - 17.913666000000376, - 11.88195799999994, - 4.721917000000758, - 12.20720799999981, - 13.215958000000683, - 16.594458999999915, - 12.10950000000048, - 15.348625000000538, - 10.109916000001249, - 15.842415999999503, - 8.411167000000205, - 14.969874999998865, - 16.412124999998923, - 11.801417000000583, - 3.1448330000002898, - 15.328292000000147, - 16.638291999999637, - 12.927084000000832 - ] - } - ] - }, - { - "name": "real clipboard permission and copy", - "status": "passed", - "caller": "node-performance", - "workflow": { - "count": 20, - "median": 116.29683300000033, - "p95": 120.81341699999939, - "values": [ - 116.79383400000006, - 119.76425000000017, - 119.5805419999997, - 115.36629100000027, - 112.14704200000051, - 114.63629199999923, - 120.81341699999939, - 122.2245839999996, - 117.87533299999996, - 111.79320800000096, - 116.18995800000084, - 116.29683300000033, - 120.13854100000026, - 118.5317500000001, - 118.46912499999962, - 110.31462499999907, - 113.45820800000001, - 112.9272089999995, - 115.90220799999952, - 116.80420799999956 - ] - }, - "operations": [ - { - "kind": "permissions", - "count": 20, - "median": 1.0088329999998678, - "p95": 1.6349579999987327, - "values": [ - 1.0803750000013679, - 1.6349579999987327, - 0.6947500000005675, - 1.0302089999986492, - 1.03216699999939, - 0.6412919999984297, - 0.8571249999986321, - 1.2017090000008466, - 1.126791000000594, - 0.6628749999999854, - 1.0497919999997976, - 0.9938750000001164, - 0.9494580000009591, - 1.2349169999997684, - 2.2287080000005517, - 1.0088329999998678, - 0.321457999998529, - 0.8031250000003638, - 0.3802079999986745, - 1.1738330000007409 - ] - } - ] - }, - { - "name": "virtual WebAuthn with actual credential and assertion", - "status": "passed", - "caller": "node-performance", - "workflow": { - "count": 20, - "median": 92.64991700000246, - "p95": 97.94737499999974, - "values": [ - 93.18054200000006, - 96.69208299999991, - 96.63541600000099, - 92.29433300000164, - 95.38000000000102, - 94.7059160000008, - 90.01079200000095, - 93.69020799999998, - 87.74616700000115, - 92.43383300000096, - 91.79500000000007, - 92.33333399999901, - 90.63595799999894, - 97.94737499999974, - 91.3603750000002, - 92.77654199999961, - 98.12470900000335, - 91.75091699999757, - 92.64991700000246, - 92.80854099999851 - ] - }, - "operations": [ - { - "kind": "authenticator", - "count": 20, - "median": 0.4124999999985448, - "p95": 0.5098330000000715, - "values": [ - 0.4756249999991269, - 0.43874999999934516, - 0.5102499999993597, - 0.42862499999864667, - 0.3906669999996666, - 0.45241600000008475, - 0.40333300000020245, - 0.39112500000010186, - 0.33924999999908323, - 0.3473329999997077, - 0.3322090000001481, - 0.4324589999996533, - 0.35783299999820883, - 0.44304099999862956, - 0.4156670000011218, - 0.3850830000010319, - 0.5098330000000715, - 0.43679100000008475, - 0.4096659999995609, - 0.4124999999985448 - ] - }, - { - "kind": "remove-authenticator", - "count": 20, - "median": 0.24720800000068266, - "p95": 0.26466699999946286, - "values": [ - 0.25916700000016135, - 0.306416999999783, - 0.2391669999997248, - 0.24754200000097626, - 0.24308299999938754, - 0.2557919999999285, - 0.23562499999934516, - 0.2398750000011205, - 0.2556249999997817, - 0.25962500000059663, - 0.23520800000187592, - 0.22416700000030687, - 0.24499999999898137, - 0.2634170000019367, - 0.24720800000068266, - 0.212374999999156, - 0.26466699999946286, - 0.26120799999989686, - 0.25383399999918765, - 0.24141600000075414 - ] - } - ] - }, - { - "name": "app offline and recovery with a connected controller", - "status": "passed", - "caller": "node-performance", - "workflow": { - "count": 20, - "median": 183.8806659999973, - "p95": 196.92733399999997, - "values": [ - 196.92733399999997, - 186.00391699999818, - 180.57187499999782, - 186.5209590000013, - 182.77941700000156, - 183.8806659999973, - 183.04399999999805, - 183.23108299999876, - 183.304874999998, - 183.14783299999908, - 197.4046249999992, - 185.46116599999732, - 183.97033300000112, - 183.0728339999987, - 183.56091699999888, - 190.30520799999795, - 195.8375840000008, - 187.387999999999, - 179.71920800000225, - 188.6694580000003 - ] - }, - "operations": [ - { - "kind": "offline", - "count": 40, - "median": 0.7266669999989972, - "p95": 2.4191250000003492, - "values": [ - 1.0482080000001588, - 1.2342089999983727, - 4.242957999998907, - 0.5797920000004524, - 0.560750000000553, - 0.46199999999953434, - 0.8086249999978463, - 0.8349579999994603, - 0.6605420000014419, - 0.6011660000003758, - 0.5836669999989681, - 0.6046659999992698, - 0.835332999999082, - 0.517041999999492, - 0.7079589999993914, - 0.6132909999978438, - 1.012125000001106, - 0.5614170000008016, - 0.7150839999994787, - 0.7266669999989972, - 1.8207089999996242, - 1.4415829999998095, - 0.6530840000013995, - 0.6888330000001588, - 1.1551669999971637, - 0.8277920000000449, - 0.7871670000022277, - 0.6113339999974414, - 0.5357499999990978, - 0.6897910000006959, - 1.1531660000000556, - 1.7938330000033602, - 2.9025409999994736, - 1.8424999999988358, - 2.4191250000003492, - 0.6282499999979336, - 0.9153750000004948, - 0.5010419999998703, - 0.907959000000119, - 0.7475830000003043 - ] - } - ] - }, - { - "name": "below-fold tall element pixels without viewport changes", - "status": "passed", - "caller": "node-performance", - "workflow": { - "count": 20, - "median": 99.92349999999715, - "p95": 100.50129200000083, - "values": [ - 99.62141700000211, - 100.21958299999824, - 100.3928339999984, - 99.43341700000019, - 100.40641599999799, - 99.78758299999754, - 99.61408300000039, - 100.09287499999846, - 99.98287500000151, - 100.21979199999987, - 99.92349999999715, - 99.55004200000258, - 100.50129200000083, - 100.35833300000013, - 99.57854199999929, - 99.97404199999801, - 99.86524999999892, - 99.67883399999846, - 99.91008399999919, - 100.62587500000154 - ] - }, - "operations": [ - { - "kind": "capture", - "count": 20, - "median": 39.47558299999946, - "p95": 44.018041999999696, - "values": [ - 38.56770800000231, - 40.770166999998764, - 40.000041000002966, - 39.9542079999992, - 39.465000000000146, - 38.752292000001034, - 39.57995899999878, - 44.018041999999696, - 41.825042000000394, - 41.86170800000036, - 38.115583000002516, - 39.72141600000032, - 41.78929099999732, - 39.47558299999946, - 33.86487500000294, - 37.46925000000192, - 36.67741599999863, - 34.97495799999888, - 36.596958999998606, - 44.39137500000288 - ] - } - ] - }, - { - "name": "real reduced-motion media setting", - "status": "passed", - "caller": "node-performance", - "workflow": { - "count": 20, - "median": 59.6070409999993, - "p95": 61.95966600000247, - "values": [ - 55.04954199999702, - 60.526459000000614, - 57.655665999998746, - 59.6070409999993, - 57.471832999999606, - 61.95966600000247, - 56.97745899999791, - 61.39345800000228, - 60.52283299999908, - 59.23466699999699, - 61.274334000001545, - 57.89587500000198, - 60.79408300000068, - 58.13812500000131, - 58.586332999999286, - 60.44308400000227, - 58.33629199999996, - 61.94379099999787, - 64.37583300000188, - 59.8057919999992 - ] - }, - "operations": [ - { - "kind": "motion", - "count": 40, - "median": 0.9339590000017779, - "p95": 1.7959589999991294, - "values": [ - 0.790499999999156, - 0.8401249999988067, - 0.7923749999972642, - 1.3769169999977748, - 1.2150829999991402, - 0.8069999999970605, - 0.9252079999969283, - 1.386040999997931, - 1.0582499999982247, - 0.7328749999978754, - 1.30120800000077, - 0.7611670000005688, - 0.6778329999979178, - 0.6573329999991984, - 2.5884579999983544, - 0.8422089999985474, - 1.3190419999991718, - 0.7720420000005106, - 1.175374999998894, - 0.7842909999999392, - 1.3344579999975394, - 1.0143750000024738, - 0.785042000003159, - 0.8379580000000715, - 1.2695000000021537, - 0.9339590000017779, - 1.7959589999991294, - 0.7272089999969467, - 1.5401249999995343, - 0.6990829999995185, - 0.9661250000026484, - 0.7722499999981665, - 1.1432919999970181, - 0.8077910000029078, - 0.8081249999995634, - 1.2097079999984999, - 3.35029200000281, - 1.2693749999998545, - 1.4818749999976717, - 1.7856670000001031 - ] - } - ] - }, - { - "name": "browser-local Interactor control after", - "status": "passed", - "caller": "app-window-performance", - "workflow": { - "count": 20, - "median": 249.5, - "p95": 254.79999995231628, - "values": [ - 246.60000002384186, - 247.29999995231628, - 251.10000002384186, - 254.79999995231628, - 249, - 248, - 250.40000009536743, - 249.5, - 253, - 250.09999990463257, - 252.80000007152557, - 246.89999997615814, - 241.70000004768372, - 251.10000002384186, - 246.70000004768372, - 248.20000004768372, - 249.60000002384186, - 246.90000009536743, - 252.20000004768372, - 256 - ] - }, - "operations": [] - } - ], - "cleanup": [ - { - "name": "adapter", - "status": "passed" - }, - { - "name": "browser", - "status": "passed" - }, - { - "name": "fixture", - "status": "passed" - } - ] -} diff --git a/benchmarks/managed/results/http-chromium.json b/benchmarks/managed/results/http-chromium.json deleted file mode 100644 index 304dded..0000000 --- a/benchmarks/managed/results/http-chromium.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "schemaVersion": 1, - "status": "passed", - "adapter": "authenticated HTTP + queued owned worker (extra evaluator IPC)", - "callerClock": "browser", - "suiteFingerprint": "fe7df6d39c22c4162f266a8a4b744971e237b41e188a457eaecd31f3957f4036", - "engine": "chromium", - "browserVersion": "156.0.8078.4", - "playwright": "1.64.1", - "node": "v24.22.1", - "platform": "darwin", - "arch": "arm64", - "cpu": "Apple M4 Max", - "samples": 20, - "setupDuration": 576.969542, - "totalDuration": 30451.181875, - "results": [ - { - "name": "browser-local Interactor control before", - "status": "passed", - "caller": "app-window-performance", - "workflow": { - "count": 20, - "median": 249.30000007152557, - "p95": 253.40000009536743, - "values": [ - 246.70000004768372, - 243.69999992847443, - 249.30000007152557, - 256.39999997615814, - 233.89999997615814, - 250.90000009536743, - 252.29999995231628, - 251.59999990463257, - 245.09999990463257, - 250.39999997615814, - 242.39999997615814, - 251.30000007152557, - 253.40000009536743, - 253.10000002384186, - 243, - 247.80000007152557, - 252.69999992847443, - 246.20000004768372, - 248.79999995231628, - 251.10000002384186 - ] - }, - "operations": [] - }, - { - "name": "native Tab and Space workflow", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 199.62212499999987, - "p95": 202.03912500000024, - "values": [ - 199.62212499999987, - 201.4695840000004, - 202.03912500000024, - 199.8389590000006, - 198.84004099999947, - 201.98491699999977, - 195.55825000000004, - 182.93583300000046, - 201.45829099999992, - 201.57025000000067, - 197.57399999999961, - 182.58308299999953, - 183.26291700000002, - 184.58979199999885, - 201.22900000000118, - 200.71512500000063, - 197.67691700000069, - 202.99099999999999, - 199.5213330000006, - 200.69774999999936 - ] - }, - "operations": [ - { - "kind": "key", - "count": 60, - "median": 13.699999928474426, - "p95": 16.600000023841858, - "values": [ - 15.399999976158142, - 10, - 15, - 16.700000047683716, - 11.399999976158142, - 12.5, - 17.299999952316284, - 9.5, - 15.899999976158142, - 12, - 11.299999952316284, - 13.5, - 14.5, - 14.100000023841858, - 12.800000071525574, - 14.200000047683716, - 13.199999928474426, - 16.300000071525574, - 14, - 13.5, - 13.399999976158142, - 8, - 15.5, - 15.199999928474426, - 19.299999952316284, - 12.300000071525574, - 15.100000023841858, - 10.5, - 15.200000047683716, - 13.5, - 14.799999952316284, - 12.199999928474426, - 13.699999928474426, - 3.8000000715255737, - 14.599999904632568, - 15, - 5.399999976158142, - 15, - 15, - 6.599999904632568, - 14.900000095367432, - 15.100000023841858, - 14.5, - 13.800000071525574, - 16.600000023841858, - 14.199999928474426, - 13.199999928474426, - 13, - 15.200000047683716, - 12.5, - 12.300000071525574, - 11.700000047683716, - 11.700000047683716, - 13.799999952316284, - 14.599999904632568, - 13.600000023841858, - 9.399999976158142, - 16.199999928474426, - 11.899999976158142, - 12.699999928474426 - ] - } - ] - }, - { - "name": "real CSS hover and pointer exit", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 136.00029200000063, - "p95": 152.83137500000157, - "values": [ - 136.00029200000063, - 150.97891699999855, - 154.96562500000073, - 144.55245800000012, - 152.83137500000157, - 148.65558299999975, - 128.6609580000004, - 135.50383299999885, - 135.0101250000007, - 131.87795800000094, - 133.67020799999955, - 138.24654200000077, - 144.71737500000017, - 130.7771249999987, - 133.11429099999987, - 140.9814580000002, - 148.55220799999915, - 148.25304099999994, - 128.95383299999958, - 133.34858299999905 - ] - }, - "operations": [ - { - "kind": "pointer", - "count": 40, - "median": 13.100000023841858, - "p95": 17.300000071525574, - "values": [ - 12.800000071525574, - 12.799999952316284, - 14.100000023841858, - 8.199999928474426, - 18.799999952316284, - 10.399999976158142, - 15.300000071525574, - 14, - 21.799999952316284, - 11.699999928474426, - 17.300000071525574, - 9.100000023841858, - 5.100000023841858, - 15.400000095367432, - 14, - 12.799999952316284, - 9.899999976158142, - 15.100000023841858, - 7.400000095367432, - 15.799999952316284, - 8.5, - 12.899999976158142, - 7.100000023841858, - 14.899999976158142, - 13.100000023841858, - 12.5, - 6.299999952316284, - 15.5, - 12.600000023841858, - 15.600000023841858, - 13.899999976158142, - 17.200000047683716, - 17.200000047683716, - 12.200000047683716, - 14.600000023841858, - 6.800000071525574, - 5, - 15.5, - 13.899999976158142, - 15.600000023841858 - ] - } - ] - }, - { - "name": "real clipboard permission and copy", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 112.1307080000006, - "p95": 120.04841599999963, - "values": [ - 107.07983299999978, - 110.49141700000109, - 108.39524999999958, - 110.16362500000105, - 115.88958300000013, - 116.68674999999894, - 116.61204100000032, - 120.04841599999963, - 112.1307080000006, - 110.0510000000013, - 108.02208299999984, - 112.71908299999996, - 110.35666700000002, - 109.13508400000137, - 113.37141700000029, - 108.04691600000115, - 119.85520899999938, - 127.74620799999866, - 116.00433300000077, - 117.61679099999856 - ] - }, - "operations": [ - { - "kind": "permissions", - "count": 20, - "median": 1.600000023841858, - "p95": 4.100000023841858, - "values": [ - 1.2999999523162842, - 1.2999999523162842, - 1.3000000715255737, - 1.5, - 2.200000047683716, - 3, - 2.6999999284744263, - 3.5999999046325684, - 2.299999952316284, - 1.3000000715255737, - 1.600000023841858, - 1.2000000476837158, - 1.600000023841858, - 1.600000023841858, - 1.7999999523162842, - 1.3000000715255737, - 4.100000023841858, - 4.5, - 2.899999976158142, - 3.700000047683716 - ] - } - ] - }, - { - "name": "virtual WebAuthn with actual credential and assertion", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 97.84062500000073, - "p95": 103.75324999999975, - "values": [ - 104.22249999999985, - 103.75324999999975, - 95.76283299999886, - 98.746000000001, - 100.5499579999996, - 98.11262499999975, - 96.95154199999888, - 99.01350000000093, - 98.50129199999901, - 100.0247500000005, - 96.36712500000067, - 96.13662499999919, - 95.89912499999991, - 97.50429099999747, - 97.84062500000073, - 95.56020799999897, - 95.18399999999747, - 96.43570800000089, - 102.07279099999869, - 101.02900000000227 - ] - }, - "operations": [ - { - "kind": "authenticator", - "count": 20, - "median": 1.399999976158142, - "p95": 1.600000023841858, - "values": [ - 1.5, - 1.600000023841858, - 1.3000000715255737, - 1.3000000715255737, - 1.600000023841858, - 1.399999976158142, - 1.7000000476837158, - 1.3000000715255737, - 1.4000000953674316, - 1.5, - 1.2999999523162842, - 1.399999976158142, - 1.2999999523162842, - 1.5, - 1.399999976158142, - 1.399999976158142, - 1.1999999284744263, - 1.399999976158142, - 1.4000000953674316, - 1.2000000476837158 - ] - }, - { - "kind": "remove-authenticator", - "count": 20, - "median": 1.3000000715255737, - "p95": 1.600000023841858, - "values": [ - 1.3000000715255737, - 1.5, - 1.4000000953674316, - 1.2999999523162842, - 1.399999976158142, - 1.399999976158142, - 1.2999999523162842, - 1.2000000476837158, - 1.3000000715255737, - 1.3000000715255737, - 1.7999999523162842, - 1.2999999523162842, - 1.2999999523162842, - 1.2999999523162842, - 1.2999999523162842, - 1.2999999523162842, - 1.3000000715255737, - 1.3000000715255737, - 1.5, - 1.600000023841858 - ] - } - ] - }, - { - "name": "app offline and recovery with a connected controller", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 194.7061250000006, - "p95": 204.6430419999997, - "values": [ - 190.8225420000017, - 194.7061250000006, - 198.328207999999, - 198.94295800000327, - 196.20920800000022, - 206.26841699999932, - 198.86699999999837, - 200.67341699999815, - 191.44591700000092, - 204.6430419999997, - 202.58820800000103, - 198.07433300000048, - 187.20858299999963, - 193.98433400000067, - 187.416500000003, - 187.12300000000323, - 192.5301670000008, - 202.3486669999984, - 183.72295900000245, - 187.93524999999863 - ] - }, - "operations": [ - { - "kind": "offline", - "count": 40, - "median": 3, - "p95": 5.899999976158142, - "values": [ - 2.399999976158142, - 2.600000023841858, - 4.200000047683716, - 3.299999952316284, - 4.200000047683716, - 3.200000047683716, - 5.899999976158142, - 2.4000000953674316, - 3.299999952316284, - 3, - 5.700000047683716, - 2.5, - 4.600000023841858, - 3.5, - 7.5, - 4.100000023841858, - 3.5, - 1.9000000953674316, - 3.600000023841858, - 3.600000023841858, - 4.400000095367432, - 2.700000047683716, - 5.900000095367432, - 2.8000000715255737, - 3.200000047683716, - 1.399999976158142, - 2.899999976158142, - 2.5, - 1.7000000476837158, - 1.6999999284744263, - 3, - 1.5, - 2.399999976158142, - 2.600000023841858, - 5.600000023841858, - 3, - 1.7999999523162842, - 1.6999999284744263, - 1.8000000715255737, - 2.600000023841858 - ] - } - ] - }, - { - "name": "below-fold tall element pixels without viewport changes", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 100.15708299999824, - "p95": 117.04891700000007, - "values": [ - 115.03637499999968, - 101.29550000000017, - 117.04891700000007, - 99.74616599999717, - 99.7155840000014, - 100.27008299999943, - 98.04762499999924, - 100.30204200000298, - 117.70845800000097, - 100.7302090000012, - 99.30995800000164, - 100.34487499999886, - 100.21274999999878, - 99.94483300000138, - 99.26350000000093, - 100.8044579999987, - 99.65925000000061, - 100.15708299999824, - 99.76599999999962, - 99.73191699999734 - ] - }, - "operations": [ - { - "kind": "capture", - "count": 20, - "median": 39.699999928474426, - "p95": 45.699999928474426, - "values": [ - 44.59999990463257, - 43, - 45.699999928474426, - 40.199999928474426, - 38.5, - 40.10000002384186, - 34.89999997615814, - 37.89999997615814, - 49.10000002384186, - 42.89999997615814, - 41.199999928474426, - 37.39999997615814, - 37, - 37.5, - 37.39999997615814, - 39, - 40.199999928474426, - 42.10000002384186, - 39.699999928474426, - 37.39999997615814 - ] - } - ] - }, - { - "name": "real reduced-motion media setting", - "status": "passed", - "caller": "controller-window-performance", - "workflow": { - "count": 20, - "median": 61.84954099999959, - "p95": 68.39312500000233, - "values": [ - 60.594167000002926, - 60.879000000000815, - 66.9497909999991, - 68.9397089999984, - 59.223250000002736, - 63.10124999999971, - 63.905917000000045, - 57.57008300000234, - 61.84954099999959, - 60.21704099999988, - 60.836999999999534, - 64.95587499999965, - 68.39312500000233, - 68.36537500000122, - 67.20745799999713, - 64.80245800000193, - 63.08245900000111, - 58.75395900000149, - 58.7042079999992, - 59.39050000000134 - ] - }, - "operations": [ - { - "kind": "motion", - "count": 40, - "median": 2.100000023841858, - "p95": 4, - "values": [ - 1.3000000715255737, - 1.3000000715255737, - 2.100000023841858, - 2.5999999046325684, - 3.399999976158142, - 2.5999999046325684, - 5.299999952316284, - 2, - 2.0999999046325684, - 1.600000023841858, - 3, - 1.899999976158142, - 3, - 2.399999976158142, - 1.399999976158142, - 1.5, - 1.6999999284744263, - 2.299999952316284, - 1.899999976158142, - 2.200000047683716, - 2.700000047683716, - 1.899999976158142, - 4, - 1.899999976158142, - 4.799999952316284, - 3.200000047683716, - 3.5, - 3.100000023841858, - 3.299999952316284, - 3.1999999284744263, - 3, - 2.899999976158142, - 3.5, - 1.899999976158142, - 1.399999976158142, - 1.899999976158142, - 2, - 1.399999976158142, - 1.3000000715255737, - 1.2999999523162842 - ] - } - ] - }, - { - "name": "browser-local Interactor control after", - "status": "passed", - "caller": "app-window-performance", - "workflow": { - "count": 20, - "median": 248.5, - "p95": 256.2000000476837, - "values": [ - 246.70000004768372, - 246.89999997615814, - 246, - 249.20000004768372, - 248.60000002384186, - 247.89999997615814, - 248, - 248.29999995231628, - 252.89999997615814, - 256.2000000476837, - 249.29999995231628, - 244.19999992847443, - 248.5, - 247.5, - 250.29999995231628, - 246.30000007152557, - 248.5, - 250.40000009536743, - 257.89999997615814, - 250.10000002384186 - ] - }, - "operations": [] - } - ], - "cleanup": [ - { - "name": "adapter", - "status": "passed" - }, - { - "name": "browser", - "status": "passed" - }, - { - "name": "fixture", - "status": "passed" - } - ] -} diff --git a/benchmarks/managed/results/vizzly-playwright.json b/benchmarks/managed/results/vizzly-playwright.json deleted file mode 100644 index 5fe08bc..0000000 --- a/benchmarks/managed/results/vizzly-playwright.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "source": "a6f62de1d9487764e003d6f64b00fd75899d46b8", - "repository": "vizzly-testing/vizzly", - "command": "npm run test:e2e -- --reporter=list,json", - "workers": 1, - "shards": 1, - "retries": 0, - "node": "v24.22.1", - "platform": "darwin", - "arch": "arm64", - "cpu": "Apple M4 Max", - "playwrightSDK": "1.64.0", - "browser": "Chromium 156.0.8078.4", - "browserRevision": "1248", - "conditions": "Serial runs; original builds, real PostgreSQL/Redis/app, screenshot capture enabled; dependency/browser install excluded.", - "caseInventorySHA256": "198d2e6027f925b17ced7f6e50d752430401099ead6679369a7a12f1050dc37a", - "samples": [ - { - "sample": 1, - "startedAt": "2026-10-30T06:24:24.257Z", - "commandMS": 313433.33462499996, - "runnerMS": 307305.13, - "serviceInclusiveMS": 317785.71904199995, - "phases": [ - { - "name": "Service setup", - "duration": 1888.48425, - "code": 0, - "signal": null - }, - { - "name": "Port database", - "duration": 65.15329199999996, - "code": 0, - "signal": null - }, - { - "name": "Port redis", - "duration": 44.30116700000008, - "code": 0, - "signal": null - }, - { - "name": "Test command", - "duration": 313433.33462499996, - "code": 0, - "signal": null - }, - { - "name": "Service cleanup", - "duration": 2350.357833999966, - "code": 0, - "signal": null - } - ], - "stats": { - "startTime": "2026-10-30T06:24:32.360Z", - "duration": 307305.13, - "expected": 339, - "skipped": 0, - "unexpected": 0, - "flaky": 0 - }, - "savedImages": 232, - "imageDimensionsSHA256": "385bf6ca1138d239af4266f1af5bef74557a14ad824a3ad743ddc1b2235d6ce3" - }, - { - "sample": 2, - "startedAt": "2026-10-30T06:36:28.318Z", - "commandMS": 306062.235542, - "runnerMS": 302365.592, - "serviceInclusiveMS": 310299.555875, - "phases": [ - { - "name": "Service setup", - "duration": 1858.793709, - "code": 0, - "signal": null - }, - { - "name": "Port database", - "duration": 65.92049999999995, - "code": 0, - "signal": null - }, - { - "name": "Port redis", - "duration": 49.8544159999999, - "code": 0, - "signal": null - }, - { - "name": "Test command", - "duration": 306062.235542, - "code": 0, - "signal": null - }, - { - "name": "Service cleanup", - "duration": 2259.021458000003, - "code": 0, - "signal": null - } - ], - "stats": { - "startTime": "2026-10-30T06:36:33.967Z", - "duration": 302365.592, - "expected": 339, - "skipped": 0, - "unexpected": 0, - "flaky": 0 - }, - "savedImages": 232, - "imageDimensionsSHA256": "385bf6ca1138d239af4266f1af5bef74557a14ad824a3ad743ddc1b2235d6ce3" - } - ], - "medians": { - "commandMS": 309747.78508349997, - "runnerMS": 304835.36100000003, - "serviceInclusiveMS": 314042.6374585 - } -} diff --git a/benchmarks/managed/run.js b/benchmarks/managed/run.js deleted file mode 100644 index c011b06..0000000 --- a/benchmarks/managed/run.js +++ /dev/null @@ -1,273 +0,0 @@ -import { strict as assert } from 'node:assert'; -import { mkdir, readFile, writeFile } from 'node:fs/promises'; -import { resolve, dirname } from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { platform, arch, cpus } from 'node:os'; -import { createHash } from 'node:crypto'; -import { chromium, firefox, webkit } from 'playwright-core'; -import metadata from 'playwright-core/package.json' with { type: 'json' }; -import { startFixture } from './fixture.js'; -import { createOperations } from './operations.js'; -import { createAdapter as direct } from './direct.js'; -import { cases } from './cases.js'; - -let [ - adapterPath = 'direct', - outputPath = '/private/tmp/dyno-managed-reference.json', - sampleArgument = '20', - engine = 'chromium', -] = process.argv.slice(2); -let samples = Number(sampleArgument); -if (!Number.isInteger(samples) || samples < 1 || samples > 1000) - throw new Error('Samples must be between 1 and 1000.'); -if (!['chromium', 'firefox', 'webkit'].includes(engine)) - throw new Error('Choose chromium, firefox, or webkit.'); -let operationBudget = Number(process.env.DYNO_EVAL_OPERATION_TIMEOUT ?? 5000); -if (!Number.isInteger(operationBudget) || operationBudget < 1) - throw new Error('Operation timeout must be a positive integer.'); -let runSignal = AbortSignal.timeout(300000); -async function bounded(label, work, budget, parent = runSignal) { - let signal = parent - ? AbortSignal.any([parent, AbortSignal.timeout(budget)]) - : AbortSignal.timeout(budget); - signal.throwIfAborted(); - let interrupted = Promise.withResolvers(); - let aborted = () => interrupted.reject(new Error(`${label} exceeded its evaluation deadline.`)); - signal.addEventListener('abort', aborted, { once: true }); - try { - return await Promise.race([work(), interrupted.promise]); - } finally { - signal.removeEventListener('abort', aborted); - } -} -let percentile = (values, fraction) => - [...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1]; -let summarize = (values) => ({ - count: values.length, - median: percentile(values, 0.5), - p95: percentile(values, 0.95), - values, -}); -let records = async (fixture) => - await (await fetch(fixture.origin + '/records', { cache: 'no-store', signal: runSignal })).json(); -let fixture, browser, adapter, setupDuration, failure, browserVersion, suiteFingerprint; -let results = []; -let cleanup = []; -let activeScenario = 'setup'; -let source = - adapterPath === 'direct' - ? new URL('./direct.js', import.meta.url) - : pathToFileURL(resolve(adapterPath)); -try { - let sources = await Promise.all( - ['run.js', 'cases.js', 'fixture.js', 'operations.js', 'direct.js'].map((name) => - readFile(new URL(name, import.meta.url)), - ), - ); - suiteFingerprint = createHash('sha256').update(Buffer.concat(sources)).digest('hex'); - // Import-time prototype setup is included in the cold setup interval. - let factory = - adapterPath === 'direct' - ? direct - : (await bounded('Adapter import', () => import(source), 15000)).createAdapter; - fixture = await bounded('Fixture startup', startFixture, 15000); - browser = await bounded( - 'Browser startup', - () => ({ chromium, firefox, webkit })[engine].launch({ headless: true }), - 30000, - ); - browserVersion = browser.version(); - let context = await browser.newContext({ - viewport: { width: 1280, height: 800 }, - deviceScaleFactor: 1, - locale: 'en-US', - timezoneId: 'UTC', - }); - let app = await context.newPage(); - await app.goto(fixture.origin); - await app.waitForFunction(() => Boolean(window.evalI)); - let opened = app.waitForEvent('popup'); - await app.evaluate(() => window.open('/controller', 'dyno-evaluation-controller')); - let controller = await opened; - await controller.waitForLoadState('domcontentloaded'); - let perform = createOperations({ context, app, engine }); - adapter = await bounded( - 'Adapter setup', - () => factory({ context, app, controller, fixture, perform, engine }), - 15000, - ); - assert.ok(['node', 'browser'].includes(adapter.clock), 'Declare the adapter caller clock.'); - if (adapter.clock === 'browser') - assert.ok( - typeof adapter.invokeName === 'string' && adapter.invokeName.length, - 'Browser adapters declare their controller function name.', - ); - else assert.equal(typeof adapter.invoke, 'function'); - let caller = adapter.clock === 'browser' ? 'controller-window-performance' : 'node-performance'; - let invokeMeasured = (input) => - bounded( - `Operation ${input.kind}`, - async () => { - if (adapter.clock === 'browser') - return controller.evaluate( - async ({ input, name }) => { - if (!Object.hasOwn(globalThis, name) || typeof globalThis[name] !== 'function') - throw new Error('Prototype browser caller is missing.'); - let start = performance.now(); - let value = await globalThis[name](input); - return { value, duration: performance.now() - start }; - }, - { input, name: adapter.invokeName }, - ); - let start = performance.now(); - let value = await adapter.invoke(input); - return { value, duration: performance.now() - start }; - }, - operationBudget, - ); - setupDuration = performance.now(); - let control = async (label) => { - activeScenario = label; - let values = []; - for (let index = -1; index < samples; index++) { - let before = await records(fixture); - let duration = await bounded( - label, - () => - app.evaluate(async () => { - let start = performance.now(); - await window.runLocalWorkflow(); - return performance.now() - start; - }), - 15000, - ); - let after = await records(fixture); - assert.equal( - after.length, - before.length + 1, - 'Each control sample must persist exactly one new record.', - ); - assert.equal(after.at(-1).name, 'Robert'); - assert.equal(after.at(-1).notes, 'A saved note'); - if (index >= 0) values.push(duration); - } - results = [ - ...results, - { - name: label, - status: 'passed', - caller: 'app-window-performance', - workflow: summarize(values), - operations: [], - }, - ]; - }; - await control('browser-local Interactor control before'); - for (let scenario of cases) { - activeScenario = scenario.name; - if (scenario.chromium && engine !== 'chromium') { - results = [ - ...results, - { - name: scenario.name, - status: 'unsupported', - reason: 'Reference scenario requires Chromium capabilities.', - }, - ]; - continue; - } - let workflows = []; - let operations = []; - for (let index = -1; index < samples; index++) { - let measured = []; - let invoke = async (input) => { - let result = await invokeMeasured(input); - measured = [...measured, { kind: input.kind, duration: result.duration }]; - return result; - }; - let started = performance.now(); - await bounded( - scenario.name, - () => scenario.run({ context, app, controller, fixture, invoke }), - 30000, - ); - if (index >= 0) { - workflows.push(performance.now() - started); - operations.push(...measured); - } - } - let kinds = [...new Set(operations.map((entry) => entry.kind))]; - results = [ - ...results, - { - name: scenario.name, - status: 'passed', - caller, - workflow: summarize(workflows), - operations: kinds.map((kind) => ({ - kind, - ...summarize( - operations.filter((entry) => entry.kind === kind).map((entry) => entry.duration), - ), - })), - }, - ]; - console.log( - `${scenario.name}: ${Math.round(percentile(workflows, 0.5))} ms median instrumented workflow`, - ); - } - await control('browser-local Interactor control after'); -} catch (error) { - failure = error; - results = [...results, { name: activeScenario, status: 'failed', error: error.message }]; -} finally { - // Every cleanup runs even after a failed or hung prototype. No later scenario runs after a failure. - for (let [name, resource] of [ - ['adapter', adapter && (() => adapter.dispose())], - ['browser', browser && (() => browser.close())], - ['fixture', fixture && (() => fixture.close())], - ]) { - if (!resource) continue; - try { - await bounded(`Cleanup ${name}`, resource, 5000, null); - cleanup = [...cleanup, { name, status: 'passed' }]; - } catch (error) { - cleanup = [...cleanup, { name, status: 'failed', error: error.message }]; - failure ??= error; - } - } - await mkdir(dirname(resolve(outputPath)), { recursive: true }); - await writeFile( - outputPath, - JSON.stringify( - { - schemaVersion: 1, - status: failure ? 'failed' : 'passed', - adapter: adapter?.name ?? adapterPath, - callerClock: adapter?.clock, - suiteFingerprint, - engine, - browserVersion, - playwright: metadata.version, - node: process.version, - platform: platform(), - arch: arch(), - cpu: cpus()[0]?.model, - samples, - setupDuration, - totalDuration: performance.now(), - results, - cleanup, - }, - null, - 2, - ) + '\n', - ); - console.log(`Saved ${outputPath}`); -} -if (failure) { - await new Promise((resolve) => process.stderr.write(failure.stack + '\n', resolve)); - // A broken external prototype can retain its own handles. The CLI verdict is final; - // Playwright's exit handler also stops its owned browser processes. - process.exit(1); -} diff --git a/docs/ci.md b/docs/ci.md index f817329..5602247 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -27,8 +27,6 @@ Tests live under `tests/core`, `tests/package`, `tests/browser`, `tests/executio Core checks and actionlint workflow validation run first. Browser and package suites run in parallel once core passes. Documentation runs separately. Browser, execution, repeat, project, reporter, native, and capture suites have separate jobs so each complete suite has its own budget. Each test job has a hard five-minute cap, including setup. Browser and execution checks run up to three independent files together; each owns its app, gateway, browser profiles, and fixture directory. Tests inside a file stay sequential. Repeat, project, reporter, native, capture, and package suites stay sequential. The macOS process-group check runs once in core instead of repeating for each browser. There are no retries or extra sleeps. -Execution and reporter checks have their own suites because the combined macOS Firefox browser suite reached 288–294 seconds. Project setup and preparation now have their own suite after execution exhausted the same cap. Repeated runs have a separate suite after the added real-browser rounds pushed macOS Firefox execution past that cap. Every test and engine/OS combination remains covered. Product suites and benchmarks keep their configured execution; this grouping only changes Dyno's own CI. - ## Results and setup Each test job uploads `output.log` and `results.xml` under its own artifact name, retained for seven days. They include failed test diagnostics; they don't retain fixture app data or screenshot files. Set `DYNO_TEST_REPORT_DIR` locally to write JUnit output alongside the console report. diff --git a/tests/fixtures/native-app-server.js b/tests/fixtures/native-app-server.js index f8e3b24..4d3f665 100644 --- a/tests/fixtures/native-app-server.js +++ b/tests/fixtures/native-app-server.js @@ -10,6 +10,9 @@ html{background:white}body{font:16px system-ui;margin:24px}label{display:block;m #secret{position:absolute;left:600px;top:400px;width:60px;height:40px;background:rgb(200,10,20)} #hover{width:200px;height:80px;background:rgb(20,60,140);margin:16px 0} #hover:hover{background:rgb(20,160,80)} +#motion{animation:native-motion 1s infinite alternate} +@keyframes native-motion{to{transform:translateX(10px)}} +@media(prefers-reduced-motion:reduce){#motion{animation:none}} #tall{position:absolute;top:1600px;left:24px;width:160px}#tall>div{height:500px} @media(max-width:500px){#secret{display:none}} @media(min-height:1000px){#tall>div{background:rgb(1,2,3)!important}} @@ -22,6 +25,7 @@ html{background:white}body{font:16px system-ui;margin:24px}label{display:block;m
Hover target
Open destination +
Motion preview