From 683d9677e1a32fa6a27f90043046b2718a93f4d1 Mon Sep 17 00:00:00 2001 From: mm Date: Wed, 11 Nov 2026 14:53:41 +0800 Subject: [PATCH] test(webui): the agent transcript names the browser panel's state when no frame arrives The browser panel has six states, each with its own testid. The diagnostic looked at two of them, and those two read the same in every failure: the empty placeholder is on screen whether the stream is slow, whether the stream failed, or whether the host cannot serve a browser at all. The product's own first-frame deadline is 60s -- the same budget the wait had -- so a panel that had already given up used to look exactly like a panel that was merely slow, and the failure could not tell the two apart. The wait now ends on whichever comes first: the frame, or the panel's own failure notice. A failure prints every browser-panel node that was on the page, with the browser's error code beside the notice's words, so the next occurrence says which state the panel was in. The budget is unchanged: 60s, and the wait never outlasts it. No expected value changed, no retry was added, no case was skipped. --- .../test/integration/agent-transcript.test.ts | 185 ++++++++++++++++-- 1 file changed, 171 insertions(+), 14 deletions(-) diff --git a/packages/webui/test/integration/agent-transcript.test.ts b/packages/webui/test/integration/agent-transcript.test.ts index e411b2fd..446aae04 100644 --- a/packages/webui/test/integration/agent-transcript.test.ts +++ b/packages/webui/test/integration/agent-transcript.test.ts @@ -7,7 +7,7 @@ import { execFileSync } from "node:child_process"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import { chromium, expect as browserExpect, type Browser } from "@playwright/test"; +import { chromium, expect as browserExpect, type Browser, type Locator, type Page } from "@playwright/test"; import { afterAll, expect, it } from "vitest"; import type { WebuiBrowserAdapterBinding } from "../../src/runtime/harness/browser.js"; @@ -153,6 +153,140 @@ afterAll(() => { else process.env.MAVIS_REGION = previousRegion; }); +/** + * The deadline for the wait on the browser panel's first frame. + * + * The product's own first-frame deadline is 60s (browser-service.ts), so a + * wait longer than this would only outlast the product's own answer to the + * same question. The wait ends on the frame, on the panel's failure notice, + * or at this deadline -- never later. + */ +const BROWSER_FRAME_TIMEOUT_MS = 60_000; + +/** + * One of the browser panel's own states, read from the page. + * + * Every state the panel can be in carries a `browser-panel-` testid, and the + * failure toast carries the browser's error code beside its words. + */ +type BrowserPanelNode = { + readonly id: string | null; + readonly text: string; + readonly errorCode: string | null; +}; + +/** + * How the wait on the browser panel's first frame ended. + * + * `frame` is the product working. `failed` is the product reporting that the + * stream gave up. `timeout` is neither. Both of the last two carry the + * panel's whole state, so the failure can say which state the panel was in. + */ +type BrowserPanelWait = + | { readonly outcome: "frame" } + | { readonly outcome: "failed"; readonly nodes: readonly BrowserPanelNode[] } + | { readonly outcome: "timeout"; readonly nodes: readonly BrowserPanelNode[] }; + +/** + * The browser panel's whole state, in one pass over the page. + * + * The two states the old diagnostic looked at said almost nothing: the + * empty placeholder shows in every one of them, so "no frame yet", "the + * stream failed" and "the host cannot serve a browser" all read the same. + * Every `browser-panel-` node is collected instead, so a failure says which + * state the panel was in. The text is cut short: the report wants the + * notice's sentence, not a screen of DOM. + */ +async function readBrowserPanelNodes(page: Page): Promise { + return page.evaluate(() => + Array.from(document.querySelectorAll('[data-testid^="browser-panel-"]')).map((node) => { + const text = (node.textContent ?? "").trim(); + return { + id: node.getAttribute("data-testid"), + text: text.length > 200 ? `${text.slice(0, 200)}…` : text, + errorCode: node.getAttribute("data-error-code"), + }; + }), + ); +} + +/** + * The panel's state at the moment it reported a failure. + * + * The live region says the stream is over; the toast beside it says why, + * and carries the browser's error code. The toast is mounted a moment after + * the notice, so it is waited for briefly -- the code and the browser's own + * wording are the parts of the failure worth having. The wait is bounded by + * the caller's deadline, so this never spends more than the frame wait + * would have. + */ +async function readBrowserPanelFailureNodes( + page: Page, + deadline: number, +): Promise { + const until = Math.min(deadline, Date.now() + 500); + for (;;) { + const nodes = await readBrowserPanelNodes(page); + if (nodes.some((node) => node.errorCode !== null) || Date.now() >= until) return nodes; + await page.waitForTimeout(50); + } +} + +/** + * Whether the panel is saying the stream failed. + * + * The live region says the stream is over; the toast beside it says why, + * and carries the browser's error code. Either one is enough. + */ +function browserPanelReportedFailure(nodes: readonly BrowserPanelNode[]): boolean { + return nodes.some((node) => node.id === "browser-panel-failed" || node.id === "browser-panel-error"); +} + +/** + * Wait for the browser panel to show a frame, or to report that no frame is + * coming. Whichever happens first ends the wait. + * + * Two endings, two different faults. A frame is the product working: the + * isolated Chromium booted, opened the page and streamed a picture back. A + * failure notice is the product reporting that the stream gave up. The + * product's own first-frame deadline is 60s (browser-service.ts) -- the same + * budget this wait has -- so a panel that had already given up used to look + * exactly like a panel that was merely slow, and the diagnostic it printed + * could not tell the two apart. The wait now ends on whichever comes first, + * and the failure says which one it was. + * + * The frame is checked first, so a panel that delivered a picture and then + * failed is still the passing case it always was. + */ +async function waitForBrowserFrameOrFailure( + page: Page, + frame: Locator, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + if (await frame.isVisible()) return { outcome: "frame" }; + const nodes = await readBrowserPanelNodes(page); + if (browserPanelReportedFailure(nodes)) { + return { outcome: "failed", nodes: await readBrowserPanelFailureNodes(page, deadline) }; + } + const remaining = deadline - Date.now(); + if (remaining <= 0) { + // The product's own deadline expires at about the same moment this + // one does, so the last reading is the one that says whether the + // panel failed just as the wait ran out. + const last = await readBrowserPanelNodes(page); + if (browserPanelReportedFailure(last)) { + return { outcome: "failed", nodes: await readBrowserPanelFailureNodes(page, deadline) }; + } + return { outcome: "timeout", nodes: last }; + } + // The sleep never runs past the deadline, so the wait costs no more + // than the single budget it was given. + await page.waitForTimeout(Math.min(250, remaining)); + } +} + it.each(["transcript", "cross-client", "external-active-mutation", "external-active-mutation-gap"] as const)("real agent rendering and recovery: %s", async (scenario) => { let releaseAnswer: () => void = () => undefined; const answerGate = new Promise((resolve) => { releaseAnswer = resolve; }); @@ -1068,20 +1202,43 @@ it.each(["transcript", "cross-client", "external-active-mutation", "external-act // event channel. That whole pipeline -- not the product's own // responsiveness -- is what this wait covers, and on a loaded CI runner // it can outlast the 30s window this test used to give it. Give the wait - // its own budget, and keep the failure readable: if the panel still - // shows its empty placeholder the frame never arrived, and if a - // capability notice is on screen the host could not serve a browser at - // all. Those are different faults and deserve different fixes. - try { - await browserExpect(browserFrame).toBeVisible({ timeout: 60_000 }); - } catch (error) { - throw new Error(`${error instanceof Error ? error.message : String(error)}\n${JSON.stringify({ - emptyFrameStillShowing: await page.getByTestId("browser-panel-frame-empty").count(), - capabilityNotices: await page.locator( - '[data-testid="browser-panel-unknown"], [data-testid="browser-panel-unauthorized"]', - ).allTextContents(), - })}`); + // its own budget. + // + // The wait now ends on one of three outcomes, and they are different + // faults. The frame arrives: the product works and the case goes on. The + // panel reports a failed stream: the product is not going to produce a + // picture, so the case fails at once, with the notice's own words and + // the browser's error code -- the product's first-frame deadline is 60s + // (browser-service.ts), the same as this wait, so a panel that had + // already given up used to look exactly like a panel that was merely + // slow, and the diagnostic could not tell the two apart. Neither: the + // wait runs out, and the failure prints the panel's whole state instead + // of two guesses about it. + const browserPanel = await waitForBrowserFrameOrFailure(page, browserFrame, BROWSER_FRAME_TIMEOUT_MS); + if (browserPanel.outcome === "failed") { + const notice = browserPanel.nodes.find((node) => node.id === "browser-panel-failed"); + const toast = browserPanel.nodes.find((node) => node.id === "browser-panel-error"); + throw new Error([ + "the browser panel reported a failed stream before any frame arrived", + `notice: ${notice?.text ?? "(none)"}`, + `toast: ${toast?.text ?? "(none)"}`, + `error code: ${toast?.errorCode ?? "(none)"}`, + JSON.stringify({ panelState: browserPanel.nodes }), + ].join("\n")); } + if (browserPanel.outcome === "timeout") { + throw new Error([ + `the browser panel showed no frame within ${BROWSER_FRAME_TIMEOUT_MS}ms and reported no failure`, + JSON.stringify({ + emptyFrameStillShowing: browserPanel.nodes.filter((node) => node.id === "browser-panel-frame-empty").length, + capabilityNotices: browserPanel.nodes + .filter((node) => node.id === "browser-panel-unknown" || node.id === "browser-panel-unauthorized") + .map((node) => node.text), + panelState: browserPanel.nodes, + }), + ].join("\n")); + } + const browserGeometry = await browserFrame.evaluate((node) => { const frame = node.getBoundingClientRect(); const panel = node.closest(".webui-browser-panel")?.getBoundingClientRect();