From 1814783de9b8295944cb09f2d831ffc33313fd50 Mon Sep 17 00:00:00 2001 From: Jalil Date: Fri, 13 Nov 2026 17:31:14 -0800 Subject: [PATCH] perf(workbot): update a computer's desktop before it pauses, and start it without polling Every runner deploy carries a new desktop guest. A computer's first desktop use after one asked the helper to start, was refused because the guest was older, installed the new guest, started again, and then asked for the desktop's health over and over, one round trip each, until the browser was up. - A start first asks, in one command, what the desktop answers and what the computer was built with. An older guest on this runner's image is installed first and started once; a start the helper would refuse is never run. - The start command waits inside the computer for the desktop to answer, so the runner no longer polls. The wait counts against the start's 40 s. - The idle check brings an older guest up to date before it pauses the computer, once nothing is running on it. A desktop that is running restarts on the new guest; one that isn't only gets the guest. A failed update is logged and the computer still pauses. Someone coming back meanwhile calls the update off as they would a checkpoint, and doesn't wait for a pause. - The health check that also asks for versions is still timed as `health`. No image file changed, so no computer is replaced and no snapshot is built. Signed-off-by: Jalil --- ee/apps/headless-runner/README.md | 8 +- ee/packages/headless-computer/src/computer.ts | 34 ++- .../src/desktop-guest-update.test.ts | 226 ++++++++++++++++++ .../headless-computer/src/desktop-timing.ts | 7 +- ee/packages/headless-computer/src/desktop.ts | 162 ++++++++----- 5 files changed, 365 insertions(+), 72 deletions(-) create mode 100644 ee/packages/headless-computer/src/desktop-guest-update.test.ts diff --git a/ee/apps/headless-runner/README.md b/ee/apps/headless-runner/README.md index f4108f1c5b8..9ad25d26fd9 100644 --- a/ee/apps/headless-runner/README.md +++ b/ee/apps/headless-runner/README.md @@ -521,9 +521,11 @@ the old one. A new guest on the same image needs no rebuild. The runner installs its guest into the computers it starts, which adds about 4 s to a new computer's first -open. Guests change most days, and a template takes about five minutes to build -and a Differently configuration change to adopt, so the template follows the -image and not the guest. +open, and into an idle computer just before pausing it, so one used since the +deploy opens without that wait. A computer paused before the deploy gets the +guest on its next open. Guests change most days, and a template takes about +five minutes to build and a Differently configuration change to adopt, so the +template follows the image and not the guest. Opt-in verification, creating and cleaning one real computer: diff --git a/ee/packages/headless-computer/src/computer.ts b/ee/packages/headless-computer/src/computer.ts index 06cf971ec22..cd0282a709a 100644 --- a/ee/packages/headless-computer/src/computer.ts +++ b/ee/packages/headless-computer/src/computer.ts @@ -85,7 +85,7 @@ export class Computers { private readonly vms = new Map>() private readonly pauseTimers = new Map>() private readonly pausing = new Map>() - /** Sessions in a pause that starting the computer calls off, so `vm()` doesn't wait for it. */ + /** Sessions in a pause, or the idle guest update before it, that starting the computer calls off, so `vm()` doesn't wait for it. */ private readonly checkpointing = new Set() /** Saved file ids already copied into each session's VM by this process. */ private readonly copied = new Map>() @@ -410,8 +410,20 @@ export class Computers { log("in_use") return this.release(sessionId) } - // Nobody is using it: the moment to move a computer built from an older image onto the current one. - if (!this.working.has(sessionId) && await this.staleImage(sessionId, vm)) vm = await this.replace(sessionId, vm, { required: false }) ?? vm + // Nobody is using it: the moment to move a computer built from an older image onto the current one, or to bring + // an older guest up to this runner's, so the paused computer already carries it. + const behind = this.working.has(sessionId) ? null : await this.behind(sessionId, vm) + if (behind === "image") vm = await this.replace(sessionId, vm, { required: false }) ?? vm + if (behind === "guest" && this.desktop) { + // Someone coming back doesn't wait for this: `vm()` calls it off as it does a checkpoint, and a start of theirs + // shares the update under way. + this.checkpointing.add(sessionId) + const began = Date.now() + // Not fatal: the computer pauses with the guest it has, and its next use updates it as before. + const outcome = await this.desktop.update(vm).then(changed => changed ? "updated" : "unchanged", (error: unknown) => `failed: ${(error instanceof Error ? error.message : String(error)).slice(0, 300)}`) + console.log(`[headless-computer] idle guest update ${JSON.stringify({ sessionId, outcome, ms: Date.now() - began })}`) + if (!this.checkpointing.delete(sessionId)) return log("called_off") + } // A provider that can neither pause nor stop keeps the computer running until its own expiry. if (!this.provider.describe().stopResume) { log("cannot_pause") @@ -509,20 +521,22 @@ export class Computers { } /** - * Whether the session's computer was built from an older image (known from its last start here, else asked) and - * moving it could help: a replacement that just came up on an older image shows new computers aren't there yet. + * What on the session's computer is older than this runner's (known from its last start here, else asked): its + * `image`, when moving it could help (a replacement that just came up on an older image shows new computers aren't + * there yet), or only its `guest`, which is installed into the computer where it is. */ - private async staleImage(sessionId: string, box: SandboxHandle) { - if (!this.desktop) return false + private async behind(sessionId: string, box: SandboxHandle): Promise<"image" | "guest" | null> { + if (!this.desktop) return null const known = this.desktops.get(sessionId) - if (known === "current") return false + if (known === "current") return null if (!known) { // A helper from before in-place updates can't carry anything over, so such a computer is only replaced once its // desktop is needed, never while idle. const versions = await this.desktop.versions(box).catch(() => null) - if (versions === null || versions.image === this.desktop.image) return false + if (versions === null) return null + if (versions.image === this.desktop.image) return versions.guest === this.desktop.guest ? null : "guest" } - return !await this.desktop.replacedInVain(box, IDLE_REPLACEMENT_RETRY_MS) + return await this.desktop.replacedInVain(box, IDLE_REPLACEMENT_RETRY_MS) ? null : "image" } /** diff --git a/ee/packages/headless-computer/src/desktop-guest-update.test.ts b/ee/packages/headless-computer/src/desktop-guest-update.test.ts new file mode 100644 index 00000000000..35e2ce02c76 --- /dev/null +++ b/ee/packages/headless-computer/src/desktop-guest-update.test.ts @@ -0,0 +1,226 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import { createHash } from "node:crypto" +import { chmodSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { test, type TestContext } from "node:test" +import { createFakeProvider } from "@openwork/sandbox/testing" +import type { RunSpec } from "@openwork/sandbox" +import identity from "../dist/desktop-identity.mjs" +import { Computers } from "./computer.js" +import { Desktop, DesktopError } from "./desktop.js" +import { DESKTOP_PROTOCOL } from "./desktop-protocol.mjs" + +const IMAGE = identity.image.trim() +const GUEST = identity.guest.trim() +const OLDER_GUEST = "f".repeat(64) +/** What a desktop on this runner's image answers a health check with while it runs `guest` (desktop-bridge.py). */ +const versionOf = (guest: string) => createHash("sha256").update(`${IMAGE}\n`).update(`${guest}\n`).digest("hex") +const browse = { view: "browser", operation: "open", args: {} } + +/** + * A computer on this runner's image with an older guest installed. Its helper behaves as desktop-bridge.py does: + * `start` refuses a guest that isn't the installed one, `install` replaces the installed guest, and the guest that is + * running only changes on `start`. `steps` is what the runner asked of it, in order. + */ +function computer(t: TestContext, input: { answering: boolean; jobs?: number; installFails?: boolean }) { + const vm = { installed: OLDER_GUEST, running: input.answering ? OLDER_GUEST as string | null : null } + const steps: string[] = [] + const lines: string[] = [] + /** Held open by a test to keep a command or an install in flight. */ + const gate = { command: Promise.resolve(), install: Promise.resolve() } + const ok = (stdout = "") => ({ exitCode: 0, stdout, stderr: "" }) + const refuse = (stderr: string) => ({ exitCode: 1, stdout: "", stderr }) + const health = () => vm.running ? JSON.stringify({ ok: true, version: versionOf(vm.running), protocol: DESKTOP_PROTOCOL }) : "" + const versions = () => JSON.stringify({ image: IMAGE, guest: vm.installed }) + const provider = createFakeProvider({ + capabilities: { platform: { os: "linux", isolation: "microvm" } }, + onOperation(operation) { if (operation.name === "pause") steps.push("pause") }, + async onRun({ command }: RunSpec) { + if (command.startsWith("ls /workspace/.jobs/*.running")) return ok(String(input.jobs ?? 0)) + if (command.startsWith("rm -f")) return ok() + if (command.endsWith("openwork-desktop-bridge command")) { steps.push("command"); await gate.command; return ok() } + if (command.endsWith("openwork-desktop-bridge versions")) { steps.push("versions"); return ok(versions()) } + const look = /^health=.* in \*([a-f0-9]{64})\*\) ;;/.exec(command) + if (look) { + steps.push("look") + return ok(vm.running && versionOf(vm.running) === look[1] ? `${health()}\n` : `${health()}\n${versions()}\n`) + } + const install = /openwork-desktop-bridge install ([a-f0-9]{64}) ([a-f0-9]{64}) [a-f0-9]{64} < /.exec(command) + if (install) { + steps.push("install") + await gate.install + if (input.installFails) return refuse("No space left on device") + vm.installed = install[2]! + return ok('{"ok": true}') + } + const start = /openwork-desktop-bridge start ([a-f0-9]{64}) ([a-f0-9]{64}) \|\| exit; /.exec(command) + if (start) { + steps.push("start") + if (start[2] !== vm.installed) return refuse("The desktop guest is out of date. Install this runner's guest.\n") + vm.running = vm.installed + return ok(`{"ok": true}\n${health()}\n`) + } + if (command.includes("call '/browser/task'")) { steps.push("task"); return ok(JSON.stringify({ ok: true })) } + throw new Error(`Unexpected command: ${command}`) + }, + }) + t.mock.method(console, "log", (line: unknown) => { lines.push(String(line)) }) + /** A runner process: an idle chat's cell leaves memory, so its pause usually runs in one that never saw the computer. */ + const process = () => new Computers({ provider, idlePauseMs: 60_000, keepDays: 1, desktop: true }, { readFile: async () => ({ text: "" }) }) + const state = () => provider.fake.sandboxes().at(-1)?.state + return { vm, steps, lines, gate, process, state } +} + +test("an older guest on this runner's image is installed first, then started once", async t => { + for (const answering of [true, false]) { + const f = computer(t, { answering }) + const computers = f.process() + assert.equal((await computers.desktopRun("first-use", browse)).isError, false) + assert.deepEqual(f.steps, ["look", "install", "start", "task"], "no start the helper would refuse, and no health checks after it") + assert.deepEqual([f.vm.installed, f.vm.running], [GUEST, GUEST]) + await computers.delete("first-use") + } +}) + +test("an idle computer gets this runner's guest before it pauses, so its next use finds it ready", async t => { + const f = computer(t, { answering: true }) + await f.process().start("idle") + await f.process().pauseIdle("idle") + assert.deepEqual(f.steps, ["versions", "look", "look", "install", "start", "pause"]) + assert.equal(f.state(), "stopped") + assert.match(f.lines.find(line => line.includes("idle guest update")) ?? "", /"sessionId":"idle","outcome":"updated"/) + f.steps.length = 0 + const next = f.process() + assert.equal((await next.desktopRun("idle", browse)).isError, false) + assert.deepEqual(f.steps, ["look", "task"]) + await next.delete("idle") + + // A desktop nobody started isn't started for this: it only gets the guest, and its first start is then the only one. + const stopped = computer(t, { answering: false }) + await stopped.process().start("idle") + await stopped.process().pauseIdle("idle") + assert.deepEqual(stopped.steps, ["versions", "look", "install", "pause"]) + assert.deepEqual([stopped.vm.installed, stopped.vm.running], [GUEST, null]) + stopped.steps.length = 0 + const later = stopped.process() + await later.desktopRun("idle", browse) + assert.deepEqual(stopped.steps, ["look", "start", "task"]) + await later.delete("idle") +}) + +test("an idle computer still pauses when its guest can't be updated", async t => { + const f = computer(t, { answering: true, installFails: true }) + await f.process().start("full") + await f.process().pauseIdle("full") + assert.equal(f.state(), "stopped") + assert.equal(f.steps.at(-1), "pause") + assert.deepEqual([f.vm.installed, f.vm.running], [OLDER_GUEST, OLDER_GUEST], "the desktop it had keeps running") + assert.match(f.lines.find(line => line.includes("idle guest update")) ?? "", /"outcome":"failed: Couldn't update the computer's desktop\. No space left on device"/) + assert.match(f.lines.find(line => line.includes("idle pause")) ?? "", /"outcome":"paused"/) +}) + +test("a computer in use keeps the guest it has", async t => { + // A background job is running on it. + const job = computer(t, { answering: true, jobs: 1 }) + await job.process().start("busy") + await job.process().pauseIdle("busy") + assert.deepEqual([job.steps, job.state()], [[], "running"]) + + // A command is in flight. + const f = computer(t, { answering: true }) + let finish = () => {} + f.gate.command = new Promise(resolve => { finish = resolve }) + const computers = f.process() + const command = computers.run("busy", "bash", { command: "sleep 100", description: "Waiting" }) + await until(() => f.steps.includes("command")) + await computers.pauseIdle("busy") + assert.deepEqual([f.steps, f.state()], [["command"], "running"]) + assert.match(f.lines.findLast(line => line.includes("idle pause")) ?? "", /"outcome":"in_use"/) + finish() + await command + await computers.delete("busy") +}) + +test("someone coming back during an idle guest update isn't made to wait for a pause", { timeout: 5_000 }, async t => { + const f = computer(t, { answering: true }) + let finish = () => {} + f.gate.install = new Promise(resolve => { finish = resolve }) + await f.process().start("back") + const computers = f.process() + const idle = computers.pauseIdle("back") + await until(() => f.steps.includes("install")) + // Would hang here if it waited for the update and the pause after it. + assert.equal((await computers.vm("back")).state, "running") + finish() + await idle + assert.equal(f.state(), "running") + assert.equal(f.steps.includes("pause"), false) + assert.match(f.lines.find(line => line.includes("idle pause")) ?? "", /"outcome":"called_off"/) + await computers.delete("back") +}) + +/** + * Stands in for `sudo -n /usr/local/sbin/openwork-desktop-bridge …`, so the runner's commands run in a real shell. After + * a start, the desktop answers its health check once it has been looked at FAKE_LOOKS times, as a browser coming up + * does. With FAKE_REFUSAL, the helper refuses to say what it was built with or to start. + */ +const HELPER = `#!/bin/sh +shift 2 +case "$1" in + versions|start) + [ -z "$FAKE_REFUSAL" ] || { echo "$FAKE_REFUSAL" >&2; exit 1; } + [ "$1" = start ] || { printf '{"image": "%s", "guest": "%s"}\\n' "$FAKE_IMAGE" "$FAKE_GUEST"; exit; } + : > "$FAKE_STATE/started"; echo '{"ok": true}' ;; + call) + [ -e "$FAKE_STATE/started" ] || { echo 'Desktop operation did not finish.' >&2; exit 1; } + looks=$(cat "$FAKE_STATE/looks" 2>/dev/null || echo 0); echo $((looks + 1)) > "$FAKE_STATE/looks" + [ "$looks" -ge "$FAKE_LOOKS" ] || exit 1 + printf '{"ok": true, "version": "%s", "protocol": 1}\\n' "$FAKE_VERSION" ;; +esac +` + +test("the start command waits inside the computer for the desktop to answer, and not at all for one that refuses", async t => { + // What the computer's own `sh` runs: dash on the image, and whatever this machine has. + for (const shell of ["/bin/sh", "/bin/dash"].filter(path => existsSync(path))) { + for (const scenario of [ + { expected: "current", looks: 3, refusal: "" }, + { expected: "desktop_unresponsive", looks: 1_000_000, refusal: "" }, + { expected: "desktop_outdated", looks: 0, refusal: "The desktop image is out of date. Rebuild its snapshot for this runner." }, + ]) { + const directory = mkdtempSync(join(tmpdir(), "desktop-helper-")) + t.after(() => rmSync(directory, { recursive: true, force: true })) + writeFileSync(join(directory, "sudo"), HELPER) + chmodSync(join(directory, "sudo"), 0o755) + const env = { PATH: `${directory}:/usr/bin:/bin`, FAKE_STATE: directory, FAKE_IMAGE: IMAGE, FAKE_GUEST: GUEST, FAKE_VERSION: versionOf(GUEST), FAKE_LOOKS: String(scenario.looks), FAKE_REFUSAL: scenario.refusal } + const commands: string[] = [] + const hurried = scenario.expected === "desktop_unresponsive" + if (hurried) t.mock.timers.enable({ apis: ["Date"], now: Date.now() }) + const provider = createFakeProvider({ + capabilities: { platform: { os: "linux", isolation: "microvm" } }, + onRun: ({ command }: RunSpec) => new Promise(resolve => { + commands.push(command) + // Most of the start's time is gone once the desktop has been looked at, so one that never answers is given up + // on in a second or two rather than in forty. + if (hurried && command.startsWith("health=")) t.mock.timers.tick(35_500) + execFile(shell, ["-c", command], { env }, (error, stdout, stderr) => resolve({ exitCode: error ? Number(error.code) : 0, stdout, stderr })) + }), + }) + const box = await provider.create({ idempotencyKey: "shell", image: provider.currentImage(), labels: {}, env: {}, storage: [], exposePorts: [], lifecycle: {} }, { timeoutMs: 1_000 }) + const starting = new Desktop(provider).start(box) + if (scenario.expected === "current") { + assert.equal(await starting, "current", shell) + assert.equal(commands.length, 2, "one look, then one command that starts the desktop and waits for it") + } else { + await assert.rejects(starting, (error: unknown) => error instanceof DesktopError && error.desktopCode === scenario.expected, `${shell}: ${scenario.expected}`) + } + if (hurried) t.mock.timers.reset() + } + } +}) + +async function until(done: () => boolean) { + for (let waited = 0; waited < 400 && !done(); waited++) await new Promise(resolve => setTimeout(resolve, 5)) + assert.ok(done()) +} diff --git a/ee/packages/headless-computer/src/desktop-timing.ts b/ee/packages/headless-computer/src/desktop-timing.ts index 449fd74caef..c7b7ae76076 100644 --- a/ee/packages/headless-computer/src/desktop-timing.ts +++ b/ee/packages/headless-computer/src/desktop-timing.ts @@ -5,10 +5,11 @@ type Step = { ms: number; count: number } /** What one bridge command asked the guest, from its command line; never the command or its payload. */ function bridgeStep(command: string) { if (/openwork-desktop-bridge start /.test(command)) return "desktop_start" - const helper = /openwork-desktop-bridge (install|versions|export|import) /.exec(`${command} `)?.[1] - if (helper) return `desktop_${helper}` + // A health check that also asks what the computer was built with is still a health check. const path = /openwork-desktop-bridge call '?\/([a-z/]+)/.exec(command)?.[1] - return path ? path.replaceAll("/", "_") : "command" + if (path) return path.replaceAll("/", "_") + const helper = /openwork-desktop-bridge (install|versions|export|import) /.exec(`${command} `)?.[1] + return helper ? `desktop_${helper}` : "command" } /** diff --git a/ee/packages/headless-computer/src/desktop.ts b/ee/packages/headless-computer/src/desktop.ts index 18c3f3e8266..20b669e2419 100644 --- a/ee/packages/headless-computer/src/desktop.ts +++ b/ee/packages/headless-computer/src/desktop.ts @@ -21,6 +21,10 @@ const MAX_CARRIED_BYTES = 100 * 1024 * 1024 * needs them all isn't going to start: the sooner that is known, the sooner it can be replaced. */ const START_BUDGET_MS = 40_000 +/** One look at the desktop, run inside the computer. */ +const HEALTH = `printf %s '{}' | sudo -n ${BRIDGE} call '/health'` +/** What a start command exits with when the desktop started and then never answered. */ +const STARTED_SILENT = 75 const viewerSchema = z.object({ token: z.string().regex(/^[A-Za-z0-9_-]{43}$/), paused: z.boolean() }) const healthSchema = z.object({ version: z.string(), protocol: z.number().optional() }).loose() const versionsSchema = z.object({ image: z.string().regex(/^[a-f0-9]{64}$/), guest: z.string().regex(/^[a-f0-9]{64}$/) }) @@ -45,13 +49,20 @@ export class DesktopError extends Error { /** The existing conversation computer owns the desktop too; it never provisions a second machine. */ export class Desktop { private readonly starting = new Map>() + /** The guest install under way for each computer, which a start and an idle update that overlap share. */ + private readonly installing = new Map>() /** Health versions of desktops already found stale but compatible, by computer, so they aren't checked again. */ private readonly compatible = new Map() /** The image this runner's computers boot from. */ readonly image = identity.image.trim() - private readonly guest = identity.guest.trim() + /** The guest this runner installs into a computer built from its image. */ + readonly guest = identity.guest.trim() + /** What a desktop on this runner's image and guest answers a health check with. */ + private readonly version: string /** privateBrowsing lets the browser open loopback and private pages: operator diagnostics on a dedicated VM only. */ - constructor(private readonly provider: SandboxProvider, private readonly options: { privateBrowsing?: boolean } = {}) {} + constructor(private readonly provider: SandboxProvider, options: { privateBrowsing?: boolean } = {}) { + this.version = createHash("sha256").update(identity.image).update(identity.guest).update(options.privateBrowsing === true ? "private-browsing" : "").digest("hex") + } start(box: SandboxHandle): Promise { const key = JSON.stringify(box.ref) @@ -63,11 +74,22 @@ export class Desktop { return pending } + /** + * Brings an idle computer on this runner's image up to this runner's guest, so the next use of its desktop doesn't + * wait for that. A desktop that is running restarts on the new guest, as that next use would have made it; one that + * isn't only gets the guest, and starts when it is first needed. Returns whether anything changed. + */ + async update(box: SandboxHandle) { + const { health, built } = await this.look(box) + if (built?.image !== this.image || built.guest === this.guest) return false + if (health) await this.start(box) + else await this.install(box, START_BUDGET_MS) + return true + } + private async prepare(box: SandboxHandle, key: string): Promise { const { run } = this.provider.blocks ?? {} if (!run || this.provider.describe().platform?.os !== "linux") throw new Error("Desktop access needs a configured Linux computer.") - const privateBrowsing = this.options.privateBrowsing === true - const version = createHash("sha256").update(identity.image).update(identity.guest).update(privateBrowsing ? "private-browsing" : "").digest("hex") const deadline = Date.now() + START_BUDGET_MS /** What a step may take: its own limit, or what is left of the start's time. */ const within = (limit: number) => { @@ -75,70 +97,88 @@ export class Desktop { if (left < 1_000) throw new DesktopError("desktop_unresponsive", "The computer's desktop didn't start. Try reconnecting.") return Math.min(limit, left) } - const ready = await this.health(box) - if (ready?.version === version) return "current" + const { health: ready, built } = await this.look(box) + if (ready?.version === this.version) return "current" if (ready && this.compatible.get(key) === ready.version) return "stale" - let started = await this.launch(box, this.image, this.guest, within(45_000)) - // Same image, older guest: install this runner's guest into the running computer instead of replacing it. - if (!started.ok && /guest is out of date/i.test(started.detail)) { - await this.install(box, within(60_000)) - started = await this.launch(box, this.image, this.guest, within(45_000)) - } - if (!started.ok && started.timedOut) { - // A timeout can leave startup running: only recheck health, never automatically replay start. - if ((await this.health(box))?.version === version) return "current" - } - if (started.ok) { - await this.healthy(box) - return "current" - } - // A helper from before images and guests had their own identities refuses two of them; another image says so. - if (!/out of date|start VERSION/i.test(started.detail)) { - if (started.missingHelper) { - throw new Error(`The computer needs a desktop snapshot built for this runner, including its restricted desktop helper. ${started.detail}`) + // A start the helper is known to refuse is never run: another image keeps its own desktop (below). + let refusal = "The desktop image is out of date. Rebuild its snapshot for this runner." + if (!built || built.image === this.image) { + // Same image, older guest: install this runner's guest into the running computer instead of replacing it, then + // start once. + const behind = built !== null && built.guest !== this.guest + if (behind) await this.install(box, within(60_000)) + let started = await this.launch(box, this.image, this.guest, within(45_000)) + // A computer that couldn't say what it was built with says so by refusing. + if (!behind && !started.ok && /guest is out of date/i.test(started.detail)) { + await this.install(box, within(60_000)) + started = await this.launch(box, this.image, this.guest, within(45_000)) + } + if (!started.ok && started.timedOut) { + // A timeout can leave startup running: only recheck health, never automatically replay start. + if ((await this.look(box)).health?.version === this.version) return "current" } - throw new DesktopError("desktop_unresponsive", `The computer's desktop didn't start. Try reconnecting.${started.detail ? ` ${started.detail}` : ""}`) + if (started.ok) return "current" + // A helper from before images and guests had their own identities refuses two of them; another image says so. + if (!/out of date|start VERSION/i.test(started.detail)) { + if (started.missingHelper) { + throw new Error(`The computer needs a desktop snapshot built for this runner, including its restricted desktop helper. ${started.detail}`) + } + throw new DesktopError("desktop_unresponsive", `The computer's desktop didn't start. Try reconnecting.${started.detail ? ` ${started.detail}` : ""}`) + } + refusal = started.detail } // Another image: keep driving its own desktop while it speaks this runner's protocol. let own = ready if (!own) { - const versions = await this.versions(box) - if (versions && (await this.launch(box, versions.image, versions.guest, within(45_000))).ok) own = await this.healthy(box) + const versions = built ?? await this.versions(box) + if (versions) own = (await this.launch(box, versions.image, versions.guest, within(45_000))).health } if (own?.protocol === DESKTOP_PROTOCOL) { this.compatible.set(key, own.version) return "stale" } - throw new DesktopError("desktop_outdated", `The computer's desktop image was built for a different runner: ${started.detail}`) - } - - private async health(box: SandboxHandle) { - const parsed = healthSchema.safeParse(await this.call(box, "/health", {}).catch(() => null)) - return parsed.success ? parsed.data : null + throw new DesktopError("desktop_outdated", `The computer's desktop image was built for a different runner: ${refusal}`) } - private async healthy(box: SandboxHandle) { - for (let attempt = 0; attempt < 30; attempt++) { - const probe = await this.health(box) - if (probe) return probe - await new Promise(resolve => setTimeout(resolve, 100)) - } - throw new DesktopError("desktop_unresponsive", "The desktop didn't respond. Try reconnecting.") + /** + * What the desktop answers and, unless that is this runner's version, the image and guest the computer was built + * with: one command, so a start knows what the helper would refuse without asking it. + */ + private async look(box: SandboxHandle) { + const result = await this.provider.blocks?.run?.(box, { + command: `health=$(${HEALTH}); printf '%s\\n' "$health"; case "$health" in *${this.version}*) ;; *) sudo -n ${BRIDGE} versions ;; esac`, + timeoutMs: 12_000, + }).catch(() => null) + const [answered = "", versions = ""] = (result?.stdout ?? "").split("\n") + const health = healthSchema.safeParse(safeJson(answered)) + const built = versionsSchema.safeParse(safeJson(versions)) + return { health: health.success ? health.data : null, built: built.success ? built.data : null } } + /** + * Starts the desktop built from `image` and `guest` and waits inside the computer for it to answer, all in one + * command: asking from here costs a round trip per look, and a browser takes several to come up. + */ private async launch(box: SandboxHandle, image: string, guest: string, timeoutMs: number) { - try { - const result = await this.bridge(box, `start ${image} ${guest}`, timeoutMs) - const detail = result.stderr.trim() - return { - ok: result.exitCode === 0, - detail: detail.slice(0, 300), - timedOut: /timed?\s*out|timeout/i.test(detail), - missingHelper: detail.includes(BRIDGE) && /command not found|no such file or directory|not found/i.test(detail), - } - } catch (error) { + // The computer gives up a little before the command's own limit, so a desktop that never answers is told apart + // from a command whose outcome is unknown. + const seconds = Math.max(1, Math.floor(timeoutMs / 1000) - 3) + const wait = `end=$(($(date +%s) + ${seconds})); until health=$(${HEALTH} 2>/dev/null); do [ "$(date +%s)" -lt "$end" ] || exit ${STARTED_SILENT}; sleep 0.1; done; printf '%s\\n' "$health"` + let result + try { result = await this.bridge(box, `start ${image} ${guest} || exit; ${wait}`, timeoutMs) } + catch (error) { const detail = error instanceof Error ? error.message : String(error) - return { ok: false, detail: detail.slice(0, 300), timedOut: (isSandboxError(error) && error.code === "timeout") || /timed?\s*out|timeout/i.test(detail), missingHelper: false } + return { ok: false, health: null, detail: detail.slice(0, 300), timedOut: (isSandboxError(error) && error.code === "timeout") || /timed?\s*out|timeout/i.test(detail), missingHelper: false } + } + if (result.exitCode === STARTED_SILENT) throw new DesktopError("desktop_unresponsive", "The desktop didn't respond. Try reconnecting.") + const detail = result.stderr.trim() + const health = result.exitCode === 0 ? healthSchema.safeParse(safeJson(result.stdout.trim().split("\n").at(-1) ?? "")) : null + return { + ok: result.exitCode === 0, + health: health?.success ? health.data : null, + detail: detail.slice(0, 300), + timedOut: /timed?\s*out|timeout/i.test(detail), + missingHelper: detail.includes(BRIDGE) && /command not found|no such file or directory|not found/i.test(detail), } } @@ -162,11 +202,21 @@ export class Desktop { return image === this.image && Date.now() - Number(at) < withinMs } - /** Hands the helper this runner's guest bundle; it checks the checksum and that the image is its own. */ - private async install(box: SandboxHandle, timeoutMs: number) { - const result = await this.upload(box, Buffer.from(guestBundle, "base64"), path => - this.bridge(box, `install ${this.image} ${this.guest} ${identity.bundleSha256} < ${path}`, timeoutMs), timeoutMs) - if (result.exitCode !== 0) throw new Error(`Couldn't update the computer's desktop. ${result.stderr.trim().slice(0, 300)}`) + /** + * Hands the helper this runner's guest bundle; it checks the checksum and that the image is its own. One at a time + * per computer: the helper writes the bundle in place, so two installs at once could leave half of one. + */ + private install(box: SandboxHandle, timeoutMs: number) { + const key = JSON.stringify(box.ref) + let pending = this.installing.get(key) + if (!pending) { + pending = this.upload(box, Buffer.from(guestBundle, "base64"), path => + this.bridge(box, `install ${this.image} ${this.guest} ${identity.bundleSha256} < ${path}`, timeoutMs), timeoutMs) + .then(result => { if (result.exitCode !== 0) throw new Error(`Couldn't update the computer's desktop. ${result.stderr.trim().slice(0, 300)}`) }) + .finally(() => this.installing.delete(key)) + this.installing.set(key, pending) + } + return pending } /** The browser profile or workspace as a gzipped tar, or null when this computer can't export it in time (or it's too big). */