From 2fa12cee2db9668b36f7bdcd02f27516d017cd60 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 25 Aug 2026 16:32:05 +0000 Subject: [PATCH 01/72] perf: scope workspace.activity.list to config-known workspaces and prune stale extensionMetadata entries - getActivityList() now iterates config-known workspace ids instead of every snapshot key on disk (fail-open fallback to the legacy union if config cannot be read); tombstone semantics for known workspaces are preserved. - remove() deletes the workspace's extensionMetadata.json entry after config deregistration (best-effort). - One-time lazy cleanup per process prunes entries whose workspace no longer exists; known-ids are fetched inside the file's serialized mutation for loss safety; survivors round-trip verbatim (upgrade/downgrade-safe). Fixes #3959 --- .../services/ExtensionMetadataService.test.ts | 68 +++++++- src/node/services/ExtensionMetadataService.ts | 39 +++++ src/node/services/workspaceService.test.ts | 160 ++++++++++++++++++ src/node/services/workspaceService.ts | 82 +++++++-- 4 files changed, 337 insertions(+), 12 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index f9813902ca..2b238ff621 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test"; -import { mkdtemp, rm, writeFile } from "fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "fs/promises"; import { tmpdir } from "os"; import * as path from "path"; import { ExtensionMetadataService } from "./ExtensionMetadataService"; @@ -377,4 +377,70 @@ describe("ExtensionMetadataService", () => { const snapshots = await service.getAllSnapshots(); expect(snapshots.get("workspace-2")).toEqual(cleared); }); + + test("pruneMissingWorkspaces drops unknown entries and keeps known ones", async () => { + await service.updateRecency("known-workspace", 100); + await service.updateRecency("stale-workspace", 200); + await service.updateRecency("another-stale", 300); + + const prunedCount = await service.pruneMissingWorkspaces(() => + Promise.resolve(new Set(["known-workspace"])) + ); + expect(prunedCount).toBe(2); + + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("known-workspace")?.recency).toBe(100); + expect(snapshots.has("stale-workspace")).toBe(false); + expect(snapshots.has("another-stale")).toBe(false); + }); + + test("pruneMissingWorkspaces preserves unrecognized fields on surviving entries", async () => { + // Upgrade↔downgrade: a newer build may persist fields this build does not + // know about; pruning stale siblings must not strip them. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "known-workspace": { + recency: 100, + streaming: false, + lastModel: null, + lastThinkingLevel: null, + agentStatus: null, + fieldFromNewerBuild: { nested: true }, + }, + "stale-workspace": { recency: 200, streaming: false }, + }, + }) + ); + + expect( + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set(["known-workspace"]))) + ).toBe(1); + + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(Object.keys(persisted.workspaces)).toEqual(["known-workspace"]); + expect( + (persisted.workspaces["known-workspace"] as { fieldFromNewerBuild?: unknown }) + .fieldFromNewerBuild + ).toEqual({ nested: true }); + }); + + test("pruneMissingWorkspaces does not rewrite the file when nothing is stale", async () => { + // Compact (non-pretty) serialization: any rewrite through save() would + // change the raw bytes, so byte-equality proves no write happened. + const rawContent = JSON.stringify({ + version: 1, + workspaces: { + "known-workspace": { recency: 100, streaming: false }, + }, + }); + await writeFile(filePath, rawContent); + + expect( + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set(["known-workspace"]))) + ).toBe(0); + expect(await readFile(filePath, "utf-8")).toBe(rawContent); + }); }); diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 02f2e35346..daa721a6d0 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -385,6 +385,45 @@ export class ExtensionMetadataService { }); } + /** + * Remove entries whose workspace no longer exists. Removed workspaces and + * sub-agents were historically never pruned here, so long-lived deployments + * accumulate thousands of stale entries that inflate every read and rewrite + * of this file (issue #3959 measured 13,895 entries for 1,513 known + * workspaces). Called once per process; `deleteWorkspace` keeps the file + * bounded afterwards. + * + * Loss safety: `getKnownWorkspaceIds` is invoked INSIDE the serialized + * mutation, before the file is loaded. All in-process writers go through the + * same mutation queue, so any entry visible in the loaded file was written + * by a mutation that completed before this one started — and a workspace is + * registered in config before its first metadata write, so a live + * workspace's id is always present in the fetched set. + * + * Upgrade↔downgrade safety: surviving entries are round-tripped verbatim + * (no coercion), so fields written by other builds are preserved and the + * on-disk format is unchanged. + */ + async pruneMissingWorkspaces( + getKnownWorkspaceIds: () => Promise> + ): Promise { + return this.withSerializedMutation(async () => { + const knownWorkspaceIds = await getKnownWorkspaceIds(); + const data = await this.load(); + let prunedCount = 0; + for (const workspaceId of Object.keys(data.workspaces)) { + if (!knownWorkspaceIds.has(workspaceId)) { + delete data.workspaces[workspaceId]; + prunedCount++; + } + } + if (prunedCount > 0) { + await this.save(data); + } + return prunedCount; + }); + } + /** * Clear all streaming flags. * Call this on app startup to clean up stale streaming states from crashes. diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4b480c2de2..7c37e31758 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1398,6 +1398,16 @@ describe("WorkspaceService bash monitor wakes", () => { const { config, cleanup } = await createTestHistoryService(); try { const workspaceId = "bash-monitor-tombstone-failed-clear"; + // getActivityList only emits entries for config-known workspaces; the + // tombstone contract below is scoped to known ids. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); let activeMonitorCount = 1; const backgroundProcessManager = Object.assign(new EventEmitter(), { cleanup: mock(() => Promise.resolve()), @@ -1451,6 +1461,16 @@ describe("WorkspaceService bash monitor wakes", () => { const { config, cleanup } = await createTestHistoryService(); try { const workspaceId = "bash-monitor-tombstone"; + // getActivityList only emits entries for config-known workspaces; the + // tombstone contract below is scoped to known ids. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); let activeMonitorCount = 1; const backgroundProcessManager = Object.assign(new EventEmitter(), { cleanup: mock(() => Promise.resolve()), @@ -3014,6 +3034,15 @@ describe("WorkspaceService workflow activity", () => { try { const workspaceId = "workflow-activity-race"; + // getActivityList only emits entries for config-known workspaces. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); const workspaceService = createWorkspaceServiceForTest({ config, historyService, @@ -3065,6 +3094,17 @@ describe("WorkspaceService workflow activity", () => { try { const workspaceId = "workflow-activity-overlap"; + // getActivityList only emits entries for config-known workspaces; keep + // this workspace known so the zero-count assertion below exercises the + // tombstone path rather than trivially missing the entry. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); const workspaceService = createWorkspaceServiceForTest({ config, historyService, @@ -3124,6 +3164,126 @@ describe("WorkspaceService workflow activity", () => { }); }); +describe("WorkspaceService activity list scoping", () => { + test("drops stale extension metadata entries and lazily prunes them once", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "activity-scoping-known"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + // Simulates the leaked entry of a removed workspace/sub-agent. + await extensionMetadata.updateRecency("removed-workspace", 200); + const pruneSpy = spyOn(extensionMetadata, "pruneMissingWorkspaces"); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList[workspaceId]?.recency).toBe(100); + expect(activityList["removed-workspace"]).toBeUndefined(); + + // The one-time lazy cleanup dropped the stale entry from disk while + // keeping the still-existing workspace's entry. + const snapshots = await extensionMetadata.getAllSnapshots(); + expect(snapshots.has("removed-workspace")).toBe(false); + expect(snapshots.get(workspaceId)?.recency).toBe(100); + + // One-time: a second bootstrap must not re-run the cleanup scan. + await workspaceService.getActivityList(); + expect(pruneSpy).toHaveBeenCalledTimes(1); + } finally { + await cleanup(); + } + }); + + test("falls back to the unscoped union when config workspaces cannot be listed", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("possibly-live", 100); + const metadataSpy = spyOn(config, "getAllWorkspaceMetadata").mockImplementation(() => + Promise.reject(new Error("config unavailable")) + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Fail open: without the config view, stale ids cannot be told apart + // from live ones, so nothing may be dropped from the list or pruned + // from disk. + const activityList = await workspaceService.getActivityList(); + expect(activityList["possibly-live"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); + } finally { + metadataSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("prunes the extension metadata entry after a workspace is removed", async () => { + const { historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "remove-prunes-metadata"; + const tempRoot = await fsPromises.mkdtemp(path.join(tmpdir(), "mux-remove-metadata-")); + try { + const sessionRoot = path.join(tempRoot, "sessions"); + await fsPromises.mkdir(path.join(sessionRoot, workspaceId), { recursive: true }); + + const deleteWorkspace = mock(() => Promise.resolve()); + const extensionMetadata = { + ...mockExtensionMetadataService, + deleteWorkspace, + } as unknown as ExtensionMetadataService; + const mockConfig: Partial = { + rootDir: path.join(tempRoot, "root"), + srcDir: "/tmp/src", + getSessionDir: mock((id: string) => path.join(sessionRoot, id)), + removeWorkspace: mock(() => Promise.resolve()), + findWorkspace: mock(() => null), + loadConfigOrDefault: mock(() => ({ projects: new Map() })), + }; + const workspaceService = createWorkspaceServiceForTest({ + config: mockConfig, + historyService, + extensionMetadata, + aiService: createMockAIService({ + isStreaming: mock(() => false), + stopStream: mock(() => Promise.resolve(Ok(undefined))), + getWorkspaceMetadata: mock(() => + Promise.resolve( + Ok(createFrontendWorkspaceMetadata({ id: workspaceId, name: workspaceId })) + ) + ), + }), + }); + + const removeResult = await workspaceService.remove(workspaceId, true); + expect(removeResult.success).toBe(true); + expect(deleteWorkspace).toHaveBeenCalledWith(workspaceId); + } finally { + await fsPromises.rm(tempRoot, { recursive: true, force: true }); + await cleanup(); + } + }); +}); + describe("WorkspaceService workflow invocation events", () => { test("emits workflow slash invocation rows through the active session chat stream", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 4e30fb0027..2b81bc707f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6347,6 +6347,20 @@ export class WorkspaceService extends EventEmitter { removedFromConfig = true; this.autoTitlingWorkspaces.delete(workspaceId); + // Deregistration succeeded: drop the workspace's activity/status entry + // so extensionMetadata.json stays bounded (stale entries were + // historically never pruned and grew monotonically, issue #3959). + // Best-effort: a missed delete is reclaimed by the one-time + // pruneStaleExtensionMetadataOnce pass on a later process start. + try { + await this.extensionMetadata.deleteWorkspace(workspaceId); + } catch (error) { + log.debug("Failed to prune extension metadata after workspace removal", { + workspaceId, + error: getErrorMessage(error), + }); + } + if (removedMetadata || persistedWorkspace) { await this.syncCodeWorkspaceFiles( removedMetadata ?? { @@ -12748,22 +12762,68 @@ export class WorkspaceService extends EventEmitter { } } + /** + * One-time lazy cleanup for pre-existing deployments: drop + * extensionMetadata.json entries whose workspace no longer exists in + * config. remove() prunes going forward, but entries that leaked before + * that hook existed (issue #3959) would otherwise bloat the file — and + * every serialized rewrite of it — forever. Runs at most once per process + * from the activity bootstrap path; never a per-read scan. + */ + private prunedStaleExtensionMetadata = false; + private async pruneStaleExtensionMetadataOnce(): Promise { + if (this.prunedStaleExtensionMetadata) { + return; + } + // Latch before awaiting so concurrent bootstraps don't queue redundant + // prunes; a failed attempt is retried on the next process start rather + // than on every read. + this.prunedStaleExtensionMetadata = true; + try { + const prunedCount = await this.extensionMetadata.pruneMissingWorkspaces(async () => { + // Fetched inside the file's serialized mutation (see + // pruneMissingWorkspaces) so a concurrently created workspace cannot + // lose its just-written entry. + const allMetadata = await this.config.getAllWorkspaceMetadata(); + return new Set(allMetadata.map((metadata) => metadata.id)); + }); + if (prunedCount > 0) { + log.info(`Pruned ${prunedCount} stale extension metadata entries`); + } + } catch (error) { + log.debug("Failed to prune stale extension metadata entries", { error }); + } + } + async getActivityList(): Promise> { try { + await this.pruneStaleExtensionMetadataOnce(); const snapshots = await this.extensionMetadata.getAllSnapshots(); - const workspaceIds = new Set(snapshots.keys()); - for (const workspaceId of this.activeWorkflowRunIdsByWorkspace.keys()) { - workspaceIds.add(workspaceId); - } - for (const workspaceId of this.bashMonitorSeenWorkspaces) { - workspaceIds.add(workspaceId); - } + // Scope the list to config-known workspaces. extensionMetadata.json was + // historically never pruned, so long-lived deployments accumulate stale + // entries for removed workspaces/sub-agents by the thousands (issue + // #3959: 13,901 entries / 2.89 MB / ~59 s per bootstrap while the + // sidebar needed ~246). Stale ids must neither inflate the payload nor + // trigger the per-id workflow-run bootstrap disk probe below. Known ids + // WITHOUT a snapshot still flow through the tombstone logic below — + // scoping only drops ids that are not in config at all. + let workspaceIds: Set; try { - for (const metadata of await this.config.getAllWorkspaceMetadata()) { - workspaceIds.add(metadata.id); - } + workspaceIds = new Set( + (await this.config.getAllWorkspaceMetadata()).map((metadata) => metadata.id) + ); } catch (error) { - log.debug("Failed to include all workspaces while listing activity", { error }); + // Fail open: without the config view, stale ids cannot be told apart + // from live ones, and dropping live entries would strand renderer + // activity state. Fall back to the legacy unscoped union. + log.debug("Failed to scope activity list to known workspaces", { error }); + workspaceIds = new Set(snapshots.keys()); + for (const workspaceId of this.activeWorkflowRunIdsByWorkspace.keys()) { + workspaceIds.add(workspaceId); + } + for (const workspaceId of this.bashMonitorSeenWorkspaces) { + workspaceIds.add(workspaceId); + } } const entries = await Promise.all( From 404a3755eae05f00a6b746566f54107e5509d876 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 25 Aug 2026 17:05:01 +0000 Subject: [PATCH 02/72] review: close cross-process prune race and cover failed task-create rollback - pruneMissingWorkspaces now loads the file BEFORE fetching known ids (fresh config disk read inside the serialized mutation), so a workspace created concurrently by another backend process cannot be misclassified as stale. - Centralize entry cleanup as WorkspaceService.discardExtensionMetadataEntry; TaskService.rollbackFailedTaskCreate now prunes the entry a failed send's scheduled metadata writes could otherwise leak after deregistration. --- .../services/ExtensionMetadataService.test.ts | 25 ++++++++++++ src/node/services/ExtensionMetadataService.ts | 20 +++++++--- src/node/services/taskService.test.ts | 13 ++++++- src/node/services/taskService.ts | 7 ++++ src/node/services/workspaceService.test.ts | 22 +++++++++++ src/node/services/workspaceService.ts | 38 ++++++++++++------- 6 files changed, 105 insertions(+), 20 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 2b238ff621..ffad58c28a 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -427,6 +427,31 @@ describe("ExtensionMetadataService", () => { ).toEqual({ nested: true }); }); + test("pruneMissingWorkspaces fetches known ids only after loading the file", async () => { + // The cross-process loss-safety argument (see the pruneMissingWorkspaces + // doc comment) requires the known-ids fetch to observe every workspace + // registration that preceded an entry visible in the loaded file — i.e. + // load first, fetch second. Fetch-first would misclassify a concurrently + // created workspace's fresh entry as stale. + await service.updateRecency("known-workspace", 100); + const order: string[] = []; + const internals = service as unknown as ExtensionMetadataServiceInternals; + const originalLoad = internals.load.bind(service); + internals.load = async () => { + order.push("load"); + return originalLoad(); + }; + try { + await service.pruneMissingWorkspaces(() => { + order.push("fetch-known-ids"); + return Promise.resolve(new Set(["known-workspace"])); + }); + } finally { + internals.load = originalLoad; + } + expect(order).toEqual(["load", "fetch-known-ids"]); + }); + test("pruneMissingWorkspaces does not rewrite the file when nothing is stale", async () => { // Compact (non-pretty) serialization: any rewrite through save() would // change the raw bytes, so byte-equality proves no write happened. diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index daa721a6d0..13f531a72a 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -394,11 +394,19 @@ export class ExtensionMetadataService { * bounded afterwards. * * Loss safety: `getKnownWorkspaceIds` is invoked INSIDE the serialized - * mutation, before the file is loaded. All in-process writers go through the - * same mutation queue, so any entry visible in the loaded file was written - * by a mutation that completed before this one started — and a workspace is - * registered in config before its first metadata write, so a live - * workspace's id is always present in the fetched set. + * mutation and strictly AFTER the file is loaded, and the callback reads + * config fresh from disk. A workspace is durably registered in config + * before its first metadata write, so every entry visible in the loaded + * file belongs to a workspace whose registration is already on disk — the + * post-load fetch therefore always includes live entries' ids, even for + * workspaces created concurrently by another backend process + * (XUM_ALLOW_MULTIPLE_INSTANCES). Fetching before the load would leave a + * window where another process registers + writes a fresh entry that the + * stale known-ids set misclassifies as prunable. + * + * (A concurrent foreign-process write landing between our load and save can + * still be lost to this whole-file rewrite — that lost-update window is + * inherent to every existing writer of this file and unchanged here.) * * Upgrade↔downgrade safety: surviving entries are round-tripped verbatim * (no coercion), so fields written by other builds are preserved and the @@ -408,8 +416,8 @@ export class ExtensionMetadataService { getKnownWorkspaceIds: () => Promise> ): Promise { return this.withSerializedMutation(async () => { - const knownWorkspaceIds = await getKnownWorkspaceIds(); const data = await this.load(); + const knownWorkspaceIds = await getKnownWorkspaceIds(); let prunedCount = 0; for (const workspaceId of Object.keys(data.workspaces)) { if (!knownWorkspaceIds.has(workspaceId)) { diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 6e590f6676..df0b1f3c7b 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -592,6 +592,7 @@ function createWorkspaceServiceMocks( emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; create: ReturnType; + discardExtensionMetadataEntry: ReturnType; } { const sendMessage = overrides?.sendMessage ?? mock((): Promise> => Promise.resolve(Ok(undefined))); @@ -678,10 +679,12 @@ function createWorkspaceServiceMocks( (): Promise> => Promise.resolve(Err("workspaceService.create not mocked")) ); + const discardExtensionMetadataEntry = mock((): Promise => Promise.resolve()); return { workspaceService: { create, + discardExtensionMetadataEntry, // No-op by default: task-create tests exercise launch flow, not the // registration-time plugin-override sanitizer (workspaceService.test.ts // covers it). Returning undefined means "clean". @@ -731,6 +734,7 @@ function createWorkspaceServiceMocks( countQueuedAgentPeerMessages, } as unknown as WorkspaceService, create, + discardExtensionMetadataEntry, sendMessage, resumeStream, clearQueue, @@ -23799,7 +23803,9 @@ describe("TaskService", () => { ); const { aiService } = createAIServiceMocks(config); const failingSendMessage = mock(() => Promise.resolve(Err("send failed"))); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage: failingSendMessage }); + const { workspaceService, discardExtensionMetadataEntry } = createWorkspaceServiceMocks({ + sendMessage: failingSendMessage, + }); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); const created = await createAgentTask(taskService, parentId, "do the thing"); @@ -23812,6 +23818,11 @@ describe("TaskService", () => { .some((w) => w.id === "aaaaaaaaaa"); expect(stillExists).toBe(false); + // Rollback must also drop the extension-metadata entry: the failed send + // may already have scheduled metadata writes that would otherwise leak a + // stale key after deregistration (#3959). + expect(discardExtensionMetadataEntry).toHaveBeenCalledWith("aaaaaaaaaa"); + const workspaceName = "agent_explore_aaaaaaaaaa"; const workspacePath = runtime.getWorkspacePath(projectPath, workspaceName); let workspacePathExists = true; diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 4a62f33058..d3332a0bd8 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7319,6 +7319,13 @@ export class TaskService { }); } + // A create that failed after sendMessage may already have scheduled + // extension-metadata writes (e.g. the recency update), which would + // recreate the entry after the deregistration above and leak a stale key + // until the next process start's lazy prune. Best-effort, like the + // config removal above. + await this.workspaceService.discardExtensionMetadataEntry(taskId); + this.workspaceService.emit("metadata", { workspaceId: taskId, metadata: null }); if (options?.preservePhysicalWorkspace) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7c37e31758..fd81cf6ec9 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3282,6 +3282,28 @@ describe("WorkspaceService activity list scoping", () => { await cleanup(); } }); + + test("discardExtensionMetadataEntry swallows deletion failures", async () => { + // Rollback paths (e.g. TaskService's failed task-create rollback) call + // this best-effort; a metadata disk failure must not abort the rollback. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const deleteWorkspace = mock(() => Promise.reject(new Error("disk full"))); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata: { + ...mockExtensionMetadataService, + deleteWorkspace, + } as unknown as ExtensionMetadataService, + }); + + await workspaceService.discardExtensionMetadataEntry("rollback-ws"); + expect(deleteWorkspace).toHaveBeenCalledWith("rollback-ws"); + } finally { + await cleanup(); + } + }); }); describe("WorkspaceService workflow invocation events", () => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2b81bc707f..f7986a8caf 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6350,16 +6350,7 @@ export class WorkspaceService extends EventEmitter { // Deregistration succeeded: drop the workspace's activity/status entry // so extensionMetadata.json stays bounded (stale entries were // historically never pruned and grew monotonically, issue #3959). - // Best-effort: a missed delete is reclaimed by the one-time - // pruneStaleExtensionMetadataOnce pass on a later process start. - try { - await this.extensionMetadata.deleteWorkspace(workspaceId); - } catch (error) { - log.debug("Failed to prune extension metadata after workspace removal", { - workspaceId, - error: getErrorMessage(error), - }); - } + await this.discardExtensionMetadataEntry(workspaceId); if (removedMetadata || persistedWorkspace) { await this.syncCodeWorkspaceFiles( @@ -12762,6 +12753,26 @@ export class WorkspaceService extends EventEmitter { } } + /** + * Best-effort removal of a deregistered workspace's activity/status entry + * from extensionMetadata.json. Used by remove() and by rollback paths that + * deregister via config.removeWorkspace directly (e.g. TaskService's failed + * task-create rollback, where a send that failed mid-create may already + * have scheduled metadata writes that would recreate the entry after + * deregistration). A missed delete is reclaimed by the one-time + * pruneStaleExtensionMetadataOnce pass on a later process start. + */ + async discardExtensionMetadataEntry(workspaceId: string): Promise { + try { + await this.extensionMetadata.deleteWorkspace(workspaceId); + } catch (error) { + log.debug("Failed to prune extension metadata after workspace deregistration", { + workspaceId, + error: getErrorMessage(error), + }); + } + } + /** * One-time lazy cleanup for pre-existing deployments: drop * extensionMetadata.json entries whose workspace no longer exists in @@ -12781,9 +12792,10 @@ export class WorkspaceService extends EventEmitter { this.prunedStaleExtensionMetadata = true; try { const prunedCount = await this.extensionMetadata.pruneMissingWorkspaces(async () => { - // Fetched inside the file's serialized mutation (see - // pruneMissingWorkspaces) so a concurrently created workspace cannot - // lose its just-written entry. + // Invoked inside the file's serialized mutation AFTER the file load, + // reading config fresh from disk (see pruneMissingWorkspaces), so a + // concurrently created workspace — even in another backend process — + // cannot lose its just-written entry. const allMetadata = await this.config.getAllWorkspaceMetadata(); return new Set(allMetadata.map((metadata) => metadata.id)); }); From 7d895ed2e19cf9be68cba78f661b4bb279572602 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 25 Aug 2026 17:25:27 +0000 Subject: [PATCH 03/72] review: abort prune and scoping when config load falls back to defaults loadConfigOrDefault swallows read/parse failures and resolves as the EMPTY default, which the destructive prune cannot distinguish from a truly empty config (it would wipe every extensionMetadata entry) and which would drop every live entry from the activity list. Thread throwOnError through getAllWorkspaceMetadata and use the strict read in both call sites, matching cleanupOrphanSessionDirs' existing internal guard. Regression test now corrupts config.json on disk (red-proofed against the unfixed code). --- src/node/config.ts | 14 +++++++-- src/node/services/workspaceService.test.ts | 35 +++++++++++----------- src/node/services/workspaceService.ts | 16 ++++++++-- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/node/config.ts b/src/node/config.ts index ab3f6882f8..78f6013c35 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2200,8 +2200,18 @@ export class Config { * If missing from config or legacy metadata, a new timestamp is assigned and * saved to config for subsequent loads. */ - async getAllWorkspaceMetadata(): Promise { - const config = this.loadConfigOrDefault(); + async getAllWorkspaceMetadata(options?: { + /** + * Throw on config read/parse failure instead of silently resolving with + * the empty default. Callers that make destructive decisions based on + * "workspace is not in config" (e.g. extension-metadata pruning) must use + * this: the swallowed-failure default is indistinguishable from a truly + * empty config. A missing config file still resolves as empty (healthy + * fresh install), matching loadConfigOrDefault. + */ + throwOnError?: boolean; + }): Promise { + const config = this.loadConfigOrDefault({ throwOnError: options?.throwOnError }); const workspaceMetadata: FrontendWorkspaceMetadata[] = []; // Read-time migrations recorded here are re-applied to a FRESH config snapshot inside // editConfig below. Persisting the local `config` snapshot directly (the old diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index fd81cf6ec9..efac595bcd 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3215,24 +3215,23 @@ describe("WorkspaceService activity list scoping", () => { path.join(config.rootDir, "extensionMetadata.json") ); await extensionMetadata.updateRecency("possibly-live", 100); - const metadataSpy = spyOn(config, "getAllWorkspaceMetadata").mockImplementation(() => - Promise.reject(new Error("config unavailable")) - ); - try { - const workspaceService = createWorkspaceServiceForTest({ - config, - historyService, - extensionMetadata, - }); - // Fail open: without the config view, stale ids cannot be told apart - // from live ones, so nothing may be dropped from the list or pruned - // from disk. - const activityList = await workspaceService.getActivityList(); - expect(activityList["possibly-live"]?.recency).toBe(100); - expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); - } finally { - metadataSpy.mockRestore(); - } + // Real on-disk corruption: loadConfigOrDefault SWALLOWS this and + // resolves with the empty default unless callers opt into the strict + // read. Without throwOnError, this state would silently wipe every + // metadata entry (prune sees an "empty" config) and drop every live + // entry from the list instead of reaching the fail-open fallback. + await fsPromises.writeFile(path.join(config.rootDir, "config.json"), "{not json"); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Fail open: without the config view, stale ids cannot be told apart + // from live ones, so nothing may be dropped from the list or pruned + // from disk. + const activityList = await workspaceService.getActivityList(); + expect(activityList["possibly-live"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); } finally { await cleanup(); } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index f7986a8caf..387bfd6260 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12796,7 +12796,14 @@ export class WorkspaceService extends EventEmitter { // reading config fresh from disk (see pruneMissingWorkspaces), so a // concurrently created workspace — even in another backend process — // cannot lose its just-written entry. - const allMetadata = await this.config.getAllWorkspaceMetadata(); + // + // throwOnError: a corrupted/unreadable config.json otherwise resolves + // as the swallowed-failure EMPTY default, which this destructive + // prune cannot distinguish from a truly empty config — it would + // delete every entry. Throwing aborts the prune (caught below); a + // missing file still resolves as a healthy empty config. Same + // precedent as cleanupOrphanSessionDirs' internal strict read. + const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); return new Set(allMetadata.map((metadata) => metadata.id)); }); if (prunedCount > 0) { @@ -12821,8 +12828,13 @@ export class WorkspaceService extends EventEmitter { // scoping only drops ids that are not in config at all. let workspaceIds: Set; try { + // throwOnError so a corrupted config.json actually reaches the + // fail-open fallback below instead of silently resolving as the + // empty default and dropping every live entry from the list. workspaceIds = new Set( - (await this.config.getAllWorkspaceMetadata()).map((metadata) => metadata.id) + (await this.config.getAllWorkspaceMetadata({ throwOnError: true })).map( + (metadata) => metadata.id + ) ); } catch (error) { // Fail open: without the config view, stale ids cannot be told apart From f0f29f75791fb4186005c6cd313c0b20642db215 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 08:54:56 +0000 Subject: [PATCH 04/72] review: reject structurally invalid project lists in strict config loads A parseable config.json with a non-array projects value, non-pair entries, non-object project configs, or a non-array workspaces value was lenient- normalized into an empty/partial workspace view even under throwOnError, so destructive not-in-config decisions (extension-metadata prune, activity scoping) saw live workspaces as removed. Strict mode now validates the projects structure before normalization; lenient loads keep self-healing. Also hardens cleanupOrphanSessionDirs' existing strict read for free. --- src/node/config.test.ts | 42 +++++++++++++++ src/node/config.ts | 26 ++++++++++ src/node/services/workspaceService.test.ts | 59 +++++++++++++--------- 3 files changed, 102 insertions(+), 25 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index d624917816..631dd8f86d 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -861,6 +861,48 @@ describe("Config", () => { }); }); + describe("strict structural validation (throwOnError)", () => { + // Destructive callers (extension-metadata pruning, orphan session-dir + // cleanup) must never receive a lenient-normalized empty/partial workspace + // view for a parseable but structurally invalid config: they would treat + // the omitted live workspaces as removed and delete their data. + const invalidShapes: Array<[string, unknown]> = [ + ["non-array projects", { projects: {} }], + ["non-pair projects entry", { projects: ["not-a-pair"] }], + ["non-object project config", { projects: [["/repo", null]] }], + ["non-array workspaces", { projects: [["/repo", { workspaces: "bogus" }]] }], + ]; + + for (const [label, shape] of invalidShapes) { + it(`rejects ${label} in strict mode while lenient mode still loads`, async () => { + fs.writeFileSync(path.join(tempDir, "config.json"), JSON.stringify(shape)); + const strictConfig = new Config(tempDir); + expect(() => strictConfig.loadConfigOrDefault({ throwOnError: true })).toThrow(); + // try/catch instead of rejects.toThrow: the node-side type-aware lint + // flags awaiting bun's expect() chain as await-thenable. + let strictMetadataRejected = false; + try { + await new Config(tempDir).getAllWorkspaceMetadata({ throwOnError: true }); + } catch { + strictMetadataRejected = true; + } + expect(strictMetadataRejected).toBe(true); + // Ordinary loads keep the historical self-healing behavior: they + // never throw for these shapes (normalized stubs may survive). + expect(() => new Config(tempDir).loadConfigOrDefault()).not.toThrow(); + }); + } + + it("accepts an absent projects key in strict mode (healthy empty config)", () => { + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ defaultProjectDir: "/tmp" }) + ); + const loaded = new Config(tempDir).loadConfigOrDefault({ throwOnError: true }); + expect(loaded.projects.size).toBe(0); + }); + }); + describe("legacy task variant compatibility", () => { it("loads variant children as ordinary sub-agents without destroying downgrade metadata", async () => { const configFile = path.join(tempDir, "config.json"); diff --git a/src/node/config.ts b/src/node/config.ts index 78f6013c35..f50a2b83ce 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1242,6 +1242,32 @@ export class Config { // Config is stored as array of [path, config] pairs. // Older/newer files may omit `projects`; treat missing/invalid values as an empty map // so top-level settings (provider/runtime/server preferences) still load. + // + // Strict mode must NOT accept that lenient normalization: callers that + // make destructive "id is not in config" decisions (extension-metadata + // pruning, orphan session-dir cleanup) would interpret a structurally + // invalid-but-parseable file (e.g. `projects: {}` or a non-array + // `workspaces`) as an empty/partial workspace set and delete live + // data. A genuinely ABSENT `projects` key stays valid in strict mode: + // it is how older/newer builds persist a config with no projects. + if (options?.throwOnError && parsed.projects !== undefined) { + if (!Array.isArray(parsed.projects)) { + throw new Error("Config projects must be an array of [path, config] pairs"); + } + for (const pair of parsed.projects) { + if (!Array.isArray(pair)) { + throw new Error("Config projects entries must be [path, config] pairs"); + } + const projectConfig: unknown = pair[1]; + if (projectConfig === null || typeof projectConfig !== "object") { + throw new Error("Config project entries must be objects"); + } + const workspaces = (projectConfig as { workspaces?: unknown }).workspaces; + if (workspaces !== undefined && !Array.isArray(workspaces)) { + throw new Error("Config project workspaces must be an array"); + } + } + } const rawPairs = Array.isArray(parsed.projects) ? parsed.projects : []; // Migrate: normalize project paths by stripping trailing slashes // This fixes configs created with paths like "/home/user/project/" diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index efac595bcd..ca7657ebf8 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3209,31 +3209,40 @@ describe("WorkspaceService activity list scoping", () => { }); test("falls back to the unscoped union when config workspaces cannot be listed", async () => { - const { config, historyService, cleanup } = await createTestHistoryService(); - try { - const extensionMetadata = new ExtensionMetadataService( - path.join(config.rootDir, "extensionMetadata.json") - ); - await extensionMetadata.updateRecency("possibly-live", 100); - // Real on-disk corruption: loadConfigOrDefault SWALLOWS this and - // resolves with the empty default unless callers opt into the strict - // read. Without throwOnError, this state would silently wipe every - // metadata entry (prune sees an "empty" config) and drop every live - // entry from the list instead of reaching the fail-open fallback. - await fsPromises.writeFile(path.join(config.rootDir, "config.json"), "{not json"); - const workspaceService = createWorkspaceServiceForTest({ - config, - historyService, - extensionMetadata, - }); - // Fail open: without the config view, stale ids cannot be told apart - // from live ones, so nothing may be dropped from the list or pruned - // from disk. - const activityList = await workspaceService.getActivityList(); - expect(activityList["possibly-live"]?.recency).toBe(100); - expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); - } finally { - await cleanup(); + // Real on-disk corruption shapes. loadConfigOrDefault SWALLOWS the first + // (parse failure) and lenient-normalizes the rest (parseable but + // structurally invalid) into an empty/partial workspace view unless + // callers opt into the strict read. Without throwOnError + strict + // structural validation, each of these states would silently wipe every + // metadata entry (prune sees an "empty" config) and drop every live + // entry from the list instead of reaching the fail-open fallback. + const corruptConfigs = [ + "{not json", + JSON.stringify({ projects: {} }), + JSON.stringify({ projects: [["/tmp/project", { workspaces: "bogus" }]] }), + ]; + for (const corruptConfig of corruptConfigs) { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("possibly-live", 100); + await fsPromises.writeFile(path.join(config.rootDir, "config.json"), corruptConfig); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Fail open: without a trustworthy config view, stale ids cannot be + // told apart from live ones, so nothing may be dropped from the list + // or pruned from disk. + const activityList = await workspaceService.getActivityList(); + expect(activityList["possibly-live"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); + } finally { + await cleanup(); + } } }); From b3f6254e25b2b146020a5c4ecaf76fd20162e3af Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 09:08:59 +0000 Subject: [PATCH 05/72] review: derive prune ids from the raw config superset, not lossy normalization Strict structural validation could not enumerate every entry the lenient normalization discards (e.g. [null, {...}] pairs filtered for invalid paths). Instead of narrowing validation further, destructive callers now union a raw-scan superset of every workspace id in the persisted projects subtree (readPersistedWorkspaceIdSuperset; over-collection is safe, under-collection destroys live data) with the strict normalized view (covers in-memory migration ids). Orphan session-dir cleanup gets the same guarantee. Throws on unreadable/unparseable config, aborting both sweeps. --- src/node/config.test.ts | 28 +++++++++++++ src/node/config.ts | 48 ++++++++++++++++++++++ src/node/services/workspaceService.test.ts | 29 +++++++++++++ src/node/services/workspaceService.ts | 28 +++++++++---- 4 files changed, 126 insertions(+), 7 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 631dd8f86d..afdafcf455 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -903,6 +903,34 @@ describe("Config", () => { }); }); + describe("readPersistedWorkspaceIdSuperset", () => { + it("collects ids from entries that lenient normalization discards", () => { + // `[null, ...]` fails the project-path filter and vanishes from the + // normalized view; the raw superset must still surface its workspace id + // so destructive callers never treat that live workspace as removed. + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ + projects: [ + [null, { workspaces: [{ id: "live-discarded", path: "/tmp/x" }] }], + ["/repo", { workspaces: [{ id: "live-normal", path: "/repo/ws", name: "ws" }] }], + ], + }) + ); + const superset = new Config(tempDir).readPersistedWorkspaceIdSuperset(); + expect(superset.has("live-discarded")).toBe(true); + expect(superset.has("live-normal")).toBe(true); + }); + + it("resolves empty for a missing file and throws on unparseable content", () => { + expect(new Config(tempDir).readPersistedWorkspaceIdSuperset().size).toBe(0); + fs.writeFileSync(path.join(tempDir, "config.json"), "{not json"); + expect(() => new Config(tempDir).readPersistedWorkspaceIdSuperset()).toThrow(); + fs.writeFileSync(path.join(tempDir, "config.json"), JSON.stringify(["array-root"])); + expect(() => new Config(tempDir).readPersistedWorkspaceIdSuperset()).toThrow(); + }); + }); + describe("legacy task variant compatibility", () => { it("loads variant children as ordinary sub-agents without destroying downgrade metadata", async () => { const configFile = path.join(tempDir, "config.json"); diff --git a/src/node/config.ts b/src/node/config.ts index f50a2b83ce..c7b69eed64 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1083,6 +1083,54 @@ export class Config { ); } + /** + * Maximal superset of workspace ids present in the RAW persisted config, + * for destructive "id is not in config" decisions (extension-metadata + * pruning, orphan session-dir cleanup). + * + * loadConfigOrDefault's validation/normalization is LOSSY: entries it + * filters or discards (invalid project paths, malformed pairs, entries a + * migration rewrites) simply vanish from the normalized view, so a pruner + * keyed on that view would treat their live workspaces as removed and + * delete their data. This scan instead walks the raw `projects` subtree + * and collects every string `id` it can find, without judging validity — + * over-collection merely retains a stale entry a little longer, while + * under-collection destroys live data. Callers should union this with the + * normalized view (which contains ids created by in-memory migrations). + * + * Throws when the file exists but cannot be read/parsed or the root is not + * a plain object; a missing file resolves as an empty set (fresh install). + */ + readPersistedWorkspaceIdSuperset(): Set { + const ids = new Set(); + if (!fs.existsSync(this.configFile)) { + return ids; + } + const parsedValue: unknown = JSON.parse(fs.readFileSync(this.configFile, "utf-8")); + if (!parsedValue || typeof parsedValue !== "object" || Array.isArray(parsedValue)) { + throw new Error("Config root must be a JSON object"); + } + const collect = (value: unknown): void => { + if (Array.isArray(value)) { + for (const entry of value) { + collect(entry); + } + return; + } + if (value !== null && typeof value === "object") { + const id = (value as { id?: unknown }).id; + if (typeof id === "string" && id.length > 0) { + ids.add(id); + } + for (const nested of Object.values(value)) { + collect(nested); + } + } + }; + collect((parsedValue as { projects?: unknown }).projects); + return ids; + } + loadConfigOrDefault(options?: { throwOnError?: boolean }): ProjectsConfig { // Read as a Buffer and hand the same snapshot to the failure handler: backing up via a // second read could preserve a concurrent writer's replacement instead of the bytes that diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ca7657ebf8..7861a3084c 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3246,6 +3246,35 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("never prunes entries whose config entry is discarded by normalization", async () => { + // A parseable config entry that lenient normalization filters out (null + // project path): the workspace vanishes from the normalized view — and + // thus from the activity list, matching every other renderer surface — + // but its metadata entry must survive the prune (raw-superset guarantee). + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("possibly-live", 100); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[null, { workspaces: [{ id: "possibly-live", path: "/tmp/x" }] }]], + }) + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + await workspaceService.getActivityList(); + expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); + } finally { + await cleanup(); + } + }); + test("prunes the extension metadata entry after a workspace is removed", async () => { const { historyService, cleanup } = await createTestHistoryService(); const workspaceId = "remove-prunes-metadata"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 387bfd6260..ecf368cdf2 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4683,6 +4683,13 @@ export class WorkspaceService extends EventEmitter { const config = this.config.loadConfigOrDefault({ throwOnError: true }); const knownIds = new Set(allMetadata.map((metadata) => metadata.id)); + // Same rationale as the extension-metadata prune: normalization is lossy, + // and a live workspace whose config entry gets filtered (e.g. invalid + // project path) must not have its session data reaped as an orphan. + // Throws on unreadable/unparseable config, aborting this cleanup. + for (const persistedId of this.config.readPersistedWorkspaceIdSuperset()) { + knownIds.add(persistedId); + } // The config load-time migration (removeLegacyXumChatEntries) drops the // removed Chat with Xum workspace from config, which would make its // session dir look orphaned here. Preserve the transcript: a downgraded @@ -12797,14 +12804,21 @@ export class WorkspaceService extends EventEmitter { // concurrently created workspace — even in another backend process — // cannot lose its just-written entry. // - // throwOnError: a corrupted/unreadable config.json otherwise resolves - // as the swallowed-failure EMPTY default, which this destructive - // prune cannot distinguish from a truly empty config — it would - // delete every entry. Throwing aborts the prune (caught below); a - // missing file still resolves as a healthy empty config. Same - // precedent as cleanupOrphanSessionDirs' internal strict read. + // Union of two views, both of which throw (aborting the prune, caught + // below) rather than resolving with a silently lossy id set: + // - the raw persisted superset covers entries loadConfigOrDefault's + // validation/normalization would filter or discard (see + // readPersistedWorkspaceIdSuperset), so a live workspace with a + // malformed config entry is never treated as removed; + // - the strict normalized view covers ids produced by in-memory + // config migrations that are not yet persisted verbatim. + // A missing config file resolves as a healthy empty set in both. + const knownIds = this.config.readPersistedWorkspaceIdSuperset(); const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); - return new Set(allMetadata.map((metadata) => metadata.id)); + for (const metadata of allMetadata) { + knownIds.add(metadata.id); + } + return knownIds; }); if (prunedCount > 0) { log.info(`Pruned ${prunedCount} stale extension metadata entries`); From a0a743e784eb0d7b3ac0bd54ced5a5f363fc4983 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 09:22:41 +0000 Subject: [PATCH 06/72] review: tombstone deleted extension metadata entries against late writers Removal cannot drain every in-flight metadata producer (e.g. the stream-abort fire-and-forget stop-status handler mid readTodos), so a late setStreaming/setTodoStatus/updateRecency could recreate a just-deleted entry and re-leak stale keys until the next process-start prune. Add per-process write tombstones in ExtensionMetadataService: deleteWorkspace and pruneMissingWorkspaces record removed ids; entry-creating writers no-op for tombstoned ids (callers still get a computed, unpersisted snapshot). Synchronous add + FIFO mutation queue leaves no gap; ids are never reused, so tombstones cannot block legitimate new workspaces. --- .../services/ExtensionMetadataService.test.ts | 35 +++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 35 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index ffad58c28a..a7bb01b2e6 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -452,6 +452,41 @@ describe("ExtensionMetadataService", () => { expect(order).toEqual(["load", "fetch-known-ids"]); }); + test("late writers cannot resurrect a deleted workspace entry", async () => { + // Removal cannot drain every in-flight metadata producer (e.g. a + // stream-abort's fire-and-forget stop-status write), so writes landing + // after deleteWorkspace must not recreate the entry on disk. + await service.updateRecency("removed-workspace", 100); + await service.deleteWorkspace("removed-workspace"); + + const lateStreaming = await service.setStreaming("removed-workspace", false, { + hasTodos: false, + }); + // Callers still get a computed snapshot; it just is not persisted. + expect(lateStreaming.streaming).toBe(false); + await service.updateRecency("removed-workspace", 200); + await service.setTodoStatus("removed-workspace", { emoji: "x", message: "late" }, true); + expect( + await service.setSidebarStatus("removed-workspace", { emoji: "x", message: "late" }) + ).toBeNull(); + + expect((await service.getAllSnapshots()).has("removed-workspace")).toBe(false); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(persisted.workspaces["removed-workspace"]).toBeUndefined(); + + // Unrelated workspaces keep writing normally. + await service.updateRecency("other-workspace", 300); + expect((await service.getAllSnapshots()).get("other-workspace")?.recency).toBe(300); + }); + + test("late writers cannot resurrect entries reclaimed by pruneMissingWorkspaces", async () => { + await service.updateRecency("stale-workspace", 100); + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set())); + + await service.updateRecency("stale-workspace", 200); + expect((await service.getAllSnapshots()).has("stale-workspace")).toBe(false); + }); + test("pruneMissingWorkspaces does not rewrite the file when nothing is stale", async () => { // Compact (non-pretty) serialization: any rewrite through save() would // change the raw bytes, so byte-equality proves no write happened. diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 13f531a72a..0341b4f02d 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -48,6 +48,20 @@ export interface ExtensionMetadataStreamingUpdate { export class ExtensionMetadataService { private readonly filePath: string; private mutationQueue: Promise = Promise.resolve(); + /** + * Per-process write tombstones for removed workspaces. Workspace removal + * cannot drain every in-flight metadata producer (e.g. a stream-abort's + * fire-and-forget stop-status handler that is still reading todos when the + * entry is deleted), so late writers would silently recreate entries for + * removed workspaces and re-leak stale keys until the next process-start + * prune. Ids are added synchronously in deleteWorkspace/prune, so combined + * with the FIFO mutation queue there is no gap: a writer enqueued before + * the delete lands first and its entry is deleted; one enqueued after + * no-ops on this set. Workspace ids are never reused (generateStableId), + * so a tombstone never blocks a legitimate new workspace; recreations of + * legacy fixed-id workspaces happen in a different (downgraded) process. + */ + private readonly deletedWorkspaceIds = new Set(); /** * Serialize all mutating operations on the shared metadata file. @@ -101,6 +115,17 @@ export class ExtensionMetadataService { recency: number, mutate: (workspace: ExtensionMetadata) => void ): Promise { + // Late write for a removed workspace: compute the snapshot for the caller + // but never persist it, so the deleted entry cannot be resurrected. + if (this.deletedWorkspaceIds.has(workspaceId)) { + const transient = this.getOrCreateWorkspaceEntry( + { version: 1, workspaces: {} }, + workspaceId, + recency + ); + mutate(transient); + return toWorkspaceActivitySnapshot(transient); + } return this.withSerializedMutation(async () => { const data = await this.load(); const workspace = this.getOrCreateWorkspaceEntry(data, workspaceId, recency); @@ -268,6 +293,10 @@ export class ExtensionMetadataService { status: ExtensionAgentStatus | null, options: { skipIfRecencyAdvancedSince?: number | null; inputHash?: string | null } = {} ): Promise { + // See deletedWorkspaceIds: never resurrect a removed workspace's entry. + if (this.deletedWorkspaceIds.has(workspaceId)) { + return null; + } return this.withSerializedMutation(async () => { const data = await this.load(); const existing = coerceExtensionMetadata(data.workspaces[workspaceId]); @@ -375,6 +404,9 @@ export class ExtensionMetadataService { * Call this when a workspace is deleted. */ async deleteWorkspace(workspaceId: string): Promise { + // Synchronously, before the queued mutation: any writer enqueued from now + // on must see the tombstone (see deletedWorkspaceIds). + this.deletedWorkspaceIds.add(workspaceId); await this.withSerializedMutation(async () => { const data = await this.load(); @@ -422,6 +454,9 @@ export class ExtensionMetadataService { for (const workspaceId of Object.keys(data.workspaces)) { if (!knownWorkspaceIds.has(workspaceId)) { delete data.workspaces[workspaceId]; + // Same guard as deleteWorkspace: a late writer must not resurrect + // an entry this pass just reclaimed. + this.deletedWorkspaceIds.add(workspaceId); prunedCount++; } } From 5c31340dae40c0fcfc04d448229f3aab4e563512 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 09:37:29 +0000 Subject: [PATCH 07/72] review: apply prune deletions to a fresh snapshot instead of rewriting the working copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-process writers (XUM_ALLOW_MULTIPLE_INSTANCES) are not covered by the in-process mutation queue: a second backend's first metadata write landing between the prune's snapshot load and its save was clobbered by the whole- file rewrite. The prune now computes stale ids from the first load, then re-loads a fresh snapshot and applies only those deletions before saving — foreign entries written in between survive, and ids are never reused so a stale id cannot have become live. Residual window equals the pre-existing per-writer stringify+atomic-write gap. --- .../services/ExtensionMetadataService.test.ts | 43 +++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 37 +++++++++++----- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index a7bb01b2e6..a2162f2680 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -479,6 +479,49 @@ describe("ExtensionMetadataService", () => { expect((await service.getAllSnapshots()).get("other-workspace")?.recency).toBe(300); }); + test("pruneMissingWorkspaces preserves entries written by another process mid-prune", async () => { + // XUM_ALLOW_MULTIPLE_INSTANCES: a second backend can register a fresh + // workspace and complete its first metadata write between this pass's + // snapshot load and its pruned rewrite. The deletion-only merge against a + // fresh reload must preserve that entry instead of clobbering the file + // with the pre-write snapshot. + await service.updateRecency("known-workspace", 100); + await service.updateRecency("stale-workspace", 200); + + const internals = service as unknown as ExtensionMetadataServiceInternals; + const originalLoad = internals.load.bind(service); + let foreignWriteInjected = false; + internals.load = async () => { + const data = await originalLoad(); + if (!foreignWriteInjected) { + foreignWriteInjected = true; + // Simulate the foreign process's write landing after our snapshot. + const foreign = await originalLoad(); + foreign.workspaces["foreign-new"] = { + recency: 300, + streaming: false, + lastModel: null, + lastThinkingLevel: null, + agentStatus: null, + }; + await writeFile(filePath, JSON.stringify(foreign)); + } + return data; + }; + try { + expect( + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set(["known-workspace"]))) + ).toBe(1); + } finally { + internals.load = originalLoad; + } + + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(persisted.workspaces["foreign-new"]).toBeDefined(); + expect(persisted.workspaces["known-workspace"]).toBeDefined(); + expect(persisted.workspaces["stale-workspace"]).toBeUndefined(); + }); + test("late writers cannot resurrect entries reclaimed by pruneMissingWorkspaces", async () => { await service.updateRecency("stale-workspace", 100); await service.pruneMissingWorkspaces(() => Promise.resolve(new Set())); diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 0341b4f02d..0e4381dae9 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -436,9 +436,16 @@ export class ExtensionMetadataService { * window where another process registers + writes a fresh entry that the * stale known-ids set misclassifies as prunable. * - * (A concurrent foreign-process write landing between our load and save can - * still be lost to this whole-file rewrite — that lost-update window is - * inherent to every existing writer of this file and unchanged here.) + * Cross-process writers (XUM_ALLOW_MULTIPLE_INSTANCES) are not serialized + * by the in-process queue, so this pass never rewrites its own working + * snapshot: it computes the stale-id set from the first load, then re-loads + * a FRESH snapshot and applies only those deletions before saving. A fresh + * entry another backend wrote between the two loads is preserved (its id + * was not in the first snapshot, so it is never classified stale), and + * since workspace ids are never reused, a stale id cannot have become live + * in between. The residual window (a foreign write landing during the + * final stringify+atomic-write) is the same lost-update window every + * existing writer of this file already has. * * Upgrade↔downgrade safety: surviving entries are round-tripped verbatim * (no coercion), so fields written by other builds are preserved and the @@ -450,18 +457,28 @@ export class ExtensionMetadataService { return this.withSerializedMutation(async () => { const data = await this.load(); const knownWorkspaceIds = await getKnownWorkspaceIds(); + const staleWorkspaceIds = Object.keys(data.workspaces).filter( + (workspaceId) => !knownWorkspaceIds.has(workspaceId) + ); + for (const workspaceId of staleWorkspaceIds) { + // Same guard as deleteWorkspace: a late in-process writer must not + // resurrect an entry this pass reclaims. + this.deletedWorkspaceIds.add(workspaceId); + } + if (staleWorkspaceIds.length === 0) { + return 0; + } + // Deletion-only merge against a fresh snapshot (see doc comment above). + const fresh = await this.load(); let prunedCount = 0; - for (const workspaceId of Object.keys(data.workspaces)) { - if (!knownWorkspaceIds.has(workspaceId)) { - delete data.workspaces[workspaceId]; - // Same guard as deleteWorkspace: a late writer must not resurrect - // an entry this pass just reclaimed. - this.deletedWorkspaceIds.add(workspaceId); + for (const workspaceId of staleWorkspaceIds) { + if (workspaceId in fresh.workspaces) { + delete fresh.workspaces[workspaceId]; prunedCount++; } } if (prunedCount > 0) { - await this.save(data); + await this.save(fresh); } return prunedCount; }); From 248cd0e879de1f3782a95aa91ffe48e2d8c150d8 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 09:49:39 +0000 Subject: [PATCH 08/72] tests: clear activity snapshot without tombstoning in goal preview fallback test The test used extensionMetadata.deleteWorkspace merely to simulate a live workspace with no snapshot; deleteWorkspace now write-tombstones removed workspaces, correctly blocking the preview persistence the test asserts. Rewrite the file directly instead. --- src/node/services/workspaceGoalService.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/node/services/workspaceGoalService.test.ts b/src/node/services/workspaceGoalService.test.ts index b6a2ed2b25..38c78384ec 100644 --- a/src/node/services/workspaceGoalService.test.ts +++ b/src/node/services/workspaceGoalService.test.ts @@ -5468,7 +5468,14 @@ describe("WorkspaceGoalService", () => { objective: "Preview without metadata", budgetCents: 1_000, }); - await extensionMetadata.deleteWorkspace(workspaceId); + // Clear the snapshot by rewriting the file directly: deleteWorkspace now + // write-tombstones removed workspaces for the rest of the process, which + // would (correctly) block the preview persistence below. This test + // simulates a LIVE workspace that merely has no activity snapshot yet. + await fs.writeFile( + path.join(config.rootDir, "extensionMetadata.json"), + JSON.stringify({ version: 1, workspaces: {} }) + ); const activityUpdates = captureGoalActivity(service); const preview = await service.previewStreamAccounting({ From ce6aa06eaeb30fb6b8964bfa4fe14aea6c52368c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 10:02:55 +0000 Subject: [PATCH 09/72] review: recheck tombstones inside queued mutations; only tombstone after successful deregistration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A writer that passed its pre-queue tombstone check while the prune was mid known-ids fetch enqueues behind the prune and previously recreated the reclaimed entry; entry-creating mutations now re-check tombstones inside the serialized queue. - rollbackFailedTaskCreate no longer discards (and write-tombstones) the task's metadata when config.removeWorkspace failed — a still-registered workspace must keep its status/goal/recency writes. --- .../services/ExtensionMetadataService.test.ts | 24 ++++++++ src/node/services/ExtensionMetadataService.ts | 40 ++++++++++--- src/node/services/taskService.test.ts | 57 +++++++++++++++++++ src/node/services/taskService.ts | 12 +++- 4 files changed, 121 insertions(+), 12 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index a2162f2680..7058cf9ebf 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -530,6 +530,30 @@ describe("ExtensionMetadataService", () => { expect((await service.getAllSnapshots()).has("stale-workspace")).toBe(false); }); + test("a writer enqueued while the prune is mid-fetch cannot resurrect a reclaimed entry", async () => { + // The prune publishes tombstones only while its queued mutation runs. A + // writer that passes its pre-queue tombstone check during the prune's + // known-ids fetch enqueues BEHIND the prune, so only the in-queue + // re-check stops it from recreating the entry the prune just reclaimed. + await service.updateRecency("stale-workspace", 100); + let releaseKnownIds!: () => void; + const gate = new Promise((resolve) => { + releaseKnownIds = resolve; + }); + const prune = service.pruneMissingWorkspaces(async () => { + await gate; + return new Set(); + }); + const lateWriter = service.updateRecency("stale-workspace", 200); + releaseKnownIds(); + expect(await prune).toBe(1); + await lateWriter; + + expect((await service.getAllSnapshots()).has("stale-workspace")).toBe(false); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(persisted.workspaces["stale-workspace"]).toBeUndefined(); + }); + test("pruneMissingWorkspaces does not rewrite the file when nothing is stale", async () => { // Compact (non-pretty) serialization: any rewrite through save() would // change the raw bytes, so byte-equality proves no write happened. diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 0e4381dae9..bc97fff19d 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -110,23 +110,40 @@ export class ExtensionMetadataService { return normalized ? toWorkspaceActivitySnapshot(normalized) : null; } + /** + * Late write for a removed workspace: compute the snapshot for the caller + * but never persist it, so the deleted entry cannot be resurrected. + */ + private buildTransientSnapshot( + workspaceId: string, + recency: number, + mutate: (workspace: ExtensionMetadata) => void + ): WorkspaceActivitySnapshot { + const transient = this.getOrCreateWorkspaceEntry( + { version: 1, workspaces: {} }, + workspaceId, + recency + ); + mutate(transient); + return toWorkspaceActivitySnapshot(transient); + } + private async mutateWorkspaceSnapshot( workspaceId: string, recency: number, mutate: (workspace: ExtensionMetadata) => void ): Promise { - // Late write for a removed workspace: compute the snapshot for the caller - // but never persist it, so the deleted entry cannot be resurrected. if (this.deletedWorkspaceIds.has(workspaceId)) { - const transient = this.getOrCreateWorkspaceEntry( - { version: 1, workspaces: {} }, - workspaceId, - recency - ); - mutate(transient); - return toWorkspaceActivitySnapshot(transient); + return this.buildTransientSnapshot(workspaceId, recency, mutate); } return this.withSerializedMutation(async () => { + // Re-check inside the queue: pruneMissingWorkspaces publishes its + // tombstones only while its queued mutation runs, so a writer that + // passed the pre-queue check and enqueued behind the prune must not + // recreate an entry the prune just reclaimed. + if (this.deletedWorkspaceIds.has(workspaceId)) { + return this.buildTransientSnapshot(workspaceId, recency, mutate); + } const data = await this.load(); const workspace = this.getOrCreateWorkspaceEntry(data, workspaceId, recency); mutate(workspace); @@ -298,6 +315,11 @@ export class ExtensionMetadataService { return null; } return this.withSerializedMutation(async () => { + // Re-check inside the queue (see mutateWorkspaceSnapshot): tombstones + // published by an already-enqueued prune must be honored here too. + if (this.deletedWorkspaceIds.has(workspaceId)) { + return null; + } const data = await this.load(); const existing = coerceExtensionMetadata(data.workspaces[workspaceId]); const workspace: ExtensionMetadata = existing ?? { diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index df0b1f3c7b..df29709bd7 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -23834,6 +23834,63 @@ describe("TaskService", () => { expect(workspacePathExists).toBe(false); }, 20_000); + test("failed config deregistration during rollback does not tombstone the task's metadata", async () => { + const config = await createTestConfig(rootDir); + stubStableIds(config, ["bbbbbbbbbb"], "bbbbbbbbbb"); + + const projectPath = await createTestProject(rootDir); + + const runtimeConfig = { type: "worktree" as const, srcBaseDir: config.srcDir }; + const runtime = createRuntime(runtimeConfig, { projectPath }); + const initLogger = createNullInitLogger(); + + const parentName = "parent-b"; + const parentCreate = await runtime.createWorkspace({ + projectPath, + branchName: parentName, + trunkBranch: "main", + directoryName: parentName, + initLogger, + }); + expect(parentCreate.success).toBe(true); + + const parentId = "2222222222"; + const parentPath = runtime.getWorkspacePath(projectPath, parentName); + + await saveWorkspaces( + config, + projectPath, + [ + { + path: parentPath, + id: parentId, + name: parentName, + createdAt: new Date().toISOString(), + runtimeConfig, + }, + ], + testTaskSettings() + ); + const { aiService } = createAIServiceMocks(config); + const failingSendMessage = mock(() => Promise.resolve(Err("send failed"))); + const { workspaceService, discardExtensionMetadataEntry } = createWorkspaceServiceMocks({ + sendMessage: failingSendMessage, + }); + const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + // Deregistration fails: the rollback must NOT discard (and thereby + // write-tombstone) metadata for a workspace that is still registered. + const removeSpy = spyOn(config, "removeWorkspace").mockImplementation(() => + Promise.reject(new Error("config locked")) + ); + try { + const created = await createAgentTask(taskService, parentId, "do the thing"); + expect(created.success).toBe(false); + expect(discardExtensionMetadataEntry).not.toHaveBeenCalled(); + } finally { + removeSpy.mockRestore(); + } + }, 20_000); + test("agent_report posts report to parent, finalizes pending task tool output, and triggers cleanup", async () => { const config = await createTestConfig(rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index d3332a0bd8..2a16a538f4 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7310,8 +7310,10 @@ export class TaskService { preservePhysicalWorkspace?: boolean; } ): Promise { + let removedFromConfig = false; try { await this.config.removeWorkspace(taskId); + removedFromConfig = true; } catch (error: unknown) { log.error("Task.create rollback: failed to remove workspace from config", { taskId, @@ -7322,9 +7324,13 @@ export class TaskService { // A create that failed after sendMessage may already have scheduled // extension-metadata writes (e.g. the recency update), which would // recreate the entry after the deregistration above and leak a stale key - // until the next process start's lazy prune. Best-effort, like the - // config removal above. - await this.workspaceService.discardExtensionMetadataEntry(taskId); + // until the next process start's lazy prune. Only after deregistration + // actually succeeded: discarding also write-tombstones the id for this + // process, which must not silence metadata for a workspace that is still + // registered because removeWorkspace failed. + if (removedFromConfig) { + await this.workspaceService.discardExtensionMetadataEntry(taskId); + } this.workspaceService.emit("metadata", { workspaceId: taskId, metadata: null }); From 3371c619e7504a21879456aa1248eb255d12ae6b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 10:21:23 +0000 Subject: [PATCH 10/72] review: suppress post-removal activity emissions, authoritative empty list, single bootstrap enumeration, presence-keyed delete - emitWorkspaceActivity drops non-null broadcasts for write-tombstoned workspaces so late producers cannot re-insert removed ids into the renderer's activity map (disk writes were already blocked). - getActivityList rethrows on failure instead of returning {}; the renderer's bootstrap loop already retries with backoff, and a successful empty object is now applied as authoritative, clearing snapshots a disconnected renderer holds for removed workspaces. - First bootstrap reuses the prune's config enumeration for scoping (getAllWorkspaceMetadata's per-workspace disk walk is paid once). - deleteWorkspace tests key presence, not truthiness, so malformed falsy entries are removed too. --- src/browser/stores/WorkspaceStore.ts | 12 +- .../services/ExtensionMetadataService.test.ts | 17 +++ src/node/services/ExtensionMetadataService.ts | 14 ++- src/node/services/workspaceService.test.ts | 109 ++++++++++++++++++ src/node/services/workspaceService.ts | 84 +++++++++----- 5 files changed, 202 insertions(+), 34 deletions(-) diff --git a/src/browser/stores/WorkspaceStore.ts b/src/browser/stores/WorkspaceStore.ts index fbe32b4351..0d32d409d2 100644 --- a/src/browser/stores/WorkspaceStore.ts +++ b/src/browser/stores/WorkspaceStore.ts @@ -3295,15 +3295,13 @@ export class WorkspaceStore { } private applyWorkspaceActivityList(snapshots: Record): void { + // An empty object is authoritative: the backend now rethrows on read + // failures (this path only runs on success), and the scoped list validly + // returns {} when no config-known workspace has activity. Applying it + // clears snapshots a disconnected renderer still holds for removed + // workspaces. const snapshotEntries = Object.entries(snapshots); - // Defensive fallback: workspace.activity.list returns {} on backend read failures. - // Preserve last-known snapshots instead of wiping sidebar activity state for all - // workspaces during a transient metadata read error. - if (snapshotEntries.length === 0) { - return; - } - const seenWorkspaceIds = new Set(); for (const [workspaceId, snapshot] of snapshotEntries) { diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 7058cf9ebf..82694c9575 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -452,6 +452,23 @@ describe("ExtensionMetadataService", () => { expect(order).toEqual(["load", "fetch-known-ids"]); }); + test("deleteWorkspace removes malformed falsy persisted entries", async () => { + // Key presence, not truthiness: a null entry must not survive removal + // (it would leak the key until the next process-start prune). + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "malformed-workspace": null, "other-workspace": { recency: 1 } }, + }) + ); + await service.deleteWorkspace("malformed-workspace"); + + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect("malformed-workspace" in persisted.workspaces).toBe(false); + expect("other-workspace" in persisted.workspaces).toBe(true); + }); + test("late writers cannot resurrect a deleted workspace entry", async () => { // Removal cannot drain every in-flight metadata producer (e.g. a // stream-abort's fire-and-forget stop-status write), so writes landing diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index bc97fff19d..4961aa56d0 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -421,6 +421,15 @@ export class ExtensionMetadataService { return this.toSnapshot(data.workspaces[workspaceId]); } + /** + * Whether this process removed the workspace's entry (see + * deletedWorkspaceIds). Lets emit paths suppress late activity broadcasts + * whose disk writes the tombstone already blocked. + */ + isWorkspaceDeleted(workspaceId: string): boolean { + return this.deletedWorkspaceIds.has(workspaceId); + } + /** * Delete metadata for a workspace. * Call this when a workspace is deleted. @@ -432,7 +441,10 @@ export class ExtensionMetadataService { await this.withSerializedMutation(async () => { const data = await this.load(); - if (data.workspaces[workspaceId]) { + // Key presence, not truthiness: malformed falsy persisted entries + // (e.g. null) must be deleted too, or removal leaves a stale key + // behind until the next process-start prune. + if (workspaceId in data.workspaces) { delete data.workspaces[workspaceId]; await this.save(data); } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7861a3084c..4cee2af290 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3208,6 +3208,115 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { + // getAllWorkspaceMetadata walks every workspace with per-workspace disk + // probes; the latency-sensitive first bootstrap must not pay it twice. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "activity-scoping-reuse"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + await extensionMetadata.updateRecency("removed-workspace", 200); + const metadataSpy = spyOn(config, "getAllWorkspaceMetadata"); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList[workspaceId]?.recency).toBe(100); + expect(activityList["removed-workspace"]).toBeUndefined(); + expect(metadataSpy).toHaveBeenCalledTimes(1); + } finally { + await cleanup(); + } + }); + + test("suppresses activity emissions for removed workspaces", async () => { + // A late in-flight producer completing after removal must not broadcast: + // the renderer would re-insert the removed id into its activity map after + // already processing the metadata-removal event. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "activity-emit-after-removal"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const events: Array<{ workspaceId: string }> = []; + workspaceService.on("activity", (event) => events.push(event)); + + await workspaceService.updateAgentStatus(workspaceId, { emoji: "🛠️", message: "Working" }); + expect(events.length).toBe(1); + + await workspaceService.discardExtensionMetadataEntry(workspaceId); + // Simulates the producer that was already in flight when removal ran. + await workspaceService.updateAgentStatus(workspaceId, { emoji: "🛠️", message: "Late" }); + expect(events.length).toBe(1); + // Clearing (null) emissions stay allowed for removed workspaces. + workspaceService.emitWorkspaceActivity(workspaceId, null); + expect(events.length).toBe(2); + } finally { + await cleanup(); + } + }); + + test("getActivityList rejects on metadata read failure instead of returning {}", async () => { + // With scoping, {} is a valid authoritative answer that clears renderer + // state; failures must be distinguishable so the renderer keeps its + // last-known snapshots and retries. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation(() => + Promise.reject(new Error("metadata unreadable")) + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + let rejected = false; + try { + await workspaceService.getActivityList(); + } catch { + rejected = true; + } + expect(rejected).toBe(true); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("falls back to the unscoped union when config workspaces cannot be listed", async () => { // Real on-disk corruption shapes. loadConfigOrDefault SWALLOWS the first // (parse failure) and lenient-normalizes the rest (parseable but diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index ecf368cdf2..d1d334d95f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3903,6 +3903,15 @@ export class WorkspaceService extends EventEmitter { workspaceId: string, snapshot: WorkspaceActivitySnapshot | null ): void { + // A late in-flight producer (e.g. a stream-abort stop-status handler mid + // todo read) can complete after removal deleted this workspace's + // metadata entry. Its disk write is already blocked by the write + // tombstone; suppress the broadcast too, or the renderer re-inserts the + // removed id into its activity map after having processed the + // metadata-removal event. Null (clearing) emissions stay allowed. + if (snapshot !== null && this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { + return; + } this.emit("activity", { workspaceId, activity: this.mergeCurrentActiveBashMonitorCount( @@ -12789,15 +12798,23 @@ export class WorkspaceService extends EventEmitter { * from the activity bootstrap path; never a per-read scan. */ private prunedStaleExtensionMetadata = false; - private async pruneStaleExtensionMetadataOnce(): Promise { + /** + * Returns the config-known (normalized-view) workspace ids captured during + * the prune so the first activity bootstrap can reuse them for scoping — + * getAllWorkspaceMetadata walks every workspace with per-workspace disk + * probes, which large deployments should not pay twice in the + * latency-sensitive bootstrap. Null when the prune was skipped or failed. + */ + private async pruneStaleExtensionMetadataOnce(): Promise | null> { if (this.prunedStaleExtensionMetadata) { - return; + return null; } // Latch before awaiting so concurrent bootstraps don't queue redundant // prunes; a failed attempt is retried on the next process start rather // than on every read. this.prunedStaleExtensionMetadata = true; try { + let normalizedIds: Set | null = null; const prunedCount = await this.extensionMetadata.pruneMissingWorkspaces(async () => { // Invoked inside the file's serialized mutation AFTER the file load, // reading config fresh from disk (see pruneMissingWorkspaces), so a @@ -12815,22 +12832,27 @@ export class WorkspaceService extends EventEmitter { // A missing config file resolves as a healthy empty set in both. const knownIds = this.config.readPersistedWorkspaceIdSuperset(); const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); - for (const metadata of allMetadata) { - knownIds.add(metadata.id); + normalizedIds = new Set(allMetadata.map((metadata) => metadata.id)); + for (const workspaceId of normalizedIds) { + knownIds.add(workspaceId); } return knownIds; }); if (prunedCount > 0) { log.info(`Pruned ${prunedCount} stale extension metadata entries`); } + return normalizedIds; } catch (error) { log.debug("Failed to prune stale extension metadata entries", { error }); + return null; } } async getActivityList(): Promise> { try { - await this.pruneStaleExtensionMetadataOnce(); + // On the first bootstrap the prune already enumerated the config; reuse + // that id set instead of paying the per-workspace disk walk twice. + const prefetchedKnownIds = await this.pruneStaleExtensionMetadataOnce(); const snapshots = await this.extensionMetadata.getAllSnapshots(); // Scope the list to config-known workspaces. extensionMetadata.json was // historically never pruned, so long-lived deployments accumulate stale @@ -12841,26 +12863,30 @@ export class WorkspaceService extends EventEmitter { // WITHOUT a snapshot still flow through the tombstone logic below — // scoping only drops ids that are not in config at all. let workspaceIds: Set; - try { - // throwOnError so a corrupted config.json actually reaches the - // fail-open fallback below instead of silently resolving as the - // empty default and dropping every live entry from the list. - workspaceIds = new Set( - (await this.config.getAllWorkspaceMetadata({ throwOnError: true })).map( - (metadata) => metadata.id - ) - ); - } catch (error) { - // Fail open: without the config view, stale ids cannot be told apart - // from live ones, and dropping live entries would strand renderer - // activity state. Fall back to the legacy unscoped union. - log.debug("Failed to scope activity list to known workspaces", { error }); - workspaceIds = new Set(snapshots.keys()); - for (const workspaceId of this.activeWorkflowRunIdsByWorkspace.keys()) { - workspaceIds.add(workspaceId); - } - for (const workspaceId of this.bashMonitorSeenWorkspaces) { - workspaceIds.add(workspaceId); + if (prefetchedKnownIds != null) { + workspaceIds = prefetchedKnownIds; + } else { + try { + // throwOnError so a corrupted config.json actually reaches the + // fail-open fallback below instead of silently resolving as the + // empty default and dropping every live entry from the list. + workspaceIds = new Set( + (await this.config.getAllWorkspaceMetadata({ throwOnError: true })).map( + (metadata) => metadata.id + ) + ); + } catch (error) { + // Fail open: without the config view, stale ids cannot be told apart + // from live ones, and dropping live entries would strand renderer + // activity state. Fall back to the legacy unscoped union. + log.debug("Failed to scope activity list to known workspaces", { error }); + workspaceIds = new Set(snapshots.keys()); + for (const workspaceId of this.activeWorkflowRunIdsByWorkspace.keys()) { + workspaceIds.add(workspaceId); + } + for (const workspaceId of this.bashMonitorSeenWorkspaces) { + workspaceIds.add(workspaceId); + } } } @@ -12921,8 +12947,14 @@ export class WorkspaceService extends EventEmitter { ) ); } catch (error) { + // Rethrow instead of returning {}: with scoping, an empty result is a + // VALID authoritative answer (no known workspace has activity) that the + // renderer must apply to clear stale entries after a disconnected + // removal. Failures must therefore be distinguishable — the renderer's + // bootstrap loop treats a rejection as transient (keeps last-known + // state and retries with backoff). log.error("Failed to list activity:", error); - return {}; + throw error; } } async getChatHistory(workspaceId: string): Promise { From 608f6637e8493ff4461ec9012dfb18d98f9fd699 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 10:44:47 +0000 Subject: [PATCH 11/72] review: verify deregistration before metadata tombstone; strict metadata read for activity list Round-9 Codex P2s: - discardExtensionMetadataEntry now verifies the id is absent from a fresh config disk read (raw persisted superset + strict normalized view) before deleting: saveConfig swallows write failures, so removeWorkspace can resolve while the workspace is still persisted, and tombstoning it would suppress all future activity writes for the process. - getActivityList reads snapshots with throwOnError so an unreadable or malformed extensionMetadata.json rejects (renderer keeps last-known state and retries) instead of self-healing into an authoritative empty list that wipes every cached snapshot. Writer paths keep the lenient self-heal. --- src/node/services/ExtensionMetadataService.ts | 18 +++-- src/node/services/workspaceService.test.ts | 73 +++++++++++++++++++ src/node/services/workspaceService.ts | 70 +++++++++++++----- 3 files changed, 139 insertions(+), 22 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 4961aa56d0..f242709750 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -186,10 +186,11 @@ export class ExtensionMetadataService { } } - private async load(): Promise { + private async load(options?: { throwOnError?: boolean }): Promise { try { await access(this.filePath, constants.F_OK); } catch { + // A missing file is a healthy empty state in both modes. return { version: 1, workspaces: {} }; } @@ -199,12 +200,17 @@ export class ExtensionMetadataService { // Validate structure if (typeof parsed !== "object" || parsed.version !== 1) { - log.error("Invalid metadata file, resetting"); - return { version: 1, workspaces: {} }; + throw new Error("Invalid extension metadata file structure"); } return parsed; } catch (error) { + // throwOnError lets read paths distinguish "file exists but is + // unreadable/malformed" from an authoritative empty state; the default + // self-heals so writers can always make progress. + if (options?.throwOnError) { + throw error; + } log.error("Failed to load metadata:", error); return { version: 1, workspaces: {} }; } @@ -544,8 +550,10 @@ export class ExtensionMetadataService { }); } - async getAllSnapshots(): Promise> { - const data = await this.load(); + async getAllSnapshots(options?: { + throwOnError?: boolean; + }): Promise> { + const data = await this.load(options); const map = new Map(); for (const [workspaceId, entry] of Object.entries(data.workspaces)) { const snapshot = this.toSnapshot(entry); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4cee2af290..2e9e5f801d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3272,6 +3272,9 @@ describe("WorkspaceService activity list scoping", () => { await workspaceService.updateAgentStatus(workspaceId, { emoji: "🛠️", message: "Working" }); expect(events.length).toBe(1); + // Discard verifies deregistration against persisted config, so remove + // the workspace first (mirroring the real removal flow). + await config.removeWorkspace(workspaceId); await workspaceService.discardExtensionMetadataEntry(workspaceId); // Simulates the producer that was already in flight when removal ran. await workspaceService.updateAgentStatus(workspaceId, { emoji: "🛠️", message: "Late" }); @@ -3284,6 +3287,73 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("discardExtensionMetadataEntry keeps the entry when the workspace is still persisted", async () => { + // saveConfig swallows write failures, so config.removeWorkspace can + // resolve while the workspace is still persisted in config.json. + // Discarding then would write-tombstone a live id and suppress all of + // its future activity writes for the rest of the process. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "discard-still-persisted"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + await workspaceService.discardExtensionMetadataEntry(workspaceId); + + expect((await extensionMetadata.getAllSnapshots()).has(workspaceId)).toBe(true); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + } finally { + await cleanup(); + } + }); + + test("getActivityList rejects when the metadata file is unreadable", async () => { + // ExtensionMetadataService.load self-heals a corrupt file into an empty + // one by default; surfacing that here as an authoritative empty list + // would make the renderer wipe every cached snapshot with no retry. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + await fsPromises.writeFile(metadataPath, "{not json", "utf-8"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + // Lenient reads (writer paths) still self-heal. + expect((await extensionMetadata.getAllSnapshots()).size).toBe(0); + + let rejected = false; + try { + await workspaceService.getActivityList(); + } catch { + rejected = true; + } + expect(rejected).toBe(true); + // The self-healing prune path must not have rewritten (reset) the file. + expect(await fsPromises.readFile(metadataPath, "utf-8")).toBe("{not json"); + } finally { + await cleanup(); + } + }); + test("getActivityList rejects on metadata read failure instead of returning {}", async () => { // With scoping, {} is a valid authoritative answer that clears renderer // state; failures must be distinguishable so the renderer keeps its @@ -3404,6 +3474,9 @@ describe("WorkspaceService activity list scoping", () => { removeWorkspace: mock(() => Promise.resolve()), findWorkspace: mock(() => null), loadConfigOrDefault: mock(() => ({ projects: new Map() })), + // The discard verifies deregistration against these before deleting. + readPersistedWorkspaceIdSuperset: mock(() => new Set()), + getAllWorkspaceMetadata: mock(() => Promise.resolve([])), }; const workspaceService = createWorkspaceServiceForTest({ config: mockConfig, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index d1d334d95f..b36b8de175 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12769,6 +12769,31 @@ export class WorkspaceService extends EventEmitter { } } + /** + * Union of the raw persisted config id superset and the strict normalized + * metadata view: + * - the raw persisted superset covers entries loadConfigOrDefault's + * validation/normalization would filter or discard (see + * readPersistedWorkspaceIdSuperset), so a live workspace with a + * malformed config entry is never treated as removed; + * - the strict normalized view covers ids produced by in-memory config + * migrations that are not yet persisted verbatim. + * Both views throw rather than resolving with a silently lossy id set; a + * missing config file resolves as a healthy empty set in both. + */ + private async readKnownWorkspaceIds(): Promise<{ + knownIds: Set; + normalizedIds: Set; + }> { + const knownIds = this.config.readPersistedWorkspaceIdSuperset(); + const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); + const normalizedIds = new Set(allMetadata.map((metadata) => metadata.id)); + for (const workspaceId of normalizedIds) { + knownIds.add(workspaceId); + } + return { knownIds, normalizedIds }; + } + /** * Best-effort removal of a deregistered workspace's activity/status entry * from extensionMetadata.json. Used by remove() and by rollback paths that @@ -12780,6 +12805,21 @@ export class WorkspaceService extends EventEmitter { */ async discardExtensionMetadataEntry(workspaceId: string): Promise { try { + // Deleting also write-tombstones the id for the rest of this process, + // so verify deregistration actually landed before publishing it: + // saveConfig swallows write failures, meaning config.removeWorkspace + // can resolve while the workspace is still persisted in config.json — + // tombstoning a still-live id would suppress all of its future + // activity writes. A failed verification (unreadable config) skips the + // delete too; like a missed delete, the entry is reclaimed by a later + // process start's prune, which re-checks against config. + const { knownIds } = await this.readKnownWorkspaceIds(); + if (knownIds.has(workspaceId)) { + log.debug("Skipping extension metadata discard: workspace still persisted in config", { + workspaceId, + }); + return; + } await this.extensionMetadata.deleteWorkspace(workspaceId); } catch (error) { log.debug("Failed to prune extension metadata after workspace deregistration", { @@ -12821,22 +12861,12 @@ export class WorkspaceService extends EventEmitter { // concurrently created workspace — even in another backend process — // cannot lose its just-written entry. // - // Union of two views, both of which throw (aborting the prune, caught - // below) rather than resolving with a silently lossy id set: - // - the raw persisted superset covers entries loadConfigOrDefault's - // validation/normalization would filter or discard (see - // readPersistedWorkspaceIdSuperset), so a live workspace with a - // malformed config entry is never treated as removed; - // - the strict normalized view covers ids produced by in-memory - // config migrations that are not yet persisted verbatim. - // A missing config file resolves as a healthy empty set in both. - const knownIds = this.config.readPersistedWorkspaceIdSuperset(); - const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); - normalizedIds = new Set(allMetadata.map((metadata) => metadata.id)); - for (const workspaceId of normalizedIds) { - knownIds.add(workspaceId); - } - return knownIds; + // Union of two views (see readKnownWorkspaceIds), both of which throw + // (aborting the prune, caught below) rather than resolving with a + // silently lossy id set. + const views = await this.readKnownWorkspaceIds(); + normalizedIds = views.normalizedIds; + return views.knownIds; }); if (prunedCount > 0) { log.info(`Pruned ${prunedCount} stale extension metadata entries`); @@ -12853,7 +12883,13 @@ export class WorkspaceService extends EventEmitter { // On the first bootstrap the prune already enumerated the config; reuse // that id set instead of paying the per-workspace disk walk twice. const prefetchedKnownIds = await this.pruneStaleExtensionMetadataOnce(); - const snapshots = await this.extensionMetadata.getAllSnapshots(); + // throwOnError: the default load self-heals an unreadable/malformed + // metadata file into an empty one, which this list would then present + // as an authoritative "no activity anywhere" answer — the renderer + // applies that by wiping every cached streaming/status/goal snapshot + // with no retry (the subscription stays connected). Rejecting instead + // keeps last-known renderer state and lets the bootstrap loop retry. + const snapshots = await this.extensionMetadata.getAllSnapshots({ throwOnError: true }); // Scope the list to config-known workspaces. extensionMetadata.json was // historically never pruned, so long-lived deployments accumulate stale // entries for removed workspaces/sub-agents by the thousands (issue From 74e0b3e6c9490db854a4eca2ff4903bf0e8be388 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 11:04:13 +0000 Subject: [PATCH 12/72] review: validate workspaces container in strict metadata loads; drop O(n) walk from discard verification Round-10 Codex P2s: - Strict metadata loads now reject a parseable file whose workspaces container is an array/primitive/null instead of enumerating it as zero entries and returning an authoritative empty activity list. - discardExtensionMetadataEntry verifies deregistration via the raw persisted id superset plus a targeted findWorkspace lookup instead of getAllWorkspaceMetadata's O(n) per-workspace fs enrichment walk. The superset read throws on an unreadable config, which also makes findWorkspace's lenient internal load safe. --- src/node/services/ExtensionMetadataService.ts | 13 +++- src/node/services/workspaceService.test.ts | 60 +++++++++++-------- src/node/services/workspaceService.ts | 59 ++++++++---------- 3 files changed, 73 insertions(+), 59 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index f242709750..79866e8d5f 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -198,8 +198,17 @@ export class ExtensionMetadataService { const content = await readFile(this.filePath, "utf-8"); const parsed = JSON.parse(content) as ExtensionMetadataFile; - // Validate structure - if (typeof parsed !== "object" || parsed.version !== 1) { + // Validate structure, including the workspaces container: a parseable + // file with e.g. an array or primitive `workspaces` would otherwise + // enumerate as zero entries and masquerade as an authoritative empty + // state in strict reads. + if ( + typeof parsed !== "object" || + parsed?.version !== 1 || + typeof parsed.workspaces !== "object" || + parsed.workspaces === null || + Array.isArray(parsed.workspaces) + ) { throw new Error("Invalid extension metadata file structure"); } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 2e9e5f801d..05fbbbc981 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3322,35 +3322,46 @@ describe("WorkspaceService activity list scoping", () => { } }); - test("getActivityList rejects when the metadata file is unreadable", async () => { + test("getActivityList rejects when the metadata file is unreadable or malformed", async () => { // ExtensionMetadataService.load self-heals a corrupt file into an empty // one by default; surfacing that here as an authoritative empty list // would make the renderer wipe every cached snapshot with no retry. - const { config, historyService, cleanup } = await createTestHistoryService(); - try { - const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); - await fsPromises.writeFile(metadataPath, "{not json", "utf-8"); - const extensionMetadata = new ExtensionMetadataService(metadataPath); - const workspaceService = createWorkspaceServiceForTest({ - config, - historyService, - extensionMetadata, - }); + // Parseable-but-malformed workspaces containers enumerate as zero + // entries, so they must follow the same failure path. + const corruptFiles = [ + "{not json", + JSON.stringify({ version: 2, workspaces: {} }), + JSON.stringify({ version: 1, workspaces: [] }), + JSON.stringify({ version: 1, workspaces: "bogus" }), + JSON.stringify({ version: 1, workspaces: null }), + ]; + for (const corruptFile of corruptFiles) { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + await fsPromises.writeFile(metadataPath, corruptFile, "utf-8"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); - // Lenient reads (writer paths) still self-heal. - expect((await extensionMetadata.getAllSnapshots()).size).toBe(0); + // Lenient reads (writer paths) still self-heal. + expect((await extensionMetadata.getAllSnapshots()).size).toBe(0); - let rejected = false; - try { - await workspaceService.getActivityList(); - } catch { - rejected = true; + let rejected = false; + try { + await workspaceService.getActivityList(); + } catch { + rejected = true; + } + expect(rejected).toBe(true); + // The self-healing prune path must not have rewritten (reset) the file. + expect(await fsPromises.readFile(metadataPath, "utf-8")).toBe(corruptFile); + } finally { + await cleanup(); } - expect(rejected).toBe(true); - // The self-healing prune path must not have rewritten (reset) the file. - expect(await fsPromises.readFile(metadataPath, "utf-8")).toBe("{not json"); - } finally { - await cleanup(); } }); @@ -3474,7 +3485,8 @@ describe("WorkspaceService activity list scoping", () => { removeWorkspace: mock(() => Promise.resolve()), findWorkspace: mock(() => null), loadConfigOrDefault: mock(() => ({ projects: new Map() })), - // The discard verifies deregistration against these before deleting. + // The discard verifies deregistration against the persisted superset + // (and the findWorkspace mock above) before deleting. readPersistedWorkspaceIdSuperset: mock(() => new Set()), getAllWorkspaceMetadata: mock(() => Promise.resolve([])), }; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index b36b8de175..a537b60859 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12769,31 +12769,6 @@ export class WorkspaceService extends EventEmitter { } } - /** - * Union of the raw persisted config id superset and the strict normalized - * metadata view: - * - the raw persisted superset covers entries loadConfigOrDefault's - * validation/normalization would filter or discard (see - * readPersistedWorkspaceIdSuperset), so a live workspace with a - * malformed config entry is never treated as removed; - * - the strict normalized view covers ids produced by in-memory config - * migrations that are not yet persisted verbatim. - * Both views throw rather than resolving with a silently lossy id set; a - * missing config file resolves as a healthy empty set in both. - */ - private async readKnownWorkspaceIds(): Promise<{ - knownIds: Set; - normalizedIds: Set; - }> { - const knownIds = this.config.readPersistedWorkspaceIdSuperset(); - const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); - const normalizedIds = new Set(allMetadata.map((metadata) => metadata.id)); - for (const workspaceId of normalizedIds) { - knownIds.add(workspaceId); - } - return { knownIds, normalizedIds }; - } - /** * Best-effort removal of a deregistered workspace's activity/status entry * from extensionMetadata.json. Used by remove() and by rollback paths that @@ -12813,8 +12788,16 @@ export class WorkspaceService extends EventEmitter { // activity writes. A failed verification (unreadable config) skips the // delete too; like a missed delete, the entry is reclaimed by a later // process start's prune, which re-checks against config. - const { knownIds } = await this.readKnownWorkspaceIds(); - if (knownIds.has(workspaceId)) { + // + // Deliberately NOT getAllWorkspaceMetadata here: that walks every + // configured workspace with per-workspace fs probes — an O(n) + // traversal just to check one id. The raw persisted superset (which + // throws on an unreadable config, also making findWorkspace's lenient + // internal load safe below) covers entries normalization would drop, + // and the targeted findWorkspace lookup covers normalized/legacy ids + // (metadata.json / generated legacy ids) the raw scan cannot see. + const knownIds = this.config.readPersistedWorkspaceIdSuperset(); + if (knownIds.has(workspaceId) || this.config.findWorkspace(workspaceId) != null) { log.debug("Skipping extension metadata discard: workspace still persisted in config", { workspaceId, }); @@ -12861,12 +12844,22 @@ export class WorkspaceService extends EventEmitter { // concurrently created workspace — even in another backend process — // cannot lose its just-written entry. // - // Union of two views (see readKnownWorkspaceIds), both of which throw - // (aborting the prune, caught below) rather than resolving with a - // silently lossy id set. - const views = await this.readKnownWorkspaceIds(); - normalizedIds = views.normalizedIds; - return views.knownIds; + // Union of two views, both of which throw (aborting the prune, caught + // below) rather than resolving with a silently lossy id set: + // - the raw persisted superset covers entries loadConfigOrDefault's + // validation/normalization would filter or discard (see + // readPersistedWorkspaceIdSuperset), so a live workspace with a + // malformed config entry is never treated as removed; + // - the strict normalized view covers ids produced by in-memory + // config migrations that are not yet persisted verbatim. + // A missing config file resolves as a healthy empty set in both. + const knownIds = this.config.readPersistedWorkspaceIdSuperset(); + const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); + normalizedIds = new Set(allMetadata.map((metadata) => metadata.id)); + for (const workspaceId of normalizedIds) { + knownIds.add(workspaceId); + } + return knownIds; }); if (prunedCount > 0) { log.info(`Pruned ${prunedCount} stale extension metadata entries`); From 703ab51a9926dd77a672cea86e10e8c41a7d5515 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 11:25:38 +0000 Subject: [PATCH 13/72] review: ENOENT-only empty metadata state, ACP activity fallback, post-await tombstone revalidation Round-11 Codex findings: - P1: ACP listSessions no longer fails wholesale when activity.list rejects (unreadable extensionMetadata.json); it degrades to no-activity recency sorting since ACP has no cached state to preserve. - P2: only ENOENT maps to the healthy empty metadata state; other read failures (EACCES/ENOTDIR/EIO) now propagate in strict reads instead of masquerading as an authoritative empty activity list. - P2: getActivityList revalidates tombstones after its per-workspace awaits so a workspace removed mid-computation cannot ride the delayed response past emitWorkspaceActivity's suppression and be re-inserted by the renderer. --- src/node/acp/agent.ts | 16 +++- src/node/services/ExtensionMetadataService.ts | 20 ++--- src/node/services/workspaceService.test.ts | 73 +++++++++++++++++++ src/node/services/workspaceService.ts | 9 ++- tests/ipc/acp.sessionMethods.test.ts | 32 +++++++- 5 files changed, 137 insertions(+), 13 deletions(-) diff --git a/src/node/acp/agent.ts b/src/node/acp/agent.ts index c467b73b92..8705a23ebc 100644 --- a/src/node/acp/agent.ts +++ b/src/node/acp/agent.ts @@ -444,7 +444,21 @@ export class MuxAgent implements Agent { const [activeWorkspaces, archivedWorkspaces, workspaceActivity] = await Promise.all([ this.server.client.workspace.list({ archived: false }), this.server.client.workspace.list({ archived: true }), - this.server.client.workspace.activity.list(), + // activity.list rejects on an unreadable/corrupt extensionMetadata.json + // so the renderer can keep last-known state instead of applying a bogus + // authoritative empty list. Here activity only refines recency + // sorting/updatedAt and there is no cached state to preserve, so a + // failure must not take down session listing for otherwise healthy + // sessions — degrade to an empty activity view instead. + this.server.client.workspace.activity + .list() + .catch((error: unknown): WorkspaceActivityById => { + console.error( + "[acp] Failed to list workspace activity; listing sessions without it", + error + ); + return {}; + }), ]); const allWorkspaces = dedupeWorkspacesById([...activeWorkspaces, ...archivedWorkspaces]); diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 79866e8d5f..9458d81ce7 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -187,13 +187,6 @@ export class ExtensionMetadataService { } private async load(options?: { throwOnError?: boolean }): Promise { - try { - await access(this.filePath, constants.F_OK); - } catch { - // A missing file is a healthy empty state in both modes. - return { version: 1, workspaces: {} }; - } - try { const content = await readFile(this.filePath, "utf-8"); const parsed = JSON.parse(content) as ExtensionMetadataFile; @@ -214,9 +207,16 @@ export class ExtensionMetadataService { return parsed; } catch (error) { - // throwOnError lets read paths distinguish "file exists but is - // unreadable/malformed" from an authoritative empty state; the default - // self-heals so writers can always make progress. + // Only a genuinely missing file is a healthy empty state. Other read + // failures (EACCES/ENOTDIR/EIO, parse or structure errors) must not + // masquerade as one: throwOnError lets read paths distinguish them + // from an authoritative empty state, while the default self-heals so + // writers can always make progress. + if (typeof error === "object" && error != null && "code" in error) { + if (error.code === "ENOENT") { + return { version: 1, workspaces: {} }; + } + } if (options?.throwOnError) { throw error; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 05fbbbc981..c278db7155 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -141,6 +141,7 @@ const mockInitStateManager: Partial = { clearInMemoryState: mock(() => undefined), }; const mockExtensionMetadataService: Partial = { + isWorkspaceDeleted: mock(() => false), setStreaming: mock(() => Promise.resolve({ recency: Date.now(), @@ -3365,6 +3366,78 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList rejects when the metadata path exists but cannot be read", async () => { + // Only a genuinely missing file (ENOENT) is a healthy empty state. Any + // other read failure (here EISDIR; EACCES/ENOTDIR/EIO in the field) must + // reject in strict reads instead of masquerading as an authoritative + // empty list. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + await fsPromises.mkdir(metadataPath, { recursive: true }); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + // Lenient reads (writer paths) still self-heal. + expect((await extensionMetadata.getAllSnapshots()).size).toBe(0); + + let rejected = false; + try { + await workspaceService.getActivityList(); + } catch { + rejected = true; + } + expect(rejected).toBe(true); + } finally { + await cleanup(); + } + }); + + test("getActivityList drops workspaces removed while the list was computing", async () => { + // A removal that lands between the snapshot read and the response must + // not ride the delayed list past emitWorkspaceActivity's tombstone + // suppression: a renderer that already processed the removal event would + // re-insert the deleted id until the next reconnect. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "removed-mid-list"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + const readSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + spyOn(extensionMetadata, "getAllSnapshots").mockImplementationOnce(async () => { + const snapshots = await readSnapshots(); + // Simulates a concurrent removal completing after this request read + // its snapshot view but before the response was assembled. + await extensionMetadata.deleteWorkspace(workspaceId); + return snapshots; + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList[workspaceId]).toBeUndefined(); + } finally { + await cleanup(); + } + }); + test("getActivityList rejects on metadata read failure instead of returning {}", async () => { // With scoping, {} is a valid authoritative answer that clears renderer // state; failures must be distinguishable so the renderer keeps its diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index a537b60859..d6f3b12c58 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12972,7 +12972,14 @@ export class WorkspaceService extends EventEmitter { ); return Object.fromEntries( entries.filter( - (entry): entry is readonly [string, WorkspaceActivitySnapshot] => entry != null + (entry): entry is readonly [string, WorkspaceActivitySnapshot] => + entry != null && + // Revalidate after the per-workspace awaits above: a workspace + // removed while this list was computing would otherwise ride the + // delayed response past emitWorkspaceActivity's tombstone + // suppression — a renderer that already processed the removal + // event would re-insert the deleted id until the next reconnect. + !this.extensionMetadata.isWorkspaceDeleted(entry[0]) ) ); } catch (error) { diff --git a/tests/ipc/acp.sessionMethods.test.ts b/tests/ipc/acp.sessionMethods.test.ts index 02395c5e84..626f72cef5 100644 --- a/tests/ipc/acp.sessionMethods.test.ts +++ b/tests/ipc/acp.sessionMethods.test.ts @@ -32,6 +32,8 @@ interface HarnessOptions { activeWorkspaces?: WorkspaceInfo[]; archivedWorkspaces?: WorkspaceInfo[]; workspaceActivity?: WorkspaceActivityById; + /** Reject activity.list with this error (simulates unreadable extensionMetadata.json). */ + activityListError?: Error; onChatEvents?: WorkspaceChatMessage[]; onChatStream?: AsyncIterable; requireTrustedProjectForCreate?: boolean; @@ -133,7 +135,12 @@ function createMockServer(options?: HarnessOptions): MockServer { return input?.archived ? archivedWorkspaces : activeWorkspaces; }, activity: { - list: async () => workspaceActivity, + list: async () => { + if (options?.activityListError) { + throw options.activityListError; + } + return workspaceActivity; + }, }, getInfo: async ({ workspaceId }: { workspaceId: string }) => allWorkspacesById.get(workspaceId) ?? null, @@ -393,6 +400,29 @@ describe("ACP session list/resume/fork support", () => { expect(harness.listCalls.slice(0, 2)).toEqual([{ archived: false }, { archived: true }]); }); + it("lists sessions when the activity list is unavailable", async () => { + // activity.list rejects when extensionMetadata.json is unreadable so the + // renderer can keep cached state; ACP has none, so session listing must + // degrade to no-activity sorting instead of failing wholesale. + const workspace = createWorkspaceInfo({ + id: "ws-no-activity", + projectPath: "/repo/a", + namedWorkspacePath: "/repo/a/.mux/ws-no-activity", + createdAt: "2026-02-18T10:00:00.000Z", + }); + const harness = createHarness({ + activeWorkspaces: [workspace], + activityListError: new Error("extension metadata unreadable"), + }); + + await harness.agent.initialize({ protocolVersion: PROTOCOL_VERSION }); + + const response = await harness.agent.listSessions({ cwd: "/repo/a/" }); + expect(response.sessions.map((session) => session.sessionId)).toEqual(["ws-no-activity"]); + // Falls back to createdAt when no activity recency is available. + expect(response.sessions[0]?.updatedAt).toBe("2026-02-18T10:00:00.000Z"); + }); + it("lists and resumes sessions from a sub-project cwd", async () => { const workspace = createWorkspaceInfo({ id: "ws-sub-project", From 995ea09e7428aa553e3874ce92ebbcb7871d0d4e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 11:43:25 +0000 Subject: [PATCH 14/72] test: update activity-list empty-payload contract; cover rejection keep-state path CI caught a stale frontend test pinning the pre-scoping contract (empty list payload preserved cached snapshots). With scoping, {} is an authoritative answer that must clear snapshots for workspaces removed while the renderer was disconnected; backend read failures reject instead. Rewrote the test to the new contract and added the rejection counterpart (cached state preserved, bootstrap retries). --- src/browser/stores/WorkspaceStore.test.ts | 82 +++++++++++++++++++++-- 1 file changed, 76 insertions(+), 6 deletions(-) diff --git a/src/browser/stores/WorkspaceStore.test.ts b/src/browser/stores/WorkspaceStore.test.ts index 0c76d0be0f..20cfd9a27b 100644 --- a/src/browser/stores/WorkspaceStore.test.ts +++ b/src/browser/stores/WorkspaceStore.test.ts @@ -3903,8 +3903,13 @@ describe("WorkspaceStore", () => { } }); - it("preserves cached activity snapshots when list returns an empty payload", async () => { + it("clears cached activity snapshots when list returns an authoritative empty payload", async () => { + // With backend scoping, {} is a valid authoritative answer (no known + // workspace has activity): a reconnecting client must clear snapshots + // for workspaces removed while it was disconnected. Read failures are + // delivered as rejections instead (covered below). const workspaceId = "activity-list-empty-payload"; + const createdAtTimestamp = new Date("2020-01-01T00:00:00.000Z").getTime(); const initialRecency = new Date("2024-01-07T00:00:00.000Z").getTime(); const snapshot: WorkspaceActivitySnapshot = { recency: initialRecency, @@ -3958,11 +3963,76 @@ describe("WorkspaceStore", () => { const sawRetryListCall = await waitUntil(() => listCallCount >= 2); expect(sawRetryListCall).toBe(true); - const stateAfterEmptyList = store.getWorkspaceState(workspaceId); - expect(stateAfterEmptyList.recencyTimestamp).toBe(initialRecency); - expect(stateAfterEmptyList.canInterrupt).toBe(true); - expect(stateAfterEmptyList.currentModel).toBe(snapshot.lastModel); - expect(stateAfterEmptyList.currentThinkingLevel).toBe(snapshot.lastThinkingLevel); + const sawClearedSnapshot = await waitUntil(() => { + const state = store.getWorkspaceState(workspaceId); + return state.recencyTimestamp === createdAtTimestamp && state.canInterrupt === false; + }); + expect(sawClearedSnapshot).toBe(true); + }); + + it("keeps cached activity snapshots when a reconnect list() rejects", async () => { + // A rejection means the backend could not read its state (corrupt + // metadata/config); unlike an authoritative empty payload it must not + // wipe last-known renderer state — the bootstrap loop retries instead. + const workspaceId = "activity-list-rejected-payload"; + const initialRecency = new Date("2024-01-07T00:00:00.000Z").getTime(); + const snapshot: WorkspaceActivitySnapshot = { + recency: initialRecency, + streaming: true, + lastModel: "claude-sonnet-4", + lastThinkingLevel: "high", + }; + + resetStore(); + + let listCallCount = 0; + mockActivityList.mockImplementation( + (): Promise> => { + listCallCount += 1; + if (listCallCount === 1) { + return Promise.resolve({ [workspaceId]: snapshot }); + } + return Promise.reject(new Error("activity list unavailable")); + } + ); + + // eslint-disable-next-line require-yield + mockActivitySubscribe.mockImplementation(async function* ( + _input?: void, + options?: { signal?: AbortSignal } + ): AsyncGenerator { + await waitForAbortSignal(options?.signal); + }); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any + store.setClient({ workspace: mockClient.workspace, terminal: mockClient.terminal } as any); + createAndAddWorkspace( + store, + workspaceId, + { + createdAt: "2020-01-01T00:00:00.000Z", + }, + false + ); + + const seededSnapshot = await waitUntil(() => { + const state = store.getWorkspaceState(workspaceId); + return state.recencyTimestamp === initialRecency && state.canInterrupt === true; + }); + expect(seededSnapshot).toBe(true); + + // Swap to a new client object to force activity subscription restart and a fresh list() call. + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any + store.setClient({ workspace: mockClient.workspace, terminal: mockClient.terminal } as any); + + const sawRejectedListCall = await waitUntil(() => listCallCount >= 2); + expect(sawRejectedListCall).toBe(true); + + const stateAfterRejectedList = store.getWorkspaceState(workspaceId); + expect(stateAfterRejectedList.recencyTimestamp).toBe(initialRecency); + expect(stateAfterRejectedList.canInterrupt).toBe(true); + expect(stateAfterRejectedList.currentModel).toBe(snapshot.lastModel); + expect(stateAfterRejectedList.currentThinkingLevel).toBe(snapshot.lastThinkingLevel); }); }); From 901ad2d516441199f6ba77a7079b65d94951d804 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 11:58:29 +0000 Subject: [PATCH 15/72] review: route ACP fallback through log helper; revalidate list against cross-process removals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-12 Codex findings: - P1: the ACP activity fallback now logs through the shared log helper (stdout stays protocol-safe: the acp adapter already redirects console stdout variants to stderr, and log writes to the file sink). - P2: getActivityList re-reads the (post-prune bounded) metadata file once after its per-workspace awaits and drops entries whose persisted snapshot vanished — catching removals from other backends (XUM_ALLOW_MULTIPLE_INSTANCES) that in-process tombstones cannot see. Best-effort: an unreadable re-read skips revalidation. --- src/node/acp/agent.ts | 6 +-- src/node/services/workspaceService.test.ts | 46 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 24 ++++++++++- 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/src/node/acp/agent.ts b/src/node/acp/agent.ts index 8705a23ebc..2fc0dd2b9b 100644 --- a/src/node/acp/agent.ts +++ b/src/node/acp/agent.ts @@ -35,6 +35,7 @@ import { buildCompactionPrompt, } from "@/common/constants/ui"; import { execFileAsync } from "@/node/utils/disposableExec"; +import { log } from "@/node/services/log"; import { RuntimeConfigSchema } from "@/common/orpc/schemas"; import type { OnChatMode, SendMessageOptions, WorkspaceChatMessage } from "@/common/orpc/types"; import type { AgentSkillDescriptor } from "@/common/types/agentSkill"; @@ -453,10 +454,7 @@ export class MuxAgent implements Agent { this.server.client.workspace.activity .list() .catch((error: unknown): WorkspaceActivityById => { - console.error( - "[acp] Failed to list workspace activity; listing sessions without it", - error - ); + log.error("[acp] Failed to list workspace activity; listing sessions without it", error); return {}; }), ]); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c278db7155..d50e393270 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3438,6 +3438,52 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList drops entries whose metadata another process removed mid-list", async () => { + // XUM_ALLOW_MULTIPLE_INSTANCES: a removal in another backend never + // reaches this process's in-memory tombstones, so the final response + // revalidates against a fresh read of the shared file instead. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "removed-by-other-process"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + await extensionMetadata.updateRecency(workspaceId, 100); + const readSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + spyOn(extensionMetadata, "getAllSnapshots").mockImplementationOnce(async (options) => { + const snapshots = await readSnapshots(options); + // Simulates another backend's removal landing after this request read + // its snapshot view: rewrite the shared file without the entry, with + // no in-process deleteWorkspace tombstone. + await fsPromises.writeFile( + metadataPath, + JSON.stringify({ version: 1, workspaces: {} }), + "utf-8" + ); + return snapshots; + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList[workspaceId]).toBeUndefined(); + // The in-process tombstone was NOT the mechanism here. + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + } finally { + await cleanup(); + } + }); + test("getActivityList rejects on metadata read failure instead of returning {}", async () => { // With scoping, {} is a valid authoritative answer that clears renderer // state; failures must be distinguishable so the renderer keeps its diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index d6f3b12c58..c7bacf4426 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12970,6 +12970,23 @@ export class WorkspaceService extends EventEmitter { } ) ); + // Cross-process counterpart of the in-process tombstone check below: + // with XUM_ALLOW_MULTIPLE_INSTANCES another backend can delete a + // workspace's metadata entry while this list computes, invisible to + // this process's deletedWorkspaceIds. Re-read the (post-prune bounded) + // file once and drop entries whose persisted snapshot vanished — keys + // only disappear through removal. Entries without a persisted snapshot + // (workflow/bash-monitor tombstones, in-memory overlays) are kept. + // Best-effort: an unreadable re-read skips this revalidation instead + // of failing an otherwise complete response. + let freshPersistedIds: ReadonlySet | null = null; + try { + freshPersistedIds = new Set( + (await this.extensionMetadata.getAllSnapshots({ throwOnError: true })).keys() + ); + } catch { + freshPersistedIds = null; + } return Object.fromEntries( entries.filter( (entry): entry is readonly [string, WorkspaceActivitySnapshot] => @@ -12979,7 +12996,12 @@ export class WorkspaceService extends EventEmitter { // delayed response past emitWorkspaceActivity's tombstone // suppression — a renderer that already processed the removal // event would re-insert the deleted id until the next reconnect. - !this.extensionMetadata.isWorkspaceDeleted(entry[0]) + !this.extensionMetadata.isWorkspaceDeleted(entry[0]) && + !( + freshPersistedIds != null && + snapshots.has(entry[0]) && + !freshPersistedIds.has(entry[0]) + ) ) ); } catch (error) { From 8109191138991c47fe6a49efb123b763a01efdda Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 12:13:38 +0000 Subject: [PATCH 16/72] review: quarantine deterministic metadata corruption; config-membership revalidation for snapshotless entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-13 Codex P2s: - Strict metadata reads quarantine deterministically corrupt files (parse/ structure errors — no errno code) to a fixed .corrupt path instead of rethrowing forever: bytes preserved for inspection, re-verified inside the serialized mutation queue so a concurrently repaired file is never moved, and the post-quarantine empty state is authoritative. Transient fs errors keep propagating. - getActivityList's final revalidation also re-checks config membership (raw superset + strict structural load + targeted findWorkspace), covering workflow/bash-monitor-only entries with no persisted snapshot that another backend deregistered mid-list. The strict load's structural validation keeps this exactly as fail-open as the initial scoping. --- src/node/services/ExtensionMetadataService.ts | 54 +++++++++++++- src/node/services/workspaceService.test.ts | 71 +++++++++++++++---- src/node/services/workspaceService.ts | 51 ++++++++++--- 3 files changed, 151 insertions(+), 25 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 9458d81ce7..6be276dce5 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1,5 +1,5 @@ import { dirname } from "path"; -import { mkdir, readFile, access } from "fs/promises"; +import { mkdir, readFile, access, rename } from "fs/promises"; import { constants } from "fs"; import writeFileAtomic from "write-file-atomic"; import { @@ -559,10 +559,60 @@ export class ExtensionMetadataService { }); } + /** + * fs errors carry an errno `code` and may be transient (EACCES/EIO/...); + * anything readFile's content produced afterwards (JSON parse or structure + * validation errors) fails identically for the same bytes on every retry. + */ + private static isDeterministicCorruption(error: unknown): boolean { + return !(typeof error === "object" && error != null && "code" in error); + } + + /** + * Move a deterministically corrupt metadata file aside so strict readers + * stop failing on every retry across process restarts (nothing else + * repairs the file until some unrelated writer happens to replace it). + * The original bytes are preserved at a fixed `.corrupt` path for + * inspection rather than deleted; the fixed name keeps quarantine bounded. + * Serialized with writers, and corruption is re-verified inside the queue + * so a concurrently repaired (just-saved) file is never quarantined. + */ + private async quarantineCorruptFile(): Promise { + return this.withSerializedMutation(async () => { + try { + await this.load({ throwOnError: true }); + return false; // Healed concurrently (or transiently unreadable before). + } catch (error) { + if (!ExtensionMetadataService.isDeterministicCorruption(error)) { + return false; + } + const quarantinePath = `${this.filePath}.corrupt`; + await rename(this.filePath, quarantinePath); + log.error( + `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` + ); + return true; + } + }); + } + async getAllSnapshots(options?: { throwOnError?: boolean; }): Promise> { - const data = await this.load(options); + let data: ExtensionMetadataFile; + try { + data = await this.load(options); + } catch (error) { + // Strict reads must propagate transient failures (renderer keeps + // last-known state and retries), but deterministic corruption would + // fail every retry forever on an idle process. Quarantine the corrupt + // bytes and re-read: the post-quarantine empty state is authoritative. + if (!ExtensionMetadataService.isDeterministicCorruption(error)) { + throw error; + } + await this.quarantineCorruptFile(); + data = await this.load(options); + } const map = new Map(); for (const [workspaceId, entry] of Object.entries(data.workspaces)) { const snapshot = this.toSnapshot(entry); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index d50e393270..4eb96eae4e 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3323,12 +3323,12 @@ describe("WorkspaceService activity list scoping", () => { } }); - test("getActivityList rejects when the metadata file is unreadable or malformed", async () => { - // ExtensionMetadataService.load self-heals a corrupt file into an empty - // one by default; surfacing that here as an authoritative empty list - // would make the renderer wipe every cached snapshot with no retry. - // Parseable-but-malformed workspaces containers enumerate as zero - // entries, so they must follow the same failure path. + test("getActivityList quarantines a deterministically corrupt metadata file", async () => { + // Parse/structure corruption fails identically on every retry, so a + // strict read that only rethrows would leave activity hydration broken + // across restarts until some unrelated writer replaced the file. The + // strict path quarantines the bytes (preserved for inspection, never + // silently deleted) and the resulting empty state is authoritative. const corruptFiles = [ "{not json", JSON.stringify({ version: 2, workspaces: {} }), @@ -3348,18 +3348,21 @@ describe("WorkspaceService activity list scoping", () => { extensionMetadata, }); - // Lenient reads (writer paths) still self-heal. + // Lenient reads (writer paths) self-heal without quarantining. expect((await extensionMetadata.getAllSnapshots()).size).toBe(0); + expect(await fsPromises.readFile(metadataPath, "utf-8")).toBe(corruptFile); - let rejected = false; + const activityList = await workspaceService.getActivityList(); + expect(activityList).toEqual({}); + // The corrupt bytes were moved aside, not destroyed. + expect(await fsPromises.readFile(`${metadataPath}.corrupt`, "utf-8")).toBe(corruptFile); + let mainFileExists = true; try { - await workspaceService.getActivityList(); + await fsPromises.access(metadataPath); } catch { - rejected = true; + mainFileExists = false; } - expect(rejected).toBe(true); - // The self-healing prune path must not have rewritten (reset) the file. - expect(await fsPromises.readFile(metadataPath, "utf-8")).toBe(corruptFile); + expect(mainFileExists).toBe(false); } finally { await cleanup(); } @@ -3484,6 +3487,48 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList drops entries another process deregistered from config mid-list", async () => { + // Covers entries without a persisted snapshot too: the metadata-file + // revalidation cannot see workflow/bash-monitor-only entries, so final + // membership is also re-checked against the shared config state. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "deregistered-by-other-process"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + const readSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + spyOn(extensionMetadata, "getAllSnapshots").mockImplementationOnce(async (options) => { + const snapshots = await readSnapshots(options); + // Simulates another backend deregistering the workspace after this + // request read its snapshot view. The metadata entry stays behind, so + // only the fresh config membership check can catch it. + await config.removeWorkspace(workspaceId); + return snapshots; + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList[workspaceId]).toBeUndefined(); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + } finally { + await cleanup(); + } + }); + test("getActivityList rejects on metadata read failure instead of returning {}", async () => { // With scoping, {} is a valid authoritative answer that clears renderer // state; failures must be distinguishable so the renderer keeps its diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c7bacf4426..f9f380a347 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12970,15 +12970,13 @@ export class WorkspaceService extends EventEmitter { } ) ); - // Cross-process counterpart of the in-process tombstone check below: - // with XUM_ALLOW_MULTIPLE_INSTANCES another backend can delete a - // workspace's metadata entry while this list computes, invisible to - // this process's deletedWorkspaceIds. Re-read the (post-prune bounded) - // file once and drop entries whose persisted snapshot vanished — keys - // only disappear through removal. Entries without a persisted snapshot - // (workflow/bash-monitor tombstones, in-memory overlays) are kept. - // Best-effort: an unreadable re-read skips this revalidation instead - // of failing an otherwise complete response. + // Cross-process counterparts of the in-process tombstone check below: + // with XUM_ALLOW_MULTIPLE_INSTANCES another backend can remove a + // workspace while this list computes, invisible to this process's + // deletedWorkspaceIds. Re-read the (post-prune bounded) metadata file + // and the raw persisted config id superset once. Best-effort: an + // unreadable re-read skips its revalidation instead of failing an + // otherwise complete response. let freshPersistedIds: ReadonlySet | null = null; try { freshPersistedIds = new Set( @@ -12987,6 +12985,33 @@ export class WorkspaceService extends EventEmitter { } catch { freshPersistedIds = null; } + let freshConfigIds: Set | null = null; + try { + // Mirror the prune's union: raw superset (covers entries lenient + // normalization would filter) plus the strictly validated view + // (covers ids of normalized entries; cheap — no fs enrichment + // probes). The strict load throwing on structurally invalid configs + // keeps this revalidation exactly as fail-open as the initial + // scoping above: an untrustworthy config view skips it entirely. + freshConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + const strictConfig = this.config.loadConfigOrDefault({ throwOnError: true }); + for (const [, project] of strictConfig.projects) { + for (const workspace of project.workspaces) { + if (workspace.id) { + freshConfigIds.add(workspace.id); + } + } + } + } catch { + freshConfigIds = null; + } + // Same guard-rails as discardExtensionMetadataEntry: the raw superset + // misses normalized/legacy ids, so ids absent from it get a targeted + // findWorkspace lookup before being treated as removed. + const isRemovedFromConfig = (workspaceId: string): boolean => + freshConfigIds != null && + !freshConfigIds.has(workspaceId) && + this.config.findWorkspace(workspaceId) == null; return Object.fromEntries( entries.filter( (entry): entry is readonly [string, WorkspaceActivitySnapshot] => @@ -12997,11 +13022,17 @@ export class WorkspaceService extends EventEmitter { // suppression — a renderer that already processed the removal // event would re-insert the deleted id until the next reconnect. !this.extensionMetadata.isWorkspaceDeleted(entry[0]) && + // Persisted snapshot vanished from the shared file mid-list + // (metadata keys only disappear through removal). Entries that + // never had a persisted snapshot are covered by the config check. !( freshPersistedIds != null && snapshots.has(entry[0]) && !freshPersistedIds.has(entry[0]) - ) + ) && + // Deregistered from the shared config mid-list — also covers + // workflow/bash-monitor-only entries with no persisted snapshot. + !isRemovedFromConfig(entry[0]) ) ); } catch (error) { From 67c65af33ed534d3e8181852321abc629ab6cf44 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 12:36:53 +0000 Subject: [PATCH 17/72] review: ENOENT-only config existence semantics; propagate legacy identity failures; superset-baseline revalidation Round-14 Codex findings: - P1: readPersistedWorkspaceIdSuperset and loadConfigOrDefault no longer use existsSync probes (false for EACCES/ENOTDIR/EIO too): they read directly and treat only ENOENT as the healthy missing-file state, so a transiently unreadable config throws in strict mode and the prune fails open instead of deleting every metadata entry. - P2: strict getAllWorkspaceMetadata propagates legacy identity-lookup failures (unreadable/unparseable session metadata.json) instead of substituting the generated path id, which would let the prune classify the real stable id's entries as stale. - Reworked the cross-process config revalidation to a like-for-like raw superset comparison (baseline captured before any await vs fresh read after): only ids that verifiably disappeared are dropped. The prior findWorkspace fallback could not resolve legacy metadata.json ids and would have false-dropped them (caught by the new fail-open test). --- src/node/config.ts | 61 ++++++++++++++++--- src/node/services/workspaceService.test.ts | 70 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 41 ++++++------- 3 files changed, 145 insertions(+), 27 deletions(-) diff --git a/src/node/config.ts b/src/node/config.ts index c7b69eed64..22d077beaa 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -80,6 +80,11 @@ import { coerceThinkingLevel, type ThinkingLevel } from "@/common/types/thinking export type { Workspace, ProjectConfig, ProjectsConfig, ProviderConfig, CanonicalProvidersConfig }; export type ProvidersConfig = CanonicalProvidersConfig | Record; +/** True only for fs errors whose errno code is ENOENT (genuinely missing path). */ +function isEnoentError(error: unknown): boolean { + return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT"; +} + function isValidHeartbeatIntervalMs(intervalMs: unknown): intervalMs is number { return ( typeof intervalMs === "number" && @@ -1103,10 +1108,20 @@ export class Config { */ readPersistedWorkspaceIdSuperset(): Set { const ids = new Set(); - if (!fs.existsSync(this.configFile)) { - return ids; + let raw: string; + try { + raw = fs.readFileSync(this.configFile, "utf-8"); + } catch (error) { + // No existsSync probe: it also returns false for EACCES/ENOTDIR/EIO, + // which would report a transiently unreadable config as an empty id + // set and let destructive callers treat every workspace as removed. + // Only a genuinely missing file is a healthy empty set (fresh install). + if (isEnoentError(error)) { + return ids; + } + throw error; } - const parsedValue: unknown = JSON.parse(fs.readFileSync(this.configFile, "utf-8")); + const parsedValue: unknown = JSON.parse(raw); if (!parsedValue || typeof parsedValue !== "object" || Array.isArray(parsedValue)) { throw new Error("Config root must be a JSON object"); } @@ -1137,8 +1152,20 @@ export class Config { // actually failed parsing. let rawBytes: Buffer | undefined; try { - if (fs.existsSync(this.configFile)) { + try { rawBytes = fs.readFileSync(this.configFile); + } catch (readError) { + // No existsSync probe: it also returns false for EACCES/ENOTDIR/EIO, + // which would silently select the fresh-install default while the + // config is merely transiently unreadable — in strict mode that + // empty view feeds destructive "not in config" decisions. Route + // non-ENOENT failures through the shared failure path below + // (throwOnError callers rethrow); only ENOENT means missing. + if (!isEnoentError(readError)) { + throw readError; + } + } + if (rawBytes !== undefined) { const parsedValue: unknown = JSON.parse(rawBytes.toString("utf-8")); if (!parsedValue || typeof parsedValue !== "object" || Array.isArray(parsedValue)) { throw new Error("Config root must be a JSON object"); @@ -2460,9 +2487,21 @@ export class Config { const metadataPath = path.join(this.getSessionDir(legacyId), "metadata.json"); let metadataFound = false; - if (fs.existsSync(metadataPath)) { - const data = fs.readFileSync(metadataPath, "utf-8"); - const metadata = JSON.parse(data) as WorkspaceMetadata; + let legacyMetadataRaw: string | undefined; + try { + legacyMetadataRaw = fs.readFileSync(metadataPath, "utf-8"); + } catch (readError) { + // Missing is normal (most entries never had a legacy + // metadata.json). Any other failure means the workspace's + // authoritative stable id is unknowable right now — surface it + // to the catch below instead of silently substituting the + // generated legacy path id via the !metadataFound branch. + if (!isEnoentError(readError)) { + throw readError; + } + } + if (legacyMetadataRaw !== undefined) { + const metadata = JSON.parse(legacyMetadataRaw) as WorkspaceMetadata; this.rememberLegacyTaskVariantWorkspace(projectPath, metadata, "metadata"); // Ensure required fields are present @@ -2636,6 +2675,14 @@ export class Config { ); } } catch (error) { + // Strict callers make destructive "id is not known" decisions (the + // extension-metadata prune): for a legacy entry whose stable id + // lives only in its unreadable/unparseable metadata.json, the + // generated-path-id fallback below would classify the real id's + // entries as stale. Propagate the identity-lookup failure instead. + if (options?.throwOnError) { + throw error; + } log.error(`Failed to load/migrate workspace metadata:`, error); // Fallback to basic metadata if migration fails const legacyId = this.generateLegacyId(projectPath, workspace.path); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4eb96eae4e..cf420d9183 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3600,6 +3600,76 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("falls back to the unscoped union when config.json exists but cannot be read", async () => { + // EISDIR here; EACCES/ENOTDIR/EIO in the field. existsSync-style probes + // report all of these as "missing", which would masquerade as an empty + // config and let the prune delete every metadata entry instead of + // failing open. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("possibly-live", 100); + const configPath = path.join(config.rootDir, "config.json"); + await fsPromises.rm(configPath, { force: true }); + await fsPromises.mkdir(configPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList["possibly-live"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); + } finally { + await cleanup(); + } + }); + + test("fails open when a legacy workspace's identity lookup fails", async () => { + // A legacy config entry without an id resolves its authoritative stable + // id from its session metadata.json. If that file is unreadable or + // unparseable, the lenient path substitutes the generated path id — the + // strict enumeration must instead propagate the failure so the prune + // cannot classify the real stable id's entries as stale. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("legacy-stable-id", 100); + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + // Corrupt the metadata.json holding that entry's stable id. + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile(path.join(legacySessionDir, "metadata.json"), "{not json"); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + // Fail open: the identity of the legacy workspace is unknowable, so + // nothing may be dropped from the list or pruned from disk. + expect(activityList["legacy-stable-id"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("legacy-stable-id")).toBe(true); + } finally { + await cleanup(); + } + }); + test("never prunes entries whose config entry is discarded by normalization", async () => { // A parseable config entry that lenient normalization filters out (null // project path): the workspace vanishes from the normalized view — and diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index f9f380a347..1cc0a72321 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12876,6 +12876,16 @@ export class WorkspaceService extends EventEmitter { // On the first bootstrap the prune already enumerated the config; reuse // that id set instead of paying the per-workspace disk walk twice. const prefetchedKnownIds = await this.pruneStaleExtensionMetadataOnce(); + // Baseline for the post-await cross-process removal revalidation at + // the end of this method: captured before any snapshot/config await so + // ids deregistered from the shared config while this list computes can + // be told apart from ids the raw scan can never see. + let initialConfigIds: ReadonlySet | null = null; + try { + initialConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + } catch { + initialConfigIds = null; + } // throwOnError: the default load self-heals an unreadable/malformed // metadata file into an empty one, which this list would then present // as an authoritative "no activity anywhere" answer — the renderer @@ -12985,33 +12995,24 @@ export class WorkspaceService extends EventEmitter { } catch { freshPersistedIds = null; } - let freshConfigIds: Set | null = null; + let freshConfigIds: ReadonlySet | null = null; try { - // Mirror the prune's union: raw superset (covers entries lenient - // normalization would filter) plus the strictly validated view - // (covers ids of normalized entries; cheap — no fs enrichment - // probes). The strict load throwing on structurally invalid configs - // keeps this revalidation exactly as fail-open as the initial - // scoping above: an untrustworthy config view skips it entirely. freshConfigIds = this.config.readPersistedWorkspaceIdSuperset(); - const strictConfig = this.config.loadConfigOrDefault({ throwOnError: true }); - for (const [, project] of strictConfig.projects) { - for (const workspace of project.workspaces) { - if (workspace.id) { - freshConfigIds.add(workspace.id); - } - } - } } catch { freshConfigIds = null; } - // Same guard-rails as discardExtensionMetadataEntry: the raw superset - // misses normalized/legacy ids, so ids absent from it get a targeted - // findWorkspace lookup before being treated as removed. + // Like-for-like raw-superset comparison only: an id that WAS persisted + // in config before the awaits and is gone from the fresh raw view was + // verifiably deregistered. Ids the raw scan cannot see (legacy entries + // whose stable id lives in a session metadata.json, in-memory migration + // ids) never appear in either view and are conservatively retained — + // dropping them on a cheap fresh view would misclassify identity-lookup + // gaps as removals. Skipped when either superset read fails. const isRemovedFromConfig = (workspaceId: string): boolean => + initialConfigIds != null && freshConfigIds != null && - !freshConfigIds.has(workspaceId) && - this.config.findWorkspace(workspaceId) == null; + initialConfigIds.has(workspaceId) && + !freshConfigIds.has(workspaceId); return Object.fromEntries( entries.filter( (entry): entry is readonly [string, WorkspaceActivitySnapshot] => From c2f7da9f1f5a895b260b7877816e6ca7b4c77496 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 12:54:25 +0000 Subject: [PATCH 18/72] test: accept the corrupt-config write gate as the trust CLI's loud failure surface With ENOENT-only existence semantics, an ENOTDIR config read now routes through handleConfigLoadFailure and the corrupt-config write gate refuses to write defaults over the unreadable file (previously existsSync treated it as missing and the failure only surfaced via post-write verification). The CLI still exits non-zero with an explanatory stderr and no success output; the test now accepts either failure surface. --- src/cli/trust.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cli/trust.test.ts b/src/cli/trust.test.ts index 02bec5459c..12bd5e7eb0 100644 --- a/src/cli/trust.test.ts +++ b/src/cli/trust.test.ts @@ -113,7 +113,14 @@ describe("xum trust CLI", () => { .quiet(); expect(result.exitCode).not.toBe(0); - expect(result.stderr.toString()).toContain("Failed to persist trust change"); + // Either failure surface is acceptable: the corrupt-config write gate + // (config.json exists but cannot be read, so editConfig refuses to write + // defaults over it) or the post-write trust verification (the write was + // silently swallowed). Both must fail loudly instead of reporting + // success. + expect(result.stderr.toString()).toMatch( + /Failed to persist trust change|Skipping config write/ + ); expect(result.stdout.toString()).toBe(""); }, 15_000); From 0c14bdcec69568a5020b634d539f227305eafeab Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 13:08:42 +0000 Subject: [PATCH 19/72] review: pre-prune removal baseline; content-guarded quarantine restore Round-16 Codex P2s (multi-instance races): - The cross-process removal baseline is captured before ANY await, including the first-bootstrap prune, so a removal landing during the prune cannot outdate the comparison for snapshotless entries. - quarantineCorruptFile re-validates the bytes it actually moved: a healthy file another backend atomically saved between validation and rename is restored via link (EEXIST-safe, never overwrites a newer file) instead of being discarded as corrupt. Quarantine failures are best-effort at the read site; the re-read decides the outcome. --- src/node/services/ExtensionMetadataService.ts | 64 ++++++++++++++----- src/node/services/workspaceService.ts | 10 +-- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 6be276dce5..3f959ba9e9 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1,5 +1,5 @@ import { dirname } from "path"; -import { mkdir, readFile, access, rename } from "fs/promises"; +import { mkdir, readFile, access, rename, link, unlink } from "fs/promises"; import { constants } from "fs"; import writeFileAtomic from "write-file-atomic"; import { @@ -195,13 +195,7 @@ export class ExtensionMetadataService { // file with e.g. an array or primitive `workspaces` would otherwise // enumerate as zero entries and masquerade as an authoritative empty // state in strict reads. - if ( - typeof parsed !== "object" || - parsed?.version !== 1 || - typeof parsed.workspaces !== "object" || - parsed.workspaces === null || - Array.isArray(parsed.workspaces) - ) { + if (!ExtensionMetadataService.isValidMetadataFileShape(parsed)) { throw new Error("Invalid extension metadata file structure"); } @@ -568,6 +562,19 @@ export class ExtensionMetadataService { return !(typeof error === "object" && error != null && "code" in error); } + private static isValidMetadataFileShape(parsed: unknown): parsed is ExtensionMetadataFile { + if (typeof parsed !== "object" || parsed === null) { + return false; + } + const candidate = parsed as { version?: unknown; workspaces?: unknown }; + return ( + candidate.version === 1 && + typeof candidate.workspaces === "object" && + candidate.workspaces !== null && + !Array.isArray(candidate.workspaces) + ); + } + /** * Move a deterministically corrupt metadata file aside so strict readers * stop failing on every retry across process restarts (nothing else @@ -586,13 +593,35 @@ export class ExtensionMetadataService { if (!ExtensionMetadataService.isDeterministicCorruption(error)) { return false; } - const quarantinePath = `${this.filePath}.corrupt`; - await rename(this.filePath, quarantinePath); - log.error( - `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` - ); - return true; } + const quarantinePath = `${this.filePath}.corrupt`; + await rename(this.filePath, quarantinePath); + // The mutation queue is process-local: another backend's atomic save + // can land a healthy file between the validation above and the rename, + // and the rename would move THAT file aside. Re-validate the bytes that + // actually got moved and undo the move when they turn out healthy. + try { + const moved: unknown = JSON.parse(await readFile(quarantinePath, "utf-8")); + if (ExtensionMetadataService.isValidMetadataFileShape(moved)) { + try { + // link (not rename) restores: it fails with EEXIST when yet + // another writer already re-created the main path, so a newer + // file is never overwritten by the restore. + await link(quarantinePath, this.filePath); + await unlink(quarantinePath); + } catch { + // EEXIST or fs without hard links: a healthy leftover sidecar is + // harmless (bounded by the fixed name) — never destroy data. + } + return false; + } + } catch { + // Still unreadable/corrupt — the expected quarantine outcome. + } + log.error( + `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` + ); + return true; }); } @@ -610,7 +639,12 @@ export class ExtensionMetadataService { if (!ExtensionMetadataService.isDeterministicCorruption(error)) { throw error; } - await this.quarantineCorruptFile(); + try { + await this.quarantineCorruptFile(); + } catch { + // Best-effort: e.g. the file vanished between validation and rename + // (another process moved it); the re-read below decides the outcome. + } data = await this.load(options); } const map = new Map(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 1cc0a72321..73306be9bf 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12875,17 +12875,19 @@ export class WorkspaceService extends EventEmitter { try { // On the first bootstrap the prune already enumerated the config; reuse // that id set instead of paying the per-workspace disk walk twice. - const prefetchedKnownIds = await this.pruneStaleExtensionMetadataOnce(); // Baseline for the post-await cross-process removal revalidation at - // the end of this method: captured before any snapshot/config await so - // ids deregistered from the shared config while this list computes can - // be told apart from ids the raw scan can never see. + // the end of this method: captured before ANY await (including the + // first-bootstrap prune, whose enumeration another backend's removal + // could otherwise outdate before this baseline is read) so ids + // deregistered from the shared config while this list computes can be + // told apart from ids the raw scan can never see. let initialConfigIds: ReadonlySet | null = null; try { initialConfigIds = this.config.readPersistedWorkspaceIdSuperset(); } catch { initialConfigIds = null; } + const prefetchedKnownIds = await this.pruneStaleExtensionMetadataOnce(); // throwOnError: the default load self-heals an unreadable/malformed // metadata file into an empty one, which this list would then present // as an authoritative "no activity anywhere" answer — the renderer From 9e364495244439c02aaf250647985b5db2249b08 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 13:25:33 +0000 Subject: [PATCH 20/72] review: copy-based quarantine restore fallback; authoritative legacy id in findWorkspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-17 Codex P2s: - quarantineCorruptFile's restore no longer treats a failed hard-link as harmless: non-EEXIST link failures fall back to COPYFILE_EXCL (same no-overwrite guarantee without hard-link support), and a restore that still fails rethrows so strict readers see a retryable failure instead of ENOENT-as-authoritative-empty while healthy bytes sit in the sidecar. - findWorkspace's legacy branch now also resolves the stable id through sessions//metadata.json — the same authoritative path getAllWorkspaceMetadata uses — so the extension-metadata discard cannot report a still-registered id-less legacy workspace as absent and tombstone its activity writes. --- src/node/config.ts | 30 ++++++++++++- src/node/services/ExtensionMetadataService.ts | 36 +++++++++++---- src/node/services/workspaceService.test.ts | 45 +++++++++++++++++++ 3 files changed, 102 insertions(+), 9 deletions(-) diff --git a/src/node/config.ts b/src/node/config.ts index 22d077beaa..4f68a0b578 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2240,8 +2240,36 @@ export class Config { } } - // Try legacy ID format as last resort + // Authoritative legacy path: getAllWorkspaceMetadata resolves an + // id-less entry's stable id from sessions// + // metadata.json (NOT the basename path above). Callers verifying + // "is this id still registered" (e.g. the extension-metadata + // discard) must see the same identity, or a stable id that lives + // only in that file would be reported absent while its workspace + // remains registered. const legacyId = this.generateLegacyId(projectPath, workspace.path); + const legacyMetadataPath = path.join(this.getSessionDir(legacyId), "metadata.json"); + if (fs.existsSync(legacyMetadataPath)) { + try { + const legacyData = fs.readFileSync(legacyMetadataPath, "utf-8"); + const legacyMetadata = JSON.parse(legacyData) as WorkspaceMetadata; + this.rememberLegacyTaskVariantWorkspace(projectPath, legacyMetadata, "metadata"); + if (legacyMetadata.id === workspaceId) { + return { + workspacePath: workspace.path, + projectPath, + attributionProjectPath, + projects: legacyMetadata.projects ?? workspace.projects, + workspaceName: undefined, + parentWorkspaceId: undefined, + }; + } + } catch { + // Ignore parse errors, try legacy ID + } + } + + // Try legacy ID format as last resort if (legacyId === workspaceId) { return { workspacePath: workspace.path, diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 3f959ba9e9..44b09f3014 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1,5 +1,5 @@ import { dirname } from "path"; -import { mkdir, readFile, access, rename, link, unlink } from "fs/promises"; +import { mkdir, readFile, access, rename, link, unlink, copyFile } from "fs/promises"; import { constants } from "fs"; import writeFileAtomic from "write-file-atomic"; import { @@ -562,6 +562,10 @@ export class ExtensionMetadataService { return !(typeof error === "object" && error != null && "code" in error); } + private static isErrnoCode(error: unknown, code: string): boolean { + return typeof error === "object" && error != null && "code" in error && error.code === code; + } + private static isValidMetadataFileShape(parsed: unknown): parsed is ExtensionMetadataFile { if (typeof parsed !== "object" || parsed === null) { return false; @@ -603,16 +607,32 @@ export class ExtensionMetadataService { try { const moved: unknown = JSON.parse(await readFile(quarantinePath, "utf-8")); if (ExtensionMetadataService.isValidMetadataFileShape(moved)) { + // Restore without ever overwriting a newer file yet another writer + // may have re-created at the main path: link and COPYFILE_EXCL + // both fail with EEXIST in that case (the healthy leftover sidecar + // is then harmless and bounded by the fixed name). try { - // link (not rename) restores: it fails with EEXIST when yet - // another writer already re-created the main path, so a newer - // file is never overwritten by the restore. await link(quarantinePath, this.filePath); - await unlink(quarantinePath); - } catch { - // EEXIST or fs without hard links: a healthy leftover sidecar is - // harmless (bounded by the fixed name) — never destroy data. + } catch (linkError) { + if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + return false; + } + try { + // Filesystems without hard-link support (or EPERM): copy-based + // restore with the same EEXIST no-overwrite guarantee. + await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + return false; + } + // Restore failed with the main path missing: rethrow so the + // strict reader propagates a retryable failure instead of + // reading ENOENT as an authoritative empty state while the + // healthy bytes sit in the sidecar. + throw copyError; + } } + await unlink(quarantinePath).catch(() => undefined); return false; } } catch { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index cf420d9183..47ffc9218e 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3323,6 +3323,51 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("discardExtensionMetadataEntry keeps entries of id-less legacy workspaces", async () => { + // An id-less legacy config entry resolves its stable id from + // sessions//metadata.json. The raw config scan + // cannot see that id, so the discard's registration check must resolve + // it through the same authoritative path getAllWorkspaceMetadata uses; + // otherwise the still-registered workspace would be reported absent and + // its activity writes permanently tombstoned for this process. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "legacy-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(stableId, 100); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + await workspaceService.discardExtensionMetadataEntry(stableId); + + expect((await extensionMetadata.getAllSnapshots()).has(stableId)).toBe(true); + expect(extensionMetadata.isWorkspaceDeleted(stableId)).toBe(false); + } finally { + await cleanup(); + } + }); + test("getActivityList quarantines a deterministically corrupt metadata file", async () => { // Parse/structure corruption fails identically on every retry, so a // strict read that only rethrows would leave activity hydration broken From 3813ecda8627ad202ac8f809d175b52263effec1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 13:43:50 +0000 Subject: [PATCH 21/72] review: propagate quarantine restore failures; refresh first-bootstrap scoping; strict findWorkspace identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-18 Codex P2s: - quarantineCorruptFile no longer swallows restore/verification failures: only deterministic sidecar parse corruption reads as 'still corrupt', and the read-site wrapper swallows only ENOENT — restore or I/O failures reject so strict readers retry instead of treating the missing main path as authoritative empty while healthy bytes sit in the sidecar. - First-bootstrap scoping re-admits snapshot ids a fresh raw config view knows: a workspace registered between the prune's enumeration and the snapshot read is no longer omitted from the authoritative list (which would clear its live-arrived renderer state). - findWorkspace gained a throwOnError mode that propagates identity-hiding failures (strict config load; unreadable/unparseable legacy session metadata.json with ENOENT-only-missing semantics), and the extension- metadata discard uses it so 'identity unknowable' fails closed instead of reading as 'not registered'. --- src/node/config.ts | 91 +++++++++++-------- src/node/services/ExtensionMetadataService.ts | 76 ++++++++++------ src/node/services/workspaceService.ts | 25 ++++- 3 files changed, 125 insertions(+), 67 deletions(-) diff --git a/src/node/config.ts b/src/node/config.ts index 4f68a0b578..5a2fe9a6d1 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2182,7 +2182,19 @@ export class Config { * Find a workspace by ID. * @returns Stored config project key plus a separate attribution project path, or null */ - findWorkspace(workspaceId: string): { + findWorkspace( + workspaceId: string, + options?: { + /** + * Propagate failures that hide a workspace's identity (unreadable or + * unparseable config / legacy session metadata.json) instead of + * skipping the entry. Callers making destructive "id is not + * registered" decisions must use this: a lenient miss is + * indistinguishable from a genuine absence. + */ + throwOnError?: boolean; + } + ): { workspacePath: string; projectPath: string; attributionProjectPath?: string; @@ -2191,7 +2203,7 @@ export class Config { parentWorkspaceId?: string; pendingAutoTitle?: boolean; } | null { - const config = this.loadConfigOrDefault(); + const config = this.loadConfigOrDefault({ throwOnError: options?.throwOnError }); for (const [projectPath, project] of config.projects) { for (const workspace of project.workspaces) { @@ -2220,24 +2232,29 @@ export class Config { // Try loading metadata with basename as ID (works for old workspaces) const metadataPath = path.join(this.getSessionDir(workspaceBasename), "metadata.json"); - if (fs.existsSync(metadataPath)) { - try { - const data = fs.readFileSync(metadataPath, "utf-8"); - const metadata = JSON.parse(data) as WorkspaceMetadata; - this.rememberLegacyTaskVariantWorkspace(projectPath, metadata, "metadata"); - if (metadata.id === workspaceId) { - return { - workspacePath: workspace.path, - projectPath, - attributionProjectPath, - projects: metadata.projects ?? workspace.projects, - workspaceName: undefined, - parentWorkspaceId: undefined, - }; - } - } catch { - // Ignore parse errors, try legacy ID + try { + const data = fs.readFileSync(metadataPath, "utf-8"); + const metadata = JSON.parse(data) as WorkspaceMetadata; + this.rememberLegacyTaskVariantWorkspace(projectPath, metadata, "metadata"); + if (metadata.id === workspaceId) { + return { + workspacePath: workspace.path, + projectPath, + attributionProjectPath, + projects: metadata.projects ?? workspace.projects, + workspaceName: undefined, + parentWorkspaceId: undefined, + }; } + } catch (error) { + // A genuinely missing file is the common case (most entries have + // no legacy metadata.json). The entry's identity may live in an + // unreadable/unparseable one though, so strict callers must not + // conclude "absent" from a failed lookup. + if (!isEnoentError(error) && options?.throwOnError) { + throw error; + } + // Ignore errors, try legacy ID } // Authoritative legacy path: getAllWorkspaceMetadata resolves an @@ -2249,24 +2266,26 @@ export class Config { // remains registered. const legacyId = this.generateLegacyId(projectPath, workspace.path); const legacyMetadataPath = path.join(this.getSessionDir(legacyId), "metadata.json"); - if (fs.existsSync(legacyMetadataPath)) { - try { - const legacyData = fs.readFileSync(legacyMetadataPath, "utf-8"); - const legacyMetadata = JSON.parse(legacyData) as WorkspaceMetadata; - this.rememberLegacyTaskVariantWorkspace(projectPath, legacyMetadata, "metadata"); - if (legacyMetadata.id === workspaceId) { - return { - workspacePath: workspace.path, - projectPath, - attributionProjectPath, - projects: legacyMetadata.projects ?? workspace.projects, - workspaceName: undefined, - parentWorkspaceId: undefined, - }; - } - } catch { - // Ignore parse errors, try legacy ID + try { + const legacyData = fs.readFileSync(legacyMetadataPath, "utf-8"); + const legacyMetadata = JSON.parse(legacyData) as WorkspaceMetadata; + this.rememberLegacyTaskVariantWorkspace(projectPath, legacyMetadata, "metadata"); + if (legacyMetadata.id === workspaceId) { + return { + workspacePath: workspace.path, + projectPath, + attributionProjectPath, + projects: legacyMetadata.projects ?? workspace.projects, + workspaceName: undefined, + parentWorkspaceId: undefined, + }; + } + } catch (error) { + // Same strict-mode contract as the basename lookup above. + if (!isEnoentError(error) && options?.throwOnError) { + throw error; } + // Ignore errors, try legacy ID } // Try legacy ID format as last resort diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 44b09f3014..d1a2b9a976 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -604,39 +604,48 @@ export class ExtensionMetadataService { // can land a healthy file between the validation above and the rename, // and the rename would move THAT file aside. Re-validate the bytes that // actually got moved and undo the move when they turn out healthy. + // Only deterministic parse corruption is the expected quarantine + // outcome here; a transient failure reading the sidecar means the + // moved bytes cannot be verified, so it propagates (retryable) rather + // than reporting a successful reset over possibly-healthy data. + let moved: unknown; + let movedParses = true; try { - const moved: unknown = JSON.parse(await readFile(quarantinePath, "utf-8")); - if (ExtensionMetadataService.isValidMetadataFileShape(moved)) { - // Restore without ever overwriting a newer file yet another writer - // may have re-created at the main path: link and COPYFILE_EXCL - // both fail with EEXIST in that case (the healthy leftover sidecar - // is then harmless and bounded by the fixed name). + moved = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; + } catch (readError) { + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + movedParses = false; + } + if (movedParses && ExtensionMetadataService.isValidMetadataFileShape(moved)) { + // Restore without ever overwriting a newer file yet another writer + // may have re-created at the main path: link and COPYFILE_EXCL + // both fail with EEXIST in that case (the healthy leftover sidecar + // is then harmless and bounded by the fixed name). + try { + await link(quarantinePath, this.filePath); + } catch (linkError) { + if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + return false; + } try { - await link(quarantinePath, this.filePath); - } catch (linkError) { - if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + // Filesystems without hard-link support (or EPERM): copy-based + // restore with the same EEXIST no-overwrite guarantee. + await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { return false; } - try { - // Filesystems without hard-link support (or EPERM): copy-based - // restore with the same EEXIST no-overwrite guarantee. - await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); - } catch (copyError) { - if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { - return false; - } - // Restore failed with the main path missing: rethrow so the - // strict reader propagates a retryable failure instead of - // reading ENOENT as an authoritative empty state while the - // healthy bytes sit in the sidecar. - throw copyError; - } + // Restore failed with the main path missing: rethrow so the + // strict reader propagates a retryable failure instead of + // reading ENOENT as an authoritative empty state while the + // healthy bytes sit in the sidecar. + throw copyError; } - await unlink(quarantinePath).catch(() => undefined); - return false; } - } catch { - // Still unreadable/corrupt — the expected quarantine outcome. + await unlink(quarantinePath).catch(() => undefined); + return false; } log.error( `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` @@ -661,9 +670,16 @@ export class ExtensionMetadataService { } try { await this.quarantineCorruptFile(); - } catch { - // Best-effort: e.g. the file vanished between validation and rename - // (another process moved it); the re-read below decides the outcome. + } catch (quarantineError) { + // ENOENT means the file vanished between validation and rename + // (another process moved it) — the re-read below decides the + // outcome. Anything else (rename denied, sidecar unverifiable, + // restore failed) must stay a retryable failure: an ENOENT re-read + // would masquerade as authoritative empty while the moved bytes may + // hold healthy data. + if (!ExtensionMetadataService.isErrnoCode(quarantineError, "ENOENT")) { + throw quarantineError; + } } data = await this.load(options); } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 73306be9bf..ac31b4e19a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12797,7 +12797,13 @@ export class WorkspaceService extends EventEmitter { // and the targeted findWorkspace lookup covers normalized/legacy ids // (metadata.json / generated legacy ids) the raw scan cannot see. const knownIds = this.config.readPersistedWorkspaceIdSuperset(); - if (knownIds.has(workspaceId) || this.config.findWorkspace(workspaceId) != null) { + // throwOnError: a lenient findWorkspace swallows unreadable legacy + // metadata.json files, and "identity unknowable" must fail closed here + // (skip the delete) rather than read as "not registered". + if ( + knownIds.has(workspaceId) || + this.config.findWorkspace(workspaceId, { throwOnError: true }) != null + ) { log.debug("Skipping extension metadata discard: workspace still persisted in config", { workspaceId, }); @@ -12906,6 +12912,23 @@ export class WorkspaceService extends EventEmitter { let workspaceIds: Set; if (prefetchedKnownIds != null) { workspaceIds = prefetchedKnownIds; + // The prune enumerated config BEFORE the snapshot read above, so a + // workspace registered in between (whose first activity write is + // already in `snapshots`) would be missing here — and an + // authoritative list omitting it would clear its live-arrived + // renderer state with no retry. Re-admit snapshot ids that a fresh + // raw config view now knows (cheap sync read; on failure the + // prefetched view stands and the miss is a transient one). + try { + const refreshedConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + for (const workspaceId of snapshots.keys()) { + if (!workspaceIds.has(workspaceId) && refreshedConfigIds.has(workspaceId)) { + workspaceIds.add(workspaceId); + } + } + } catch (error) { + log.debug("Failed to refresh config ids for first-bootstrap scoping", { error }); + } } else { try { // throwOnError so a corrupted config.json actually reaches the From a5c9018496d0afdaf17446965c2ac86e7ff97339 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 14:21:43 +0000 Subject: [PATCH 22/72] review: address round-19 findings - getActivityList first-bootstrap refresh admits every id in the refreshed raw config view, not only ids with a persisted snapshot, so concurrently registered workspaces with workflow/bash-monitor-only activity get their per-id probe - strict getAllWorkspaceMetadata fails closed when a legacy metadata.json parses without a usable id (e.g. {}), instead of enumerating an id-less entry that would classify the real stable id as stale - quarantine leaves a valid empty main metadata file behind (exclusive link/COPYFILE_EXCL create) and load() treats missing-main-with-sidecar as a retryable failure instead of a healthy empty state, closing the mid-quarantine ENOENT window - emitWorkspaceActivity runs the removed-workspace tombstone check on the MERGED payload: workflow/bash-monitor cache overlays can turn a null snapshot into a non-null activity that would re-insert the deleted id --- src/node/config.ts | 15 +++ .../services/ExtensionMetadataService.test.ts | 27 ++++ src/node/services/ExtensionMetadataService.ts | 53 +++++++- src/node/services/workspaceService.test.ts | 122 +++++++++++++++++- src/node/services/workspaceService.ts | 45 ++++--- 5 files changed, 232 insertions(+), 30 deletions(-) diff --git a/src/node/config.ts b/src/node/config.ts index 5a2fe9a6d1..1a9595f769 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2607,6 +2607,21 @@ export class Config { // different value here would hand the UI an ID that findWorkspace cannot // resolve until the next reload. metadata.id = workspace.id ?? metadata.id; + // Strict callers make destructive "id is not known" decisions (the + // extension-metadata prune): metadata.json that parses but carries + // no usable id (e.g. `{}`, or a non-object like an array) resolves + // to an id-less entry here, and the workspace's REAL stable id — + // recorded nowhere else — would be classified as stale and its + // activity data permanently deleted. Successful JSON parsing does + // not establish identity; fail closed instead. + if ( + options?.throwOnError && + !(typeof metadata.id === "string" && metadata.id.length > 0) + ) { + throw new Error( + `Legacy workspace metadata at ${metadataPath} resolved without a usable id` + ); + } metadata.name = workspace.name ?? metadata.name; metadata.createdAt = workspace.createdAt ?? metadata.createdAt; metadata.runtimeConfig = workspace.runtimeConfig ?? metadata.runtimeConfig; diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 82694c9575..cb5dc66532 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -587,4 +587,31 @@ describe("ExtensionMetadataService", () => { ).toBe(0); expect(await readFile(filePath, "utf-8")).toBe(rawContent); }); + + test("a missing main file with the quarantine sidecar present is not an empty state", async () => { + // Mid-quarantine window: the corrupt (or concurrently repaired) bytes + // were renamed to the sidecar and the empty replacement is not written + // yet. Strict readers must get a retryable failure — an authoritative {} + // returned here could not be retracted by the subsequent restore. + await writeFile(`${filePath}.corrupt`, "{not json"); + // Main file intentionally absent (beforeEach never creates it). + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + // The failure is classified transient (errno-carrying), so no quarantine + // cascade: the main path stays absent rather than being reset to empty. + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toBe("{not json"); + + // Lenient (writer) reads keep self-healing so mutations make progress. + expect((await service.getAllSnapshots()).size).toBe(0); + + // Without the sidecar, a missing main file stays a healthy empty state. + await rm(`${filePath}.corrupt`); + expect((await service.getAllSnapshots({ throwOnError: true })).size).toBe(0); + }); }); diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index d1a2b9a976..a5ede60d40 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1,5 +1,5 @@ import { dirname } from "path"; -import { mkdir, readFile, access, rename, link, unlink, copyFile } from "fs/promises"; +import { mkdir, readFile, access, rename, link, unlink, copyFile, writeFile } from "fs/promises"; import { constants } from "fs"; import writeFileAtomic from "write-file-atomic"; import { @@ -208,7 +208,22 @@ export class ExtensionMetadataService { // writers can always make progress. if (typeof error === "object" && error != null && "code" in error) { if (error.code === "ENOENT") { - return { version: 1, workspaces: {} }; + // A missing main file WITH the quarantine sidecar present is not a + // healthy empty state: it is the (rare) window where quarantine + // moved the file aside and has not yet written the empty + // replacement — the moved bytes may even be a concurrent writer's + // healthy repair awaiting restore. Strict readers must see a + // retryable failure (ENOENT carries an errno code, so callers + // classify it as transient), not an authoritative {} that a + // subsequent restore cannot retract. Lenient (writer) paths keep + // self-healing below so mutations always make progress. + const quarantined = await access(`${this.filePath}.corrupt`).then( + () => true, + () => false + ); + if (!quarantined) { + return { version: 1, workspaces: {} }; + } } } if (options?.throwOnError) { @@ -647,6 +662,40 @@ export class ExtensionMetadataService { await unlink(quarantinePath).catch(() => undefined); return false; } + // Replace the quarantined main file with a valid EMPTY file instead of + // leaving the path missing: between the rename above and here, strict + // readers (this or another process) would otherwise observe ENOENT. + // A missing-main window is dangerous because load() treats plain + // ENOENT as a healthy empty state — combined with a concurrent writer + // repairing the file right before the rename, a reader in the gap + // could return an authoritative {} that a later restore cannot + // retract. With a real empty file the corrupt→empty transition is + // atomic (link/COPYFILE_EXCL below never overwrite a file a + // concurrent writer re-created first), and load()'s sidecar check + // turns any remaining missing-main window into a retryable failure + // instead of an empty read. + const emptyTmpPath = `${this.filePath}.empty.tmp`; + await writeFile( + emptyTmpPath, + JSON.stringify({ version: 1, workspaces: {} } satisfies ExtensionMetadataFile, null, 2), + "utf-8" + ); + try { + await link(emptyTmpPath, this.filePath); + } catch (linkError) { + if (!ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + try { + await copyFile(emptyTmpPath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + // Main path still missing: rethrow (retryable) so the caller's + // re-read hits the sidecar guard instead of reading ENOENT. + throw copyError; + } + } + } + } + await unlink(emptyTmpPath).catch(() => undefined); log.error( `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` ); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ca0d6e9a82..6319d104cc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3246,6 +3246,57 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("first bootstrap admits snapshotless ids registered after the prune enumerated config", async () => { + // A workspace another backend registers after the prune captured its id + // set may have workflow- or bash-monitor-only activity and therefore no + // extensionMetadata snapshot. Admission must come from the refreshed raw + // config view — filtering through snapshot keys would skip the per-id + // workflow probe entirely and return an authoritative list without it. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const lateWorkspaceId = "late-registered-workspace"; + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realSuperset = config.readPersistedWorkspaceIdSuperset.bind(config); + let supersetCalls = 0; + const supersetSpy = spyOn(config, "readPersistedWorkspaceIdSuperset").mockImplementation( + () => { + supersetCalls += 1; + const ids = realSuperset(); + // Calls 1 (pre-await baseline) and 2 (prune enumeration) see the + // pre-registration config; the refresh and the post-await + // revalidation see the concurrently registered workspace. + if (supersetCalls >= 3) { + ids.add(lateWorkspaceId); + } + return ids; + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Workflow-only activity: discoverable by the per-id probe, never a + // persisted snapshot. + await workspaceService.emitWorkflowRunActivity({ + workspaceId: lateWorkspaceId, + runId: "late-run", + status: "running", + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[lateWorkspaceId]?.activeWorkflowRunCount).toBe(1); + } finally { + supersetSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("suppresses activity emissions for removed workspaces", async () => { // A late in-flight producer completing after removal must not broadcast: // the renderer would re-insert the removed id into its activity map after @@ -3285,6 +3336,18 @@ describe("WorkspaceService activity list scoping", () => { // Clearing (null) emissions stay allowed for removed workspaces. workspaceService.emitWorkspaceActivity(workspaceId, null); expect(events.length).toBe(2); + // A late workflow-run producer can also fire after removal: its cache + // entry turns a null snapshot into a non-null merged payload, which + // must be suppressed exactly like a non-null snapshot emission — the + // tombstone check runs on the merged payload, not the raw snapshot. + await workspaceService.emitWorkflowRunActivity({ + workspaceId, + runId: "late-run", + status: "running", + }); + expect(events.length).toBe(2); + workspaceService.emitWorkspaceActivity(workspaceId, null); + expect(events.length).toBe(2); } finally { await cleanup(); } @@ -3403,13 +3466,13 @@ describe("WorkspaceService activity list scoping", () => { expect(activityList).toEqual({}); // The corrupt bytes were moved aside, not destroyed. expect(await fsPromises.readFile(`${metadataPath}.corrupt`, "utf-8")).toBe(corruptFile); - let mainFileExists = true; - try { - await fsPromises.access(metadataPath); - } catch { - mainFileExists = false; - } - expect(mainFileExists).toBe(false); + // Quarantine must leave a valid EMPTY main file behind (never a + // missing path): readers of a missing-main-plus-sidecar state treat + // it as a retryable mid-quarantine window, not authoritative empty. + expect(JSON.parse(await fsPromises.readFile(metadataPath, "utf-8"))).toEqual({ + version: 1, + workspaces: {}, + }); } finally { await cleanup(); } @@ -3708,6 +3771,51 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("fails open when a legacy metadata.json parses without a usable id", async () => { + // Successful JSON parsing does not establish identity: `{}` (or an + // array) passes the parse but resolves an id-less entry, and the raw + // config has no id to contribute. Strict enumeration must fail closed + // exactly like the unparseable case above, or the prune classifies the + // real stable id's entries as stale and deletes them. + const idlessMetadataFiles = ["{}", "[]"]; + for (const idlessMetadataFile of idlessMetadataFiles) { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("legacy-stable-id", 100); + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + idlessMetadataFile + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.["legacy-stable-id"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("legacy-stable-id")).toBe(true); + } finally { + await cleanup(); + } + } + }); + test("never prunes entries whose config entry is discarded by normalization", async () => { // A parseable config entry that lenient normalization filters out (null // project path): the workspace vanishes from the normalized view — and diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index d044ac48df..2dc79fcfa2 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3903,25 +3903,27 @@ export class WorkspaceService extends EventEmitter { workspaceId: string, snapshot: WorkspaceActivitySnapshot | null ): void { + const activity = this.mergeCurrentActiveBashMonitorCount( + workspaceId, + this.mergeCachedActiveWorkflowRuns( + workspaceId, + this.overlayPendingGoal(workspaceId, snapshot) + ) + ); // A late in-flight producer (e.g. a stream-abort stop-status handler mid // todo read) can complete after removal deleted this workspace's // metadata entry. Its disk write is already blocked by the write // tombstone; suppress the broadcast too, or the renderer re-inserts the // removed id into its activity map after having processed the - // metadata-removal event. Null (clearing) emissions stay allowed. - if (snapshot !== null && this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { + // metadata-removal event. Null (clearing) emissions stay allowed — the + // check runs on the MERGED payload (not the raw snapshot) because the + // workflow/bash-monitor cache overlays above can turn a null snapshot + // into a non-null activity from still-populated caches, which would + // re-insert the deleted id just the same. + if (activity !== null && this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { return; } - this.emit("activity", { - workspaceId, - activity: this.mergeCurrentActiveBashMonitorCount( - workspaceId, - this.mergeCachedActiveWorkflowRuns( - workspaceId, - this.overlayPendingGoal(workspaceId, snapshot) - ) - ), - }); + this.emit("activity", { workspaceId, activity }); } /** @@ -12914,18 +12916,19 @@ export class WorkspaceService extends EventEmitter { if (prefetchedKnownIds != null) { workspaceIds = prefetchedKnownIds; // The prune enumerated config BEFORE the snapshot read above, so a - // workspace registered in between (whose first activity write is - // already in `snapshots`) would be missing here — and an + // workspace registered in between would be missing here — and an // authoritative list omitting it would clear its live-arrived - // renderer state with no retry. Re-admit snapshot ids that a fresh - // raw config view now knows (cheap sync read; on failure the - // prefetched view stands and the miss is a transient one). + // renderer state with no retry. Admit every id a fresh raw config + // view now knows, NOT just ids with a persisted snapshot: a + // concurrently registered workspace with workflow- or bash-monitor- + // only activity has no extensionMetadata entry, and its on-disk + // workflow runs are only discovered by the per-id probe below. + // (Cheap sync read; on failure the prefetched view stands and the + // miss is a transient one.) try { const refreshedConfigIds = this.config.readPersistedWorkspaceIdSuperset(); - for (const workspaceId of snapshots.keys()) { - if (!workspaceIds.has(workspaceId) && refreshedConfigIds.has(workspaceId)) { - workspaceIds.add(workspaceId); - } + for (const workspaceId of refreshedConfigIds) { + workspaceIds.add(workspaceId); } } catch (error) { log.debug("Failed to refresh config ids for first-bootstrap scoping", { error }); From bcd4656f7bbcf6cff975940b65827e5999ae6829 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 14:24:12 +0000 Subject: [PATCH 23/72] review: model the null activity.list failure signal in the ACP session test The merge adopted main's contract where getActivityList logs and returns null instead of rejecting; the harness's rejection injection no longer matches any backend behavior and made listSessions fail wholesale. --- tests/ipc/acp.sessionMethods.test.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/ipc/acp.sessionMethods.test.ts b/tests/ipc/acp.sessionMethods.test.ts index 626f72cef5..6c607da846 100644 --- a/tests/ipc/acp.sessionMethods.test.ts +++ b/tests/ipc/acp.sessionMethods.test.ts @@ -32,8 +32,12 @@ interface HarnessOptions { activeWorkspaces?: WorkspaceInfo[]; archivedWorkspaces?: WorkspaceInfo[]; workspaceActivity?: WorkspaceActivityById; - /** Reject activity.list with this error (simulates unreadable extensionMetadata.json). */ - activityListError?: Error; + /** + * Resolve activity.list with null — the backend's read-failure signal for + * an unreadable extensionMetadata.json (getActivityList never rejects; it + * logs and returns null so callers can tell failure from an idle {}). + */ + activityListUnavailable?: boolean; onChatEvents?: WorkspaceChatMessage[]; onChatStream?: AsyncIterable; requireTrustedProjectForCreate?: boolean; @@ -136,8 +140,8 @@ function createMockServer(options?: HarnessOptions): MockServer { }, activity: { list: async () => { - if (options?.activityListError) { - throw options.activityListError; + if (options?.activityListUnavailable) { + return null; } return workspaceActivity; }, @@ -401,9 +405,10 @@ describe("ACP session list/resume/fork support", () => { }); it("lists sessions when the activity list is unavailable", async () => { - // activity.list rejects when extensionMetadata.json is unreadable so the - // renderer can keep cached state; ACP has none, so session listing must - // degrade to no-activity sorting instead of failing wholesale. + // activity.list resolves null when extensionMetadata.json is unreadable + // so the renderer can keep cached state; ACP has none, so session + // listing must degrade to no-activity sorting instead of failing + // wholesale. const workspace = createWorkspaceInfo({ id: "ws-no-activity", projectPath: "/repo/a", @@ -412,7 +417,7 @@ describe("ACP session list/resume/fork support", () => { }); const harness = createHarness({ activeWorkspaces: [workspace], - activityListError: new Error("extension metadata unreadable"), + activityListUnavailable: true, }); await harness.agent.initialize({ protocolVersion: PROTOCOL_VERSION }); From 373809ea5104a041c721f053f3ed3648c3f35900 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 14:40:00 +0000 Subject: [PATCH 24/72] review: address round-21 findings - strict findWorkspace fails closed when a legacy metadata.json parses without a usable id ({} / []), on both the basename and the authoritative generated-legacy-id lookups, so discardExtensionMetadataEntry cannot tombstone a still-registered workspace whose stable id is unknowable - quarantine recovery is resumable: the post-rename completion (restore healthy sidecar bytes / reset corrupt ones to a valid empty main file) is factored into completeQuarantineRecovery, and a strict read hitting the missing-main-plus-sidecar crash signature resumes it instead of failing on every retry until an unrelated writer saves --- src/node/config.ts | 27 ++ .../services/ExtensionMetadataService.test.ts | 46 ++-- src/node/services/ExtensionMetadataService.ts | 235 +++++++++++------- src/node/services/workspaceService.test.ts | 42 ++++ 4 files changed, 241 insertions(+), 109 deletions(-) diff --git a/src/node/config.ts b/src/node/config.ts index 1a9595f769..99225740f3 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2236,6 +2236,21 @@ export class Config { const data = fs.readFileSync(metadataPath, "utf-8"); const metadata = JSON.parse(data) as WorkspaceMetadata; this.rememberLegacyTaskVariantWorkspace(projectPath, metadata, "metadata"); + // Parseable-but-id-less metadata (e.g. `{}`) leaves this entry's + // identity unknowable: strict callers (the extension-metadata + // discard) must not conclude "not registered" from it, or a live + // workspace whose stable id lived only here gets its activity + // deleted and write-tombstoned. Mirrors the strict enumeration + // guard in getAllWorkspaceMetadata. The catch below rethrows this + // for strict callers and keeps ignoring it for lenient ones. + if ( + options?.throwOnError && + !(typeof metadata.id === "string" && metadata.id.length > 0) + ) { + throw new Error( + `Legacy workspace metadata at ${metadataPath} resolved without a usable id` + ); + } if (metadata.id === workspaceId) { return { workspacePath: workspace.path, @@ -2270,6 +2285,18 @@ export class Config { const legacyData = fs.readFileSync(legacyMetadataPath, "utf-8"); const legacyMetadata = JSON.parse(legacyData) as WorkspaceMetadata; this.rememberLegacyTaskVariantWorkspace(projectPath, legacyMetadata, "metadata"); + // Same unknowable-identity guard as the basename lookup above: + // this is the authoritative file getAllWorkspaceMetadata resolves + // stable ids from, so an id-less parse here must fail closed in + // strict mode rather than fall through to "not registered". + if ( + options?.throwOnError && + !(typeof legacyMetadata.id === "string" && legacyMetadata.id.length > 0) + ) { + throw new Error( + `Legacy workspace metadata at ${legacyMetadataPath} resolved without a usable id` + ); + } if (legacyMetadata.id === workspaceId) { return { workspacePath: workspace.path, diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index cb5dc66532..bca1e3fc71 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -588,30 +588,38 @@ describe("ExtensionMetadataService", () => { expect(await readFile(filePath, "utf-8")).toBe(rawContent); }); - test("a missing main file with the quarantine sidecar present is not an empty state", async () => { - // Mid-quarantine window: the corrupt (or concurrently repaired) bytes - // were renamed to the sidecar and the empty replacement is not written - // yet. Strict readers must get a retryable failure — an authoritative {} - // returned here could not be retracted by the subsequent restore. + test("a strict read resumes a crash-interrupted quarantine of corrupt bytes", async () => { + // Crash window: quarantine renamed main -> sidecar and died before + // writing the empty replacement. The next strict read must finish the + // recovery — never return an authoritative {} while the main path is + // missing, and never keep failing until an unrelated writer saves. await writeFile(`${filePath}.corrupt`, "{not json"); // Main file intentionally absent (beforeEach never creates it). - let strictError: unknown = null; - try { - await service.getAllSnapshots({ throwOnError: true }); - } catch (error) { - strictError = error; - } - expect(strictError).not.toBeNull(); - // The failure is classified transient (errno-carrying), so no quarantine - // cascade: the main path stays absent rather than being reset to empty. + // The moved bytes really were corrupt: recovery resets the main path to + // a valid empty file and keeps the bytes quarantined for inspection. + expect((await service.getAllSnapshots({ throwOnError: true })).size).toBe(0); expect(await readFile(`${filePath}.corrupt`, "utf-8")).toBe("{not json"); + expect(JSON.parse(await readFile(filePath, "utf-8"))).toEqual({ + version: 1, + workspaces: {}, + }); + }); - // Lenient (writer) reads keep self-healing so mutations make progress. - expect((await service.getAllSnapshots()).size).toBe(0); + test("a strict read restores healthy bytes stranded in the sidecar by a crashed quarantine", async () => { + // The crash can also strand a HEALTHY file in the sidecar: a concurrent + // writer repaired the main file right before quarantine's rename moved + // it aside. Recovery must restore it — resetting to empty would discard + // live activity data. + const healthy = { + version: 1, + workspaces: { "ws-1": { recency: 42, streaming: false } }, + }; + await writeFile(`${filePath}.corrupt`, JSON.stringify(healthy)); - // Without the sidecar, a missing main file stays a healthy empty state. - await rm(`${filePath}.corrupt`); - expect((await service.getAllSnapshots({ throwOnError: true })).size).toBe(0); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-1")?.recency).toBe(42); + // Restored to the main path (sidecar consumed by the restore). + expect(JSON.parse(await readFile(filePath, "utf-8"))).toEqual(healthy); }); }); diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index a5ede60d40..5721637021 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -214,9 +214,11 @@ export class ExtensionMetadataService { // replacement — the moved bytes may even be a concurrent writer's // healthy repair awaiting restore. Strict readers must see a // retryable failure (ENOENT carries an errno code, so callers - // classify it as transient), not an authoritative {} that a - // subsequent restore cannot retract. Lenient (writer) paths keep - // self-healing below so mutations always make progress. + // classify it as transient; getAllSnapshots additionally resumes a + // crash-interrupted recovery from this signature), not an + // authoritative {} that a subsequent restore cannot retract. + // Lenient (writer) paths keep self-healing below so mutations + // always make progress. const quarantined = await access(`${this.filePath}.corrupt`).then( () => true, () => false @@ -615,91 +617,136 @@ export class ExtensionMetadataService { } const quarantinePath = `${this.filePath}.corrupt`; await rename(this.filePath, quarantinePath); - // The mutation queue is process-local: another backend's atomic save - // can land a healthy file between the validation above and the rename, - // and the rename would move THAT file aside. Re-validate the bytes that - // actually got moved and undo the move when they turn out healthy. - // Only deterministic parse corruption is the expected quarantine - // outcome here; a transient failure reading the sidecar means the - // moved bytes cannot be verified, so it propagates (retryable) rather - // than reporting a successful reset over possibly-healthy data. - let moved: unknown; - let movedParses = true; + return this.completeQuarantineRecovery(quarantinePath); + }); + } + + /** + * Finish a quarantine whose main file was already moved to the sidecar: + * restore the sidecar to the main path when its bytes turn out healthy, + * otherwise leave the corrupt bytes quarantined and reset the main path to + * a valid empty file. Factored out of quarantineCorruptFile so a recovery + * interrupted by a crash between the rename and this completion can be + * RESUMED on the next strict read (see resumeQuarantineRecovery) instead + * of leaving strict reads failing until an unrelated writer saves. + * Must run inside withSerializedMutation. + */ + private async completeQuarantineRecovery(quarantinePath: string): Promise { + // The mutation queue is process-local: another backend's atomic save + // can land a healthy file between the validation above and the rename, + // and the rename would move THAT file aside. Re-validate the bytes that + // actually got moved and undo the move when they turn out healthy. + // Only deterministic parse corruption is the expected quarantine + // outcome here; a transient failure reading the sidecar means the + // moved bytes cannot be verified, so it propagates (retryable) rather + // than reporting a successful reset over possibly-healthy data. + let moved: unknown; + let movedParses = true; + try { + moved = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; + } catch (readError) { + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + movedParses = false; + } + if (movedParses && ExtensionMetadataService.isValidMetadataFileShape(moved)) { + // Restore without ever overwriting a newer file yet another writer + // may have re-created at the main path: link and COPYFILE_EXCL + // both fail with EEXIST in that case (the healthy leftover sidecar + // is then harmless and bounded by the fixed name). try { - moved = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; - } catch (readError) { - if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { - throw readError; + await link(quarantinePath, this.filePath); + } catch (linkError) { + if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + return false; } - movedParses = false; - } - if (movedParses && ExtensionMetadataService.isValidMetadataFileShape(moved)) { - // Restore without ever overwriting a newer file yet another writer - // may have re-created at the main path: link and COPYFILE_EXCL - // both fail with EEXIST in that case (the healthy leftover sidecar - // is then harmless and bounded by the fixed name). try { - await link(quarantinePath, this.filePath); - } catch (linkError) { - if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + // Filesystems without hard-link support (or EPERM): copy-based + // restore with the same EEXIST no-overwrite guarantee. + await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { return false; } - try { - // Filesystems without hard-link support (or EPERM): copy-based - // restore with the same EEXIST no-overwrite guarantee. - await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); - } catch (copyError) { - if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { - return false; - } - // Restore failed with the main path missing: rethrow so the - // strict reader propagates a retryable failure instead of - // reading ENOENT as an authoritative empty state while the - // healthy bytes sit in the sidecar. + // Restore failed with the main path missing: rethrow so the + // strict reader propagates a retryable failure instead of + // reading ENOENT as an authoritative empty state while the + // healthy bytes sit in the sidecar. + throw copyError; + } + } + await unlink(quarantinePath).catch(() => undefined); + return false; + } + // Replace the quarantined main file with a valid EMPTY file instead of + // leaving the path missing: between the rename above and here, strict + // readers (this or another process) would otherwise observe ENOENT. + // A missing-main window is dangerous because load() treats plain + // ENOENT as a healthy empty state — combined with a concurrent writer + // repairing the file right before the rename, a reader in the gap + // could return an authoritative {} that a later restore cannot + // retract. With a real empty file the corrupt→empty transition is + // atomic (link/COPYFILE_EXCL below never overwrite a file a + // concurrent writer re-created first), and load()'s sidecar check + // turns any remaining missing-main window into a retryable failure + // instead of an empty read. + const emptyTmpPath = `${this.filePath}.empty.tmp`; + await writeFile( + emptyTmpPath, + JSON.stringify({ version: 1, workspaces: {} } satisfies ExtensionMetadataFile, null, 2), + "utf-8" + ); + try { + await link(emptyTmpPath, this.filePath); + } catch (linkError) { + if (!ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + try { + await copyFile(emptyTmpPath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + // Main path still missing: rethrow (retryable) so the caller's + // re-read hits the sidecar guard instead of reading ENOENT. throw copyError; } } - await unlink(quarantinePath).catch(() => undefined); - return false; } - // Replace the quarantined main file with a valid EMPTY file instead of - // leaving the path missing: between the rename above and here, strict - // readers (this or another process) would otherwise observe ENOENT. - // A missing-main window is dangerous because load() treats plain - // ENOENT as a healthy empty state — combined with a concurrent writer - // repairing the file right before the rename, a reader in the gap - // could return an authoritative {} that a later restore cannot - // retract. With a real empty file the corrupt→empty transition is - // atomic (link/COPYFILE_EXCL below never overwrite a file a - // concurrent writer re-created first), and load()'s sidecar check - // turns any remaining missing-main window into a retryable failure - // instead of an empty read. - const emptyTmpPath = `${this.filePath}.empty.tmp`; - await writeFile( - emptyTmpPath, - JSON.stringify({ version: 1, workspaces: {} } satisfies ExtensionMetadataFile, null, 2), - "utf-8" + } + await unlink(emptyTmpPath).catch(() => undefined); + log.error( + `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` + ); + return true; + } + + /** + * Resume a quarantine that a crash interrupted between quarantineCorruptFile's + * rename and its completion: the main file is missing while the sidecar still + * holds the moved bytes. Without this, every strict read rethrows the ENOENT + * (load()'s sidecar guard classifies it retryable) until an unrelated writer + * happens to save — activity hydration would retry forever on an idle + * process. No-op when the main file reappeared or no sidecar exists. + */ + private async resumeQuarantineRecovery(): Promise { + await this.withSerializedMutation(async () => { + // Re-check inside the queue: a concurrent writer save or a sibling + // strict read may already have recovered the main path. + const mainExists = await access(this.filePath).then( + () => true, + () => false ); - try { - await link(emptyTmpPath, this.filePath); - } catch (linkError) { - if (!ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { - try { - await copyFile(emptyTmpPath, this.filePath, constants.COPYFILE_EXCL); - } catch (copyError) { - if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { - // Main path still missing: rethrow (retryable) so the caller's - // re-read hits the sidecar guard instead of reading ENOENT. - throw copyError; - } - } - } + if (mainExists) { + return; } - await unlink(emptyTmpPath).catch(() => undefined); - log.error( - `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` + const quarantinePath = `${this.filePath}.corrupt`; + const sidecarExists = await access(quarantinePath).then( + () => true, + () => false ); - return true; + if (!sidecarExists) { + return; + } + await this.completeQuarantineRecovery(quarantinePath); }); } @@ -714,21 +761,29 @@ export class ExtensionMetadataService { // last-known state and retries), but deterministic corruption would // fail every retry forever on an idle process. Quarantine the corrupt // bytes and re-read: the post-quarantine empty state is authoritative. - if (!ExtensionMetadataService.isDeterministicCorruption(error)) { - throw error; - } - try { - await this.quarantineCorruptFile(); - } catch (quarantineError) { - // ENOENT means the file vanished between validation and rename - // (another process moved it) — the re-read below decides the - // outcome. Anything else (rename denied, sidecar unverifiable, - // restore failed) must stay a retryable failure: an ENOENT re-read - // would masquerade as authoritative empty while the moved bytes may - // hold healthy data. - if (!ExtensionMetadataService.isErrnoCode(quarantineError, "ENOENT")) { - throw quarantineError; + if (ExtensionMetadataService.isDeterministicCorruption(error)) { + try { + await this.quarantineCorruptFile(); + } catch (quarantineError) { + // ENOENT means the file vanished between validation and rename + // (another process moved it) — the re-read below decides the + // outcome. Anything else (rename denied, sidecar unverifiable, + // restore failed) must stay a retryable failure: an ENOENT re-read + // would masquerade as authoritative empty while the moved bytes may + // hold healthy data. + if (!ExtensionMetadataService.isErrnoCode(quarantineError, "ENOENT")) { + throw quarantineError; + } } + } else if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + // load() only propagates ENOENT while the quarantine sidecar exists: + // a crash between quarantine's rename and its completion left the + // recovery half-done, and nothing else finishes it (lenient writer + // reads self-heal in memory without saving). Resume it here so + // strict reads stop failing on every retry across restarts. + await this.resumeQuarantineRecovery(); + } else { + throw error; } data = await this.load(options); } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 6319d104cc..f02e79635e 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3433,6 +3433,48 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("discardExtensionMetadataEntry keeps entries when legacy metadata parses without an id", async () => { + // Same identity-unknowable contract as the unparseable case: a legacy + // metadata.json that parses as `{}` carries no id, so the strict + // findWorkspace lookup must fail closed rather than fall through to + // "not registered" — the entry under the real (unknowable) stable id + // would otherwise be deleted and write-tombstoned while its workspace + // remains registered. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "legacy-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile(path.join(legacySessionDir, "metadata.json"), "{}"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(stableId, 100); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + await workspaceService.discardExtensionMetadataEntry(stableId); + + expect((await extensionMetadata.getAllSnapshots()).has(stableId)).toBe(true); + expect(extensionMetadata.isWorkspaceDeleted(stableId)).toBe(false); + } finally { + await cleanup(); + } + }); + test("getActivityList quarantines a deterministically corrupt metadata file", async () => { // Parse/structure corruption fails identically on every retry, so a // strict read that only rethrows would leave activity hydration broken From 10c950f0fe633bbb7affe2fbc92fdb5d46222963 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 15:57:23 +0000 Subject: [PATCH 25/72] review: close the post-recovery ENOENT TOCTOU in metadata load An ENOENT read can race another process's COMPLETED quarantine recovery: by the time the sidecar probe runs, the healthy main file is restored and the sidecar consumed, so the absent sidecar proves nothing about the stale failure. load() now re-reads the main path (bounded attempts) and only treats a still-missing file as the healthy empty state. --- .../services/ExtensionMetadataService.test.ts | 28 +++++ src/node/services/ExtensionMetadataService.ts | 108 +++++++++++------- 2 files changed, 93 insertions(+), 43 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index bca1e3fc71..9282f06f0a 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -606,6 +606,34 @@ describe("ExtensionMetadataService", () => { }); }); + test("an ENOENT read races a completed recovery in another process and re-reads", async () => { + // Post-recovery TOCTOU: our read hits ENOENT while another process holds + // the file mid-quarantine, and by the time we probe for the sidecar that + // process has already restored the healthy main file AND consumed the + // sidecar. The absent sidecar proves nothing about the stale ENOENT — + // load must re-read the main path instead of returning authoritative {}. + const healthy = { + version: 1, + workspaces: { "ws-1": { recency: 42, streaming: false } }, + }; + // Main file intentionally absent at first read (beforeEach never creates + // it). Simulate the other process's completed recovery at probe time. + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(service); + internals.probeQuarantineSidecar = async () => { + // The concurrent process restored the healthy file and unlinked the + // sidecar before our probe ran. + await writeFile(filePath, JSON.stringify(healthy)); + internals.probeQuarantineSidecar = originalProbe; + return false; + }; + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-1")?.recency).toBe(42); + }); + test("a strict read restores healthy bytes stranded in the sidecar by a crashed quarantine", async () => { // The crash can also strand a HEALTHY file in the sidecar: a concurrent // writer repaired the main file right before quarantine's rename moved diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 5721637021..b21964d173 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -186,54 +186,76 @@ export class ExtensionMetadataService { } } + /** + * Seam for load()'s missing-main handling (and deterministic TOCTOU tests): + * reports whether the quarantine sidecar currently exists. + */ + private probeQuarantineSidecar(): Promise { + return access(`${this.filePath}.corrupt`).then( + () => true, + () => false + ); + } + private async load(options?: { throwOnError?: boolean }): Promise { - try { - const content = await readFile(this.filePath, "utf-8"); - const parsed = JSON.parse(content) as ExtensionMetadataFile; - - // Validate structure, including the workspaces container: a parseable - // file with e.g. an array or primitive `workspaces` would otherwise - // enumerate as zero entries and masquerade as an authoritative empty - // state in strict reads. - if (!ExtensionMetadataService.isValidMetadataFileShape(parsed)) { - throw new Error("Invalid extension metadata file structure"); - } + // Bounded because the ENOENT branch below re-reads: each retry only + // happens after a fresh ENOENT with no sidecar, so the loop converges. + const MAX_READ_ATTEMPTS = 3; + let lastError: unknown; + for (let attempt = 1; attempt <= MAX_READ_ATTEMPTS; attempt++) { + try { + const content = await readFile(this.filePath, "utf-8"); + const parsed = JSON.parse(content) as ExtensionMetadataFile; + + // Validate structure, including the workspaces container: a parseable + // file with e.g. an array or primitive `workspaces` would otherwise + // enumerate as zero entries and masquerade as an authoritative empty + // state in strict reads. + if (!ExtensionMetadataService.isValidMetadataFileShape(parsed)) { + throw new Error("Invalid extension metadata file structure"); + } - return parsed; - } catch (error) { - // Only a genuinely missing file is a healthy empty state. Other read - // failures (EACCES/ENOTDIR/EIO, parse or structure errors) must not - // masquerade as one: throwOnError lets read paths distinguish them - // from an authoritative empty state, while the default self-heals so - // writers can always make progress. - if (typeof error === "object" && error != null && "code" in error) { - if (error.code === "ENOENT") { - // A missing main file WITH the quarantine sidecar present is not a - // healthy empty state: it is the (rare) window where quarantine - // moved the file aside and has not yet written the empty - // replacement — the moved bytes may even be a concurrent writer's - // healthy repair awaiting restore. Strict readers must see a - // retryable failure (ENOENT carries an errno code, so callers - // classify it as transient; getAllSnapshots additionally resumes a - // crash-interrupted recovery from this signature), not an - // authoritative {} that a subsequent restore cannot retract. - // Lenient (writer) paths keep self-healing below so mutations - // always make progress. - const quarantined = await access(`${this.filePath}.corrupt`).then( - () => true, - () => false - ); - if (!quarantined) { - return { version: 1, workspaces: {} }; - } + return parsed; + } catch (error) { + // Only a genuinely missing file is a healthy empty state. Other read + // failures (EACCES/ENOTDIR/EIO, parse or structure errors) must not + // masquerade as one: throwOnError lets read paths distinguish them + // from an authoritative empty state, while the default self-heals so + // writers can always make progress. + lastError = error; + if (!ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + break; + } + // A missing main file WITH the quarantine sidecar present is not a + // healthy empty state: it is the (rare) window where quarantine + // moved the file aside and has not yet written the empty + // replacement — the moved bytes may even be a concurrent writer's + // healthy repair awaiting restore. Strict readers must see a + // retryable failure (ENOENT carries an errno code, so callers + // classify it as transient; getAllSnapshots additionally resumes a + // crash-interrupted recovery from this signature), not an + // authoritative {} that a subsequent restore cannot retract. + // Lenient (writer) paths keep self-healing below so mutations + // always make progress. + if (await this.probeQuarantineSidecar()) { + break; + } + // No sidecar either. With multiple instances the failed read may + // have raced another process's COMPLETED recovery — main restored + // (or reset to empty) and the sidecar already consumed — so the + // absent sidecar proves nothing about the earlier ENOENT. Re-read + // the main path instead of trusting the stale failure; only a file + // still missing on the final attempt is a healthy empty state. + if (attempt === MAX_READ_ATTEMPTS) { + return { version: 1, workspaces: {} }; } } - if (options?.throwOnError) { - throw error; - } - log.error("Failed to load metadata:", error); - return { version: 1, workspaces: {} }; } + if (options?.throwOnError) { + throw lastError; + } + log.error("Failed to load metadata:", lastError); + return { version: 1, workspaces: {} }; } private async save(data: ExtensionMetadataFile): Promise { From ad6e8ca1af533251a0b2e77312c038baa1015f2e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 16:07:55 +0000 Subject: [PATCH 26/72] review: lenient writers complete a crash-interrupted quarantine inline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lenient mutation reading during another process's quarantine window (main missing, sidecar present) previously self-healed to {} and saved a partial file, which the no-overwrite restore then refused to replace — stranding every other workspace's metadata in the sidecar. Lenient loads now complete the recovery inline (unqueued deliberately: the promise-chain mutation queue is not reentrant; every recovery step is idempotent and no-overwrite) and re-read, and the quarantine-window failure never resolves as an empty file even on attempt exhaustion. --- .../services/ExtensionMetadataService.test.ts | 26 ++++++++++++ src/node/services/ExtensionMetadataService.ts | 41 ++++++++++++++----- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 9282f06f0a..0d6b5340e5 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -634,6 +634,32 @@ describe("ExtensionMetadataService", () => { expect(snapshots.get("ws-1")?.recency).toBe(42); }); + test("a lenient writer completes a crash-interrupted quarantine instead of clobbering it", async () => { + // Same crash window as above, but hit by a normal (lenient) mutation. + // Treating it as an empty file would save a partial one-entry file at + // the main path — and the pending restore (deliberately no-overwrite) + // would then strand every other workspace's metadata in the sidecar. + // The writer must instead complete the recovery inline and apply its + // mutation on top of the restored data. + const healthy = { + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }; + await writeFile(`${filePath}.corrupt`, JSON.stringify(healthy)); + // Main file intentionally absent (beforeEach never creates it). + + await service.updateRecency("ws-new", 100); + + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(Object.keys(persisted.workspaces).sort()).toEqual(["ws-new", "ws-other"]); + // Sidecar consumed by the restore. + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("a strict read restores healthy bytes stranded in the sidecar by a crashed quarantine", async () => { // The crash can also strand a HEALTHY file in the sidecar: a concurrent // writer repaired the main file right before quarantine's rename moved diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index b21964d173..40c1f2fcdc 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -199,9 +199,13 @@ export class ExtensionMetadataService { private async load(options?: { throwOnError?: boolean }): Promise { // Bounded because the ENOENT branch below re-reads: each retry only - // happens after a fresh ENOENT with no sidecar, so the loop converges. + // happens after a fresh ENOENT, so the loop converges. const MAX_READ_ATTEMPTS = 3; let lastError: unknown; + // True when the final failed attempt hit the mid-quarantine window + // (main missing, sidecar present). That failure must never resolve as + // an empty file, even leniently — see the sidecar branch below. + let blockedByQuarantineWindow = false; for (let attempt = 1; attempt <= MAX_READ_ATTEMPTS; attempt++) { try { const content = await readFile(this.filePath, "utf-8"); @@ -223,6 +227,7 @@ export class ExtensionMetadataService { // from an authoritative empty state, while the default self-heals so // writers can always make progress. lastError = error; + blockedByQuarantineWindow = false; if (!ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { break; } @@ -230,15 +235,31 @@ export class ExtensionMetadataService { // healthy empty state: it is the (rare) window where quarantine // moved the file aside and has not yet written the empty // replacement — the moved bytes may even be a concurrent writer's - // healthy repair awaiting restore. Strict readers must see a - // retryable failure (ENOENT carries an errno code, so callers - // classify it as transient; getAllSnapshots additionally resumes a - // crash-interrupted recovery from this signature), not an - // authoritative {} that a subsequent restore cannot retract. - // Lenient (writer) paths keep self-healing below so mutations - // always make progress. + // healthy repair awaiting restore. Resolving this window as an + // empty file is never safe, lenient or strict: a lenient WRITER + // would mutate {} and save, recreating the main path — and the + // pending restore (deliberately no-overwrite) would then strand + // every other workspace's metadata in the sidecar for good. if (await this.probeQuarantineSidecar()) { - break; + blockedByQuarantineWindow = true; + if (options?.throwOnError) { + // Strict readers surface a retryable failure (ENOENT carries an + // errno code, so callers classify it as transient) and + // getAllSnapshots resumes the recovery through the mutation + // queue — never an authoritative {} that a subsequent restore + // cannot retract. + break; + } + // Lenient callers complete the recovery INLINE and re-read. + // Inline (unqueued) is deliberate: most lenient loads run inside + // withSerializedMutation, whose promise-chain queue is not + // reentrant — enqueueing resumeQuarantineRecovery here would + // deadlock. Racing a concurrent recovery is safe because every + // step is no-overwrite (link/COPYFILE_EXCL) and idempotent. A + // recovery failure propagates: failing one mutation in this + // pathological window beats destroying the sidecar's data. + await this.completeQuarantineRecovery(`${this.filePath}.corrupt`); + continue; } // No sidecar either. With multiple instances the failed read may // have raced another process's COMPLETED recovery — main restored @@ -251,7 +272,7 @@ export class ExtensionMetadataService { } } } - if (options?.throwOnError) { + if (options?.throwOnError || blockedByQuarantineWindow) { throw lastError; } log.error("Failed to load metadata:", lastError); From dd32db79e2f8ee3cb2b7b038111ce43dc8163193 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 16:18:55 +0000 Subject: [PATCH 27/72] review: propagate non-ENOENT sidecar probe failures An EACCES/EIO probe failure was reported as sidecar-absent, letting a missing-main read resolve as an authoritative empty file while recoverable metadata may sit in the unprobeable sidecar. Only a verified ENOENT counts as absence; other probe errors propagate so the read stays retryable. --- .../services/ExtensionMetadataService.test.ts | 23 +++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 14 ++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 0d6b5340e5..1ec524096f 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -634,6 +634,29 @@ describe("ExtensionMetadataService", () => { expect(snapshots.get("ws-1")?.recency).toBe(42); }); + test("a failing sidecar probe keeps a missing-main read retryable", async () => { + // Main file absent and the sidecar probe fails with EACCES: the + // sidecar's existence is unknowable, so the read must not resolve as an + // authoritative empty file — recoverable metadata may sit in the + // unprobeable sidecar. Only a verified ENOENT counts as absence. + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + internals.probeQuarantineSidecar = () => { + const error = new Error("permission denied") as NodeJS.ErrnoException; + error.code = "EACCES"; + return Promise.reject(error); + }; + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + }); + test("a lenient writer completes a crash-interrupted quarantine instead of clobbering it", async () => { // Same crash window as above, but hit by a normal (lenient) mutation. // Treating it as an empty file would save a partial one-entry file at diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 40c1f2fcdc..aea684e7b0 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -193,7 +193,19 @@ export class ExtensionMetadataService { private probeQuarantineSidecar(): Promise { return access(`${this.filePath}.corrupt`).then( () => true, - () => false + (error: unknown) => { + if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + return false; + } + // EACCES/EIO/...: the sidecar's existence is unknowable, and only a + // verified absence may let an ENOENT main read resolve as a healthy + // empty file — recoverable metadata may sit in the unprobeable + // sidecar. Propagate so the read stays a retryable failure for + // strict readers and fails the mutation for lenient writers (same + // tradeoff as the in-window recovery: failing one operation beats + // clobbering the sidecar's data with a partial save). + throw error; + } ); } From 06cd1aecc6669f64668538123e2e06d570d0e258 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 17:39:58 +0000 Subject: [PATCH 28/72] review: revalidate raw-invisible legacy ids through the authoritative identity path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Legacy stable ids resolved from session metadata.json can never appear in the raw config-id baseline, so the raw-superset removal comparison is blind to their cross-process removal — a snapshotless legacy entry removed mid-list would ride the delayed authoritative response back into the renderer. Retained entries absent from the raw baseline are now rechecked via strict findWorkspace: a verified 'not registered' drops the entry, an unknowable identity conservatively retains it, and raw-visible ids skip the lookup entirely. --- src/node/services/workspaceService.test.ts | 94 +++++++++++++++++++++- src/node/services/workspaceService.ts | 35 +++++++- 2 files changed, 123 insertions(+), 6 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index f02e79635e..520d2f10b2 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3255,6 +3255,18 @@ describe("WorkspaceService activity list scoping", () => { const { config, historyService, cleanup } = await createTestHistoryService(); try { const lateWorkspaceId = "late-registered-workspace"; + // Registered for real (the mid-list registration lands in config.json + // in the modeled race); the spies below hide it from the baseline and + // prune reads so only the refresh discovers it — the authoritative + // removal recheck must then still find it registered. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: lateWorkspaceId, + name: lateWorkspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); const extensionMetadata = new ExtensionMetadataService( path.join(config.rootDir, "extensionMetadata.json") ); @@ -3267,12 +3279,21 @@ describe("WorkspaceService activity list scoping", () => { // Calls 1 (pre-await baseline) and 2 (prune enumeration) see the // pre-registration config; the refresh and the post-await // revalidation see the concurrently registered workspace. - if (supersetCalls >= 3) { - ids.add(lateWorkspaceId); + if (supersetCalls <= 2) { + ids.delete(lateWorkspaceId); } return ids; } ); + const realMetadata = config.getAllWorkspaceMetadata.bind(config); + const metadataSpy = spyOn(config, "getAllWorkspaceMetadata").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + // The prune's authoritative enumeration also predates the + // registration in the modeled race. + const all = await realMetadata(options); + return all.filter((metadata) => metadata.id !== lateWorkspaceId); + } + ); try { const workspaceService = createWorkspaceServiceForTest({ config, @@ -3291,6 +3312,7 @@ describe("WorkspaceService activity list scoping", () => { expect(activityList?.[lateWorkspaceId]?.activeWorkflowRunCount).toBe(1); } finally { supersetSpy.mockRestore(); + metadataSpy.mockRestore(); } } finally { await cleanup(); @@ -3475,6 +3497,74 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList drops snapshotless legacy entries removed mid-list", async () => { + // A legacy id-less config entry's stable id is resolved authoritatively + // during enumeration and can never appear in the raw config-id baseline, + // so the raw-superset removal comparison is blind to it. If another + // backend removes the workspace while the per-id reads run, a + // snapshotless (workflow-only) entry has no metadata-file revalidation + // to catch it either — the authoritative findWorkspace recheck must + // drop it instead of reinserting the removed workspace. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "legacy-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Workflow-only activity: no persisted snapshot. + await workspaceService.emitWorkflowRunActivity({ + workspaceId: stableId, + runId: "legacy-run", + status: "running", + }); + // Simulate the cross-process removal between the entry computation and + // the revalidation phase: the fresh metadata re-read is the first + // revalidation step, so rewriting config.json there lands mid-list. + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile(configPath, JSON.stringify({ projects: [] })); + } + return realGetAllSnapshots(options); + } + ); + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[stableId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("getActivityList quarantines a deterministically corrupt metadata file", async () => { // Parse/structure corruption fails identically on every retry, so a // strict read that only rethrows would leave activity hydration broken diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2dc79fcfa2..c29b7ee2f2 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13034,14 +13034,38 @@ export class WorkspaceService extends EventEmitter { // in config before the awaits and is gone from the fresh raw view was // verifiably deregistered. Ids the raw scan cannot see (legacy entries // whose stable id lives in a session metadata.json, in-memory migration - // ids) never appear in either view and are conservatively retained — - // dropping them on a cheap fresh view would misclassify identity-lookup - // gaps as removals. Skipped when either superset read fails. + // ids) never appear in either view — dropping them on a cheap fresh + // view would misclassify identity-lookup gaps as removals, so they are + // revalidated through the authoritative identity path below instead. + // Skipped when either superset read fails. const isRemovedFromConfig = (workspaceId: string): boolean => initialConfigIds != null && freshConfigIds != null && initialConfigIds.has(workspaceId) && !freshConfigIds.has(workspaceId); + // Raw-invisible ids (legacy stable ids resolved from session + // metadata.json during enumeration) need their own removal + // revalidation: the raw baseline can never contain them, so the + // comparison above is blind to their cross-process removal — a + // snapshotless (workflow/bash-monitor-only) legacy entry removed + // mid-list would otherwise ride the delayed authoritative response + // back into the renderer. Revalidate through the same authoritative + // lookup that produced the id: a verified "not registered" drops the + // entry, while an unknowable identity (unreadable/id-less + // metadata.json throws in strict mode) conservatively retains it. + // Cost note: ids present in the raw baseline skip this entirely, so + // modern deployments (every workspace id persisted in config) never + // pay the per-id lookup. + const isRemovedPerAuthoritativeIdentity = (workspaceId: string): boolean => { + if (initialConfigIds == null || initialConfigIds.has(workspaceId)) { + return false; + } + try { + return this.config.findWorkspace(workspaceId, { throwOnError: true }) == null; + } catch { + return false; + } + }; return Object.fromEntries( entries.filter( (entry): entry is readonly [string, WorkspaceActivitySnapshot] => @@ -13062,7 +13086,10 @@ export class WorkspaceService extends EventEmitter { ) && // Deregistered from the shared config mid-list — also covers // workflow/bash-monitor-only entries with no persisted snapshot. - !isRemovedFromConfig(entry[0]) + !isRemovedFromConfig(entry[0]) && + // Raw-invisible (legacy stable) ids: authoritative-lookup + // counterpart of the raw-superset comparison above. + !isRemovedPerAuthoritativeIdentity(entry[0]) ) ); } catch (error) { From 1f5466ed9770a759ece0ba122e471893669f250c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 17:55:28 +0000 Subject: [PATCH 29/72] review: address round-26 findings - replace the per-id findWorkspace removal recheck with ONE strict getAllWorkspaceMetadata enumeration, computed only when a retained entry is missing from the raw baseline: per-id lookups re-read and scan the whole config per entry (O(n^2) on legacy-heavy first bootstraps), recreating the stall this PR removes; an unknowable identity anywhere skips the recheck and conservatively retains raw-invisible ids - clear process-local write tombstones for ids observed registered in fresh config-derived evidence (raw superset / authoritative enumeration): a downgraded concurrent backend can legitimately re-register a deterministic legacy id this process pruned, and the stale tombstone would otherwise suppress the revived workspace's writes and filter it from activity lists until restart --- src/node/services/ExtensionMetadataService.ts | 19 ++++++ src/node/services/workspaceService.test.ts | 62 ++++++++++++++++++- src/node/services/workspaceService.ts | 62 ++++++++++++++----- 3 files changed, 125 insertions(+), 18 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index aea684e7b0..2a07703e24 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -511,6 +511,25 @@ export class ExtensionMetadataService { return this.deletedWorkspaceIds.has(workspaceId); } + /** + * Drop write tombstones for ids a fresh config view proves are registered. + * Tombstones are process-local removal knowledge and the shared config is + * the authority: with XUM_ALLOW_MULTIPLE_INSTANCES a downgraded concurrent + * backend can legitimately re-register a deterministic legacy id this + * process pruned earlier, and without this hook the stale tombstone would + * suppress every one of the revived workspace's metadata writes (and + * filter it from activity lists) until restart. Called from the activity + * bootstrap with fresh config-derived id sets only — never with snapshot + * or in-memory cache keys, which do not prove registration. + */ + clearTombstonesForRegisteredIds(registeredIds: ReadonlySet): void { + for (const workspaceId of this.deletedWorkspaceIds) { + if (registeredIds.has(workspaceId)) { + this.deletedWorkspaceIds.delete(workspaceId); + } + } + } + /** * Delete metadata for a workspace. * Call this when a workspace is deleted. diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 520d2f10b2..0c16ccf9c0 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -142,6 +142,7 @@ const mockInitStateManager: Partial = { }; const mockExtensionMetadataService: Partial = { isWorkspaceDeleted: mock(() => false), + clearTombstonesForRegisteredIds: mock(() => undefined), setStreaming: mock(() => Promise.resolve({ recency: Date.now(), @@ -3286,12 +3287,18 @@ describe("WorkspaceService activity list scoping", () => { } ); const realMetadata = config.getAllWorkspaceMetadata.bind(config); + let metadataCalls = 0; const metadataSpy = spyOn(config, "getAllWorkspaceMetadata").mockImplementation( async (options?: { throwOnError?: boolean }) => { - // The prune's authoritative enumeration also predates the - // registration in the modeled race. + // Only the prune's enumeration (first call) predates the + // registration in the modeled race; the revalidation's fresh + // authoritative enumeration sees the registered workspace. + metadataCalls += 1; const all = await realMetadata(options); - return all.filter((metadata) => metadata.id !== lateWorkspaceId); + if (metadataCalls === 1) { + return all.filter((metadata) => metadata.id !== lateWorkspaceId); + } + return all; } ); try { @@ -3375,6 +3382,55 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("a re-registered id sheds its tombstone on the next activity list", async () => { + // Tombstones are process-local removal knowledge; the shared config is + // the authority. A downgraded concurrent backend can legitimately + // re-register a deterministic legacy id this process pruned — the next + // activity list observes the id in fresh config evidence and must lift + // the write suppression instead of muting the revived workspace until + // restart. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "revived-legacy-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const workspaceEntry = { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" as const }, + }; + await config.addWorkspace(projectPath, workspaceEntry); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Removal flow: deregister, then discard (delete + tombstone). + await config.removeWorkspace(workspaceId); + await workspaceService.discardExtensionMetadataEntry(workspaceId); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(true); + // Writes are suppressed while tombstoned. + await extensionMetadata.updateRecency(workspaceId, 200); + expect((await extensionMetadata.getAllSnapshots()).has(workspaceId)).toBe(false); + + // The "other backend" re-registers the same id in the shared config. + await config.addWorkspace(projectPath, workspaceEntry); + + await workspaceService.getActivityList(); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + // Writes persist again after the revival. + await extensionMetadata.updateRecency(workspaceId, 300); + expect((await extensionMetadata.getAllSnapshots()).get(workspaceId)?.recency).toBe(300); + } finally { + await cleanup(); + } + }); + test("discardExtensionMetadataEntry keeps the entry when the workspace is still persisted", async () => { // saveConfig swallows write failures, so config.removeWorkspace can // resolve while the workspace is still persisted in config.json. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c29b7ee2f2..2195546a0f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13049,23 +13049,55 @@ export class WorkspaceService extends EventEmitter { // comparison above is blind to their cross-process removal — a // snapshotless (workflow/bash-monitor-only) legacy entry removed // mid-list would otherwise ride the delayed authoritative response - // back into the renderer. Revalidate through the same authoritative - // lookup that produced the id: a verified "not registered" drops the - // entry, while an unknowable identity (unreadable/id-less - // metadata.json throws in strict mode) conservatively retains it. - // Cost note: ids present in the raw baseline skip this entirely, so - // modern deployments (every workspace id persisted in config) never - // pay the per-id lookup. - const isRemovedPerAuthoritativeIdentity = (workspaceId: string): boolean => { - if (initialConfigIds == null || initialConfigIds.has(workspaceId)) { - return false; - } + // back into the renderer. Revalidate against ONE fresh authoritative + // enumeration (per-id findWorkspace lookups would re-read and scan the + // whole config per entry — O(n²) on legacy-heavy first bootstraps, + // recreating the very stall this scoping removes): a verified "not + // registered" drops the entry, while an unknowable identity anywhere + // (unreadable/id-less metadata.json throws in strict mode) skips the + // recheck and conservatively retains every raw-invisible id. Computed + // only when a retained entry is actually missing from the raw + // baseline, so modern deployments (every workspace id persisted in + // config) never pay the extra walk. + let authoritativeIds: ReadonlySet | null = null; + if ( + initialConfigIds != null && + entries.some((entry) => entry != null && !initialConfigIds.has(entry[0])) + ) { try { - return this.config.findWorkspace(workspaceId, { throwOnError: true }) == null; - } catch { - return false; + authoritativeIds = new Set( + (await this.config.getAllWorkspaceMetadata({ throwOnError: true })).map( + (metadata) => metadata.id + ) + ); + } catch (error) { + log.debug("Failed to enumerate authoritative ids for removal revalidation", { error }); + authoritativeIds = null; } - }; + } + const isRemovedPerAuthoritativeIdentity = (workspaceId: string): boolean => + initialConfigIds != null && + !initialConfigIds.has(workspaceId) && + authoritativeIds != null && + !authoritativeIds.has(workspaceId); + // Tombstones are process-local removal knowledge; the shared config is + // the authority. A downgraded concurrent backend can legitimately + // re-register a deterministic legacy id this process pruned earlier — + // observing the id in a FRESH config-derived view (raw superset or the + // strict authoritative enumeration above; never snapshot/cache keys, + // which do not prove registration) makes the tombstone stale, so its + // write suppression and list filtering must end. Cleared before the + // revalidation filter below so a re-registered id's entry survives. + if (freshConfigIds != null || authoritativeIds != null) { + const registeredIds = new Set(); + for (const workspaceId of freshConfigIds ?? []) { + registeredIds.add(workspaceId); + } + for (const workspaceId of authoritativeIds ?? []) { + registeredIds.add(workspaceId); + } + this.extensionMetadata.clearTombstonesForRegisteredIds(registeredIds); + } return Object.fromEntries( entries.filter( (entry): entry is readonly [string, WorkspaceActivitySnapshot] => From 3afe226733c5f77335caa281dc4155b6176206a6 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 18:13:02 +0000 Subject: [PATCH 30/72] review: address round-27 findings - strict config validation rejects array-valued project configs (arrays pass typeof object; lenient normalization silently converts them to a project with no workspaces, misclassifying every workspace as removed) - pruneMissingWorkspaces rechecks known ids against a second fresh enumeration after the fresh file load: an id re-registered (with new activity) by a downgraded backend mid-prune is spared and its write tombstone lifted, instead of being deleted on the stale classification - a syntactically valid metadata file with version !== 1 is treated as an unsupported newer schema: propagated (retryable) in BOTH read modes, never quarantined/reset and never self-healed to {} by lenient writers, so a downgrade round-trip cannot destroy newer activity state --- src/node/config.ts | 10 ++- .../services/ExtensionMetadataService.test.ts | 76 +++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 60 ++++++++++++++- src/node/services/workspaceService.test.ts | 12 ++- 4 files changed, 154 insertions(+), 4 deletions(-) diff --git a/src/node/config.ts b/src/node/config.ts index 99225740f3..f5139c5c20 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1334,7 +1334,15 @@ export class Config { throw new Error("Config projects entries must be [path, config] pairs"); } const projectConfig: unknown = pair[1]; - if (projectConfig === null || typeof projectConfig !== "object") { + // Arrays pass typeof "object": lenient normalization would turn + // an array-valued project config into a project with no + // workspaces, and destructive strict callers would then classify + // every one of its workspaces as removed. + if ( + projectConfig === null || + typeof projectConfig !== "object" || + Array.isArray(projectConfig) + ) { throw new Error("Config project entries must be objects"); } const workspaces = (projectConfig as { workspaces?: unknown }).workspaces; diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 1ec524096f..39834697c1 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -588,6 +588,82 @@ describe("ExtensionMetadataService", () => { expect(await readFile(filePath, "utf-8")).toBe(rawContent); }); + test("pruneMissingWorkspaces spares ids re-registered mid-prune", async () => { + // With multiple instances, a downgraded backend can re-register a + // deterministic legacy id (and write its activity) between the prune's + // enumeration and its deletion pass. The recheck against a second fresh + // enumeration must spare that entry and lift its write tombstone — + // deleting on the stale classification would destroy data a later + // tombstone-clear cannot restore. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "known-workspace": { recency: 100, streaming: false }, + "revived-workspace": { recency: 200, streaming: false }, + }, + }) + ); + let fetches = 0; + const prunedCount = await service.pruneMissingWorkspaces(() => { + fetches += 1; + // First enumeration: revived-workspace looks stale. Recheck: it was + // re-registered concurrently. + return Promise.resolve( + fetches === 1 + ? new Set(["known-workspace"]) + : new Set(["known-workspace", "revived-workspace"]) + ); + }); + + expect(prunedCount).toBe(0); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(Object.keys(persisted.workspaces).sort()).toEqual([ + "known-workspace", + "revived-workspace", + ]); + // Tombstone lifted: subsequent writes for the revived id persist. + expect(service.isWorkspaceDeleted("revived-workspace")).toBe(false); + }); + + test("a newer build's metadata version is unsupported, never quarantined or self-healed", async () => { + // Downgrade safety: a syntactically valid file with version !== 1 was + // written by a newer schema. Quarantining/resetting it (or a lenient + // writer self-healing to {} and saving version-1 bytes over it) would + // make the upgrade back find an empty canonical file and lose all + // activity state. Both read modes must propagate and leave the bytes + // untouched. + const newerFile = JSON.stringify({ version: 2, workspaces: {}, futureField: true }); + await writeFile(filePath, newerFile); + + // Strict read: retryable failure, no quarantine. + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + + // Lenient write: the mutation fails instead of clobbering the file. + let writeError: unknown = null; + try { + await service.updateRecency("ws-1", 123); + } catch (error) { + writeError = error; + } + expect(writeError).not.toBeNull(); + + // Bytes untouched, no sidecar created. + expect(await readFile(filePath, "utf-8")).toBe(newerFile); + const sidecarExists = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => true, + () => false + ); + expect(sidecarExists).toBe(false); + }); + test("a strict read resumes a crash-interrupted quarantine of corrupt bytes", async () => { // Crash window: quarantine renamed main -> sidecar and died before // writing the empty replacement. The next strict read must finish the diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 2a07703e24..b67ea7464e 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -16,6 +16,14 @@ import type { WorkspaceActivitySnapshot } from "@/common/types/workspace"; import type { GoalSnapshot } from "@/common/types/goal"; import { log } from "@/node/services/log"; +/** + * Marker code for "file written by a newer schema version" load failures. + * Shaped like an errno code so the corruption/quarantine classification + * (isDeterministicCorruption: errors WITH a code are not quarantinable) + * treats downgrade encounters as retryable, never as resettable corruption. + */ +const UNSUPPORTED_METADATA_VERSION_CODE = "XUM_UNSUPPORTED_METADATA_VERSION"; + /** * Stateless service for managing workspace metadata used by VS Code extension integration. * @@ -223,6 +231,16 @@ export class ExtensionMetadataService { const content = await readFile(this.filePath, "utf-8"); const parsed = JSON.parse(content) as ExtensionMetadataFile; + // A syntactically valid file whose version is not 1 was written by a + // build with a newer schema — NOT corruption. It must never be + // quarantined/reset (upgrading back would find the canonical file + // empty and lose all activity state) and never self-healed to {} by + // a lenient writer (saving version-1 bytes over it is the same data + // loss). Both read modes propagate; see the catch below. + if (ExtensionMetadataService.isUnsupportedVersion(parsed)) { + throw ExtensionMetadataService.unsupportedVersionError(); + } + // Validate structure, including the workspaces container: a parseable // file with e.g. an array or primitive `workspaces` would otherwise // enumerate as zero entries and masquerade as an authoritative empty @@ -284,7 +302,11 @@ export class ExtensionMetadataService { } } } - if (options?.throwOnError || blockedByQuarantineWindow) { + if ( + options?.throwOnError || + blockedByQuarantineWindow || + ExtensionMetadataService.isErrnoCode(lastError, UNSUPPORTED_METADATA_VERSION_CODE) + ) { throw lastError; } log.error("Failed to load metadata:", lastError); @@ -604,8 +626,21 @@ export class ExtensionMetadataService { } // Deletion-only merge against a fresh snapshot (see doc comment above). const fresh = await this.load(); + // Re-fetch the known ids AFTER the fresh load: with multiple instances + // a downgraded backend can re-register a deterministic legacy id (and + // write new activity for it) between the enumeration above and here. + // Deleting on the stale classification would destroy that new entry's + // recency/goal/status — clearing the tombstone later cannot restore + // data. A re-registered id is dropped from the deletion set and its + // write tombstone lifted. If the recheck fails, abort the prune (throw + // to the caller's catch) rather than deleting on stale knowledge. + const recheckedKnownIds = await getKnownWorkspaceIds(); let prunedCount = 0; for (const workspaceId of staleWorkspaceIds) { + if (recheckedKnownIds.has(workspaceId)) { + this.deletedWorkspaceIds.delete(workspaceId); + continue; + } if (workspaceId in fresh.workspaces) { delete fresh.workspaces[workspaceId]; prunedCount++; @@ -657,6 +692,29 @@ export class ExtensionMetadataService { return typeof error === "object" && error != null && "code" in error && error.code === code; } + /** + * A structurally sound file whose version is not 1: written by a newer + * schema, not corrupt. Carries the marker code so isDeterministicCorruption + * classifies it as non-quarantinable and load() refuses to self-heal it. + */ + private static isUnsupportedVersion(parsed: unknown): boolean { + return ( + typeof parsed === "object" && + parsed !== null && + !Array.isArray(parsed) && + "version" in parsed && + (parsed as { version?: unknown }).version !== 1 + ); + } + + private static unsupportedVersionError(): NodeJS.ErrnoException { + const error = new Error( + "Unsupported extension metadata file version (written by a newer build)" + ) as NodeJS.ErrnoException; + error.code = UNSUPPORTED_METADATA_VERSION_CODE; + return error; + } + private static isValidMetadataFileShape(parsed: unknown): parsed is ExtensionMetadataFile { if (typeof parsed !== "object" || parsed === null) { return false; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 0c16ccf9c0..592e4fec1f 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3241,7 +3241,10 @@ describe("WorkspaceService activity list scoping", () => { expect(activityList).not.toBeNull(); expect(activityList?.[workspaceId]?.recency).toBe(100); expect(activityList?.["removed-workspace"]).toBeUndefined(); - expect(metadataSpy).toHaveBeenCalledTimes(1); + // Both walks belong to the prune itself (enumeration + the fresh + // pre-deletion recheck that spares concurrently re-registered ids); + // the list's SCOPING reuses the prune's ids instead of paying a third. + expect(metadataSpy).toHaveBeenCalledTimes(2); } finally { await cleanup(); } @@ -3627,9 +3630,11 @@ describe("WorkspaceService activity list scoping", () => { // across restarts until some unrelated writer replaced the file. The // strict path quarantines the bytes (preserved for inspection, never // silently deleted) and the resulting empty state is authoritative. + // Note: a valid file with version !== 1 is deliberately NOT here — that + // is a newer build's schema, treated as unsupported (propagated, never + // quarantined/reset) so a downgrade round-trip cannot destroy it. const corruptFiles = [ "{not json", - JSON.stringify({ version: 2, workspaces: {} }), JSON.stringify({ version: 1, workspaces: [] }), JSON.stringify({ version: 1, workspaces: "bogus" }), JSON.stringify({ version: 1, workspaces: null }), @@ -3863,6 +3868,9 @@ describe("WorkspaceService activity list scoping", () => { "{not json", JSON.stringify({ projects: {} }), JSON.stringify({ projects: [["/tmp/project", { workspaces: "bogus" }]] }), + // Arrays pass typeof "object": lenient normalization turns an + // array-valued project config into a project with no workspaces. + JSON.stringify({ projects: [["/tmp/project", []]] }), ]; for (const corruptConfig of corruptConfigs) { const { config, historyService, cleanup } = await createTestHistoryService(); From dc4561a7fefbb16b2c87be9a0016eb52960ec285 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 18:25:31 +0000 Subject: [PATCH 31/72] review: address round-28 findings - quarantine recovery restores an unsupported-version (newer schema) sidecar to the main path instead of resetting it to an empty version-1 file: the bytes are preserved data, and the subsequent read fails with the non-destructive unsupported-version signal - getActivityList merges in-scope snapshot ADDITIONS from the fresh revalidation re-read: another backend can register a workspace and persist its first activity after this process's initial snapshot read, and the process-local activity subscription supplies no cross-process delta to heal the omission; additions pass the same removal guards as retained entries --- .../services/ExtensionMetadataService.test.ts | 28 ++++++++++ src/node/services/ExtensionMetadataService.ts | 13 ++++- src/node/services/workspaceService.test.ts | 52 +++++++++++++++++++ src/node/services/workspaceService.ts | 50 +++++++++++++++--- 4 files changed, 136 insertions(+), 7 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 39834697c1..8f644c0153 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -759,6 +759,34 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("recovery restores an unsupported-version sidecar instead of resetting it", async () => { + // A newer build's file can end up in the sidecar (crash-interrupted + // quarantine on a downgraded install, or a newer backend saving between + // the corruption check and the rename). It is preserved data, not + // corruption: recovery must put it back at the main path — installing + // the empty version-1 reset would hand the newer build an empty + // canonical file and lose all its activity state. + const newerFile = JSON.stringify({ version: 2, workspaces: {}, futureField: true }); + await writeFile(`${filePath}.corrupt`, newerFile); + // Main file intentionally absent (crash window). + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + // The restored file still fails the CURRENT build's read — but with the + // non-destructive unsupported-version signal, not an empty reset. + expect(strictError).not.toBeNull(); + expect(await readFile(filePath, "utf-8")).toBe(newerFile); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("a strict read restores healthy bytes stranded in the sidecar by a crashed quarantine", async () => { // The crash can also strand a HEALTHY file in the sidecar: a concurrent // writer repaired the main file right before quarantine's rename moved diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index b67ea7464e..e10ab93b64 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -782,7 +782,18 @@ export class ExtensionMetadataService { } movedParses = false; } - if (movedParses && ExtensionMetadataService.isValidMetadataFileShape(moved)) { + if ( + movedParses && + (ExtensionMetadataService.isValidMetadataFileShape(moved) || + // A newer build's schema stranded in the sidecar (crash-interrupted + // quarantine on a downgraded install, or a newer backend saving the + // main file between the in-queue corruption check and the rename) is + // preserved data, not corruption: restore it instead of falling + // through to the empty-reset below, which would hand the newer build + // an empty canonical file. The subsequent re-read then fails with + // the non-destructive unsupported-version signal. + ExtensionMetadataService.isUnsupportedVersion(moved)) + ) { // Restore without ever overwriting a newer file yet another writer // may have re-created at the main path: link and COPYFILE_EXCL // both fail with EEXIST in that case (the healthy leftover sidecar diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 592e4fec1f..8a24cd05a5 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3556,6 +3556,58 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList merges snapshots another process persisted mid-list", async () => { + // Another backend can register a workspace and persist its first + // activity after this process's initial snapshot read. The refreshed + // config admits the id, but the per-id computation saw a null snapshot + // and no local caches, so the entry would be omitted — and the activity + // subscription is process-local, so no delta ever heals it. The fresh + // revalidation re-read must merge the addition. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "late-snapshot-workspace"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + // Simulate the cross-process write landing between the initial read + // (call 1) and the revalidation re-read (call 2). + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 1) { + return realGetAllSnapshots(options); + } + await extensionMetadata.updateRecency(workspaceId, 777); + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[workspaceId]?.recency).toBe(777); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("getActivityList drops snapshotless legacy entries removed mid-list", async () => { // A legacy id-less config entry's stable id is resolved authoritatively // during enumeration and can never appear in the raw config-id baseline, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2195546a0f..0bfe6f9cee 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13016,14 +13016,14 @@ export class WorkspaceService extends EventEmitter { // and the raw persisted config id superset once. Best-effort: an // unreadable re-read skips its revalidation instead of failing an // otherwise complete response. - let freshPersistedIds: ReadonlySet | null = null; + let freshSnapshots: ReadonlyMap | null = null; try { - freshPersistedIds = new Set( - (await this.extensionMetadata.getAllSnapshots({ throwOnError: true })).keys() - ); + freshSnapshots = await this.extensionMetadata.getAllSnapshots({ throwOnError: true }); } catch { - freshPersistedIds = null; + freshSnapshots = null; } + const freshPersistedIds: ReadonlySet | null = + freshSnapshots != null ? new Set(freshSnapshots.keys()) : null; let freshConfigIds: ReadonlySet | null = null; try { freshConfigIds = this.config.readPersistedWorkspaceIdSuperset(); @@ -13098,7 +13098,7 @@ export class WorkspaceService extends EventEmitter { } this.extensionMetadata.clearTombstonesForRegisteredIds(registeredIds); } - return Object.fromEntries( + const activityById = Object.fromEntries( entries.filter( (entry): entry is readonly [string, WorkspaceActivitySnapshot] => entry != null && @@ -13124,6 +13124,44 @@ export class WorkspaceService extends EventEmitter { !isRemovedPerAuthoritativeIdentity(entry[0]) ) ); + // Addition-side counterpart of the fresh re-read: another backend can + // register a workspace AND persist its first activity after this + // process's initial snapshot read. The refreshed config admits the id + // into scope, but its per-id computation saw a null snapshot (and no + // local workflow/monitor caches, which are process-local), so the + // entry was omitted — and the activity subscription cannot heal that + // (it is backed by this process's EventEmitter, so cross-process + // writes produce no delta). Merge in-scope additions from the fresh + // re-read, subject to the same removal guards as retained entries. + if (freshSnapshots != null) { + for (const workspaceId of workspaceIds) { + if (workspaceId in activityById) { + continue; + } + const lateSnapshot = freshSnapshots.get(workspaceId) ?? null; + if ( + lateSnapshot == null || + this.extensionMetadata.isWorkspaceDeleted(workspaceId) || + isRemovedFromConfig(workspaceId) || + isRemovedPerAuthoritativeIdentity(workspaceId) + ) { + continue; + } + // Same sync overlay path emitWorkspaceActivity uses; the per-id + // disk workflow probe already ran for this in-scope id above. + const merged = this.mergeCurrentActiveBashMonitorCount( + workspaceId, + this.mergeCachedActiveWorkflowRuns( + workspaceId, + this.overlayPendingGoal(workspaceId, lateSnapshot) + ) + ); + if (merged != null) { + activityById[workspaceId] = merged; + } + } + } + return activityById; } catch (error) { log.error("Failed to list activity:", error); // null (not {}) so the renderer can tell a read failure from a legitimately From e924e3e7cec8041f4ec1f27b1e3d4c7cadbd8023 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 18:40:09 +0000 Subject: [PATCH 32/72] review: address round-29 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the mid-list snapshot merge admits ids the fresh raw config view proves registered (not only the stale per-id scope), and the authoritative removal check exempts ids visible in the fresh raw view — a workspace registered and written entirely between the scope reads and the revalidation re-reads now surfaces in the authoritative response - EEXIST during quarantine restore is no longer treated as successful recovery: the re-created main file may be a PARTIAL snapshot an older backend self-healed from the missing-main window, so sidecar-only entries are reconciled into it (main wins per key; tombstoned ids stay out) and the sidecar consumed; resumed recoveries reconcile the same way when both files exist --- .../services/ExtensionMetadataService.test.ts | 39 +++++++++ src/node/services/ExtensionMetadataService.ts | 79 ++++++++++++++++--- src/node/services/workspaceService.test.ts | 51 ++++++++++++ src/node/services/workspaceService.ts | 19 ++++- 4 files changed, 177 insertions(+), 11 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 8f644c0153..01e9fd5281 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -787,6 +787,45 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("recovery reconciles a partial main file recreated during the crash window", async () => { + // An older backend can observe the missing-main window as an empty file + // and save a PARTIAL snapshot before recovery restores the sidecar. + // EEXIST is not success: the sidecar's other entries must be merged into + // the recreated file (main wins per key) instead of being abandoned. + const sidecar = { + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }; + await writeFile(`${filePath}.corrupt`, JSON.stringify(sidecar)); + // Main file intentionally absent; the probe hook below recreates it as a + // partial file mid-recovery (modeling the concurrent older backend). + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(service); + internals.probeQuarantineSidecar = async () => { + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 300, streaming: false } }, + }) + ); + internals.probeQuarantineSidecar = originalProbe; + return true; + }; + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-new")?.recency).toBe(300); + expect(snapshots.get("ws-other")?.recency).toBe(42); + // Sidecar consumed by the reconcile. + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("a strict read restores healthy bytes stranded in the sidecar by a crashed quarantine", async () => { // The crash can also strand a HEALTHY file in the sidecar: a concurrent // writer repaired the main file right before quarantine's rename moved diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index e10ab93b64..293b3746c9 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -795,14 +795,17 @@ export class ExtensionMetadataService { ExtensionMetadataService.isUnsupportedVersion(moved)) ) { // Restore without ever overwriting a newer file yet another writer - // may have re-created at the main path: link and COPYFILE_EXCL - // both fail with EEXIST in that case (the healthy leftover sidecar - // is then harmless and bounded by the fixed name). + // may have re-created at the main path: link and COPYFILE_EXCL both + // fail with EEXIST in that case. EEXIST is NOT a successful recovery + // though — the re-created file may be a PARTIAL snapshot an older + // backend self-healed from the missing-main window (older builds read + // ENOENT as empty and save their one mutated entry), so the sidecar's + // other entries must be reconciled into it rather than abandoned. try { await link(quarantinePath, this.filePath); } catch (linkError) { if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { - return false; + return this.reconcileRecreatedMainWithSidecar(quarantinePath); } try { // Filesystems without hard-link support (or EPERM): copy-based @@ -810,7 +813,7 @@ export class ExtensionMetadataService { await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); } catch (copyError) { if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { - return false; + return this.reconcileRecreatedMainWithSidecar(quarantinePath); } // Restore failed with the main path missing: rethrow so the // strict reader propagates a retryable failure instead of @@ -874,25 +877,81 @@ export class ExtensionMetadataService { await this.withSerializedMutation(async () => { // Re-check inside the queue: a concurrent writer save or a sibling // strict read may already have recovered the main path. - const mainExists = await access(this.filePath).then( + const quarantinePath = `${this.filePath}.corrupt`; + const sidecarExists = await access(quarantinePath).then( () => true, () => false ); - if (mainExists) { + if (!sidecarExists) { return; } - const quarantinePath = `${this.filePath}.corrupt`; - const sidecarExists = await access(quarantinePath).then( + const mainExists = await access(this.filePath).then( () => true, () => false ); - if (!sidecarExists) { + if (mainExists) { + // Main was recreated during the crash window — possibly a PARTIAL + // file another backend self-healed from the missing-main state. + // Merge the sidecar's other entries back instead of abandoning them. + await this.reconcileRecreatedMainWithSidecar(quarantinePath); return; } await this.completeQuarantineRecovery(quarantinePath); }); } + /** + * A restore found the main path already re-created (EEXIST), or a resumed + * recovery found both files present. The re-created file may be a PARTIAL + * snapshot an older backend self-healed from the missing-main window; + * treating it as authoritative would permanently hide every other + * workspace's recency/goal/status while the full data sits in the sidecar. + * Merge sidecar-only entries into the main file (main wins per key — its + * writes are newer; ids this process write-tombstoned stay out) and consume + * the sidecar. Only same-schema (version 1) sidecars can be merged: a + * corrupt or newer-schema sidecar is left as the bounded fixed-name + * leftover. Must run inside withSerializedMutation. Returns false (no + * empty reset happened). + */ + private async reconcileRecreatedMainWithSidecar(quarantinePath: string): Promise { + let sidecarParsed: unknown; + try { + sidecarParsed = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; + } catch { + return false; + } + if (!ExtensionMetadataService.isValidMetadataFileShape(sidecarParsed)) { + return false; + } + let main: ExtensionMetadataFile; + try { + const parsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; + if (!ExtensionMetadataService.isValidMetadataFileShape(parsed)) { + // Corrupt/newer-schema main: leave both files for the normal read + // classification paths rather than merging across schemas. + return false; + } + main = parsed; + } catch { + return false; + } + let modified = false; + for (const [workspaceId, entry] of Object.entries(sidecarParsed.workspaces)) { + if (workspaceId in main.workspaces || this.deletedWorkspaceIds.has(workspaceId)) { + continue; + } + main.workspaces[workspaceId] = entry; + modified = true; + } + if (modified) { + await this.save(main); + } + // Consumed either way: every surviving sidecar entry is now represented + // at the main path. + await unlink(quarantinePath).catch(() => undefined); + return false; + } + async getAllSnapshots(options?: { throwOnError?: boolean; }): Promise> { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 8a24cd05a5..67df47cd1d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3608,6 +3608,57 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList merges workspaces registered and written entirely mid-list", async () => { + // Harder variant of the mid-list merge: the workspace is registered AND + // written after every scope read (baseline, prune, refresh), so it is in + // neither the per-id scope nor the initial snapshots — only the fresh + // revalidation views (snapshot re-read + raw config re-read) know it. + // The merge must admit ids those fresh views agree on. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "brand-new-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + // The other backend registers the workspace and persists its + // first activity between the initial read and the revalidation + // re-read (before the fresh raw config re-read). + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(workspaceId, 888); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[workspaceId]?.recency).toBe(888); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("getActivityList drops snapshotless legacy entries removed mid-list", async () => { // A legacy id-less config entry's stable id is resolved authoritatively // during enumeration and can never appear in the raw config-id baseline, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 0bfe6f9cee..93f579acb0 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13078,6 +13078,11 @@ export class WorkspaceService extends EventEmitter { const isRemovedPerAuthoritativeIdentity = (workspaceId: string): boolean => initialConfigIds != null && !initialConfigIds.has(workspaceId) && + // An id visible in the FRESH raw view is verifiably registered + // regardless of what the (possibly earlier) authoritative + // enumeration saw — e.g. a workspace registered after that + // enumeration ran must not read as removed. + !(freshConfigIds?.has(workspaceId) ?? false) && authoritativeIds != null && !authoritativeIds.has(workspaceId); // Tombstones are process-local removal knowledge; the shared config is @@ -13134,7 +13139,19 @@ export class WorkspaceService extends EventEmitter { // writes produce no delta). Merge in-scope additions from the fresh // re-read, subject to the same removal guards as retained entries. if (freshSnapshots != null) { - for (const workspaceId of workspaceIds) { + // Merge scope: the (possibly stale) per-id scope PLUS fresh-snapshot + // ids the fresh raw config view proves registered — a workspace + // registered and written between the scope reads and the fresh + // re-read is in both fresh views but in neither stale one. + const mergeCandidateIds = new Set(workspaceIds); + if (freshConfigIds != null) { + for (const workspaceId of freshSnapshots.keys()) { + if (freshConfigIds.has(workspaceId)) { + mergeCandidateIds.add(workspaceId); + } + } + } + for (const workspaceId of mergeCandidateIds) { if (workspaceId in activityById) { continue; } From bbeeb2f538a8dc8442d182ec4092d9645978b39f Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 19:03:11 +0000 Subject: [PATCH 33/72] review: address round-30 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - reconcileRecreatedMainWithSidecar propagates transient I/O failures (EACCES/EIO-class) from both the sidecar and main reads instead of reporting success: an unverifiable sidecar must keep the strict read retryable, or the caller accepts a partial recreated main while the healthy sidecar is never inspected again; only deterministic parse corruption stays as the bounded leftover (ENOENT means a concurrent recovery consumed the file) - an unsupported-version (newer-schema) sidecar now wins the canonical path when reconcile finds a recreated version-1 main: accepting the partial file would strand the newer data permanently (nothing re-inspects the sidecar once the main path exists, and a later quarantine rename would destroy it); the recreated file is preserved as its own bounded fixed-name leftover, and a crash mid-swap leaves the resumable missing-main + sidecar state - the mid-list snapshot merge bootstraps active workflow runs from disk for candidates admitted by the fresh revalidation re-reads: those ids never went through the per-id loop, so a cached-only merge omitted activeWorkflowRunCount for exactly the cross-process registrations the merge exists to bootstrap - clearTombstonesForRegisteredIds only clears tombstones captured in a snapshot taken before the registration evidence reads: a same-process removal landing during the authoritative enumeration await publishes its tombstone after the evidence was captured, and the stale evidence must not clear it (the pre-removal snapshot would ride back into the renderer and late producers could persist the entry again) --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- .../services/ExtensionMetadataService.test.ts | 96 ++++++++++++++++++- src/node/services/ExtensionMetadataService.ts | 96 +++++++++++++++++-- src/node/services/workspaceService.test.ts | 71 ++++++++++++++ src/node/services/workspaceService.ts | 33 +++++-- 4 files changed, 282 insertions(+), 14 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 01e9fd5281..8be88cfb6b 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test"; -import { mkdtemp, readFile, rm, writeFile } from "fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises"; import { tmpdir } from "os"; import * as path from "path"; import { ExtensionMetadataService } from "./ExtensionMetadataService"; @@ -826,6 +826,100 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("reconcile restores an unsupported-version sidecar over a recreated partial main", async () => { + // Downgrade-overlap variant of the reconcile: a NEWER build's schema is + // stranded in the sidecar while an older-schema backend recreates a + // partial main during the crash window. Accepting the partial file would + // lose the newer data permanently — nothing re-inspects the sidecar once + // the main path exists, and a later quarantine's rename would destroy + // it. The newer bytes must win the canonical path; the recreated file is + // preserved as its own leftover. + const newerFile = JSON.stringify({ version: 2, workspaces: {}, futureField: true }); + await writeFile(`${filePath}.corrupt`, newerFile); + // Main file intentionally absent; the probe hook recreates it as a + // partial version-1 file mid-recovery (modeling the older backend). + const partial = JSON.stringify({ + version: 1, + workspaces: { "ws-partial": { recency: 300, streaming: false } }, + }); + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(service); + internals.probeQuarantineSidecar = async () => { + await writeFile(filePath, partial); + internals.probeQuarantineSidecar = originalProbe; + return true; + }; + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + // The swapped-in file still fails the CURRENT build's read — but with + // the non-destructive unsupported-version signal, not the partial data. + expect(strictError).not.toBeNull(); + expect(await readFile(filePath, "utf-8")).toBe(newerFile); + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe(partial); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("an unreadable sidecar keeps reconcile retryable instead of accepting the partial main", async () => { + // Transient I/O failure reading the sidecar during reconcile: reporting + // success would make the read accept the recreated partial main while + // the (possibly healthy) sidecar is never inspected again. A directory + // at the sidecar path yields a deterministic errno (EISDIR) standing in + // for EACCES/EIO-class failures. + await mkdir(`${filePath}.corrupt`); + const partial = JSON.stringify({ + version: 1, + workspaces: { "ws-partial": { recency: 300, streaming: false } }, + }); + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(service); + internals.probeQuarantineSidecar = async () => { + await writeFile(filePath, partial); + internals.probeQuarantineSidecar = originalProbe; + return true; + }; + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + expect((strictError as NodeJS.ErrnoException).code).toBe("EISDIR"); + // Once the sidecar becomes readable (here: gone), the retry proceeds. + await rm(`${filePath}.corrupt`, { recursive: true }); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-partial")?.recency).toBe(300); + }); + + test("clearTombstonesForRegisteredIds only clears tombstones the evidence postdates", async () => { + await service.updateRecency("ws-1", 100); + // Evidence snapshot captured BEFORE the removal: a tombstone published + // afterwards (same-process removal racing the activity list's evidence + // reads) must survive stale evidence claiming the id is registered. + const preEvidence = service.getTombstonedIds(); + await service.deleteWorkspace("ws-1"); + service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), preEvidence); + expect(service.isWorkspaceDeleted("ws-1")).toBe(true); + // The next bootstrap snapshots the tombstone before its evidence reads, + // so a genuinely re-registered id is cleared then. + service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), service.getTombstonedIds()); + expect(service.isWorkspaceDeleted("ws-1")).toBe(false); + }); + test("a strict read restores healthy bytes stranded in the sidecar by a crashed quarantine", async () => { // The crash can also strand a HEALTHY file in the sidecar: a concurrent // writer repaired the main file right before quarantine's rename moved diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 293b3746c9..723b32b6c3 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -543,15 +543,35 @@ export class ExtensionMetadataService { * filter it from activity lists) until restart. Called from the activity * bootstrap with fresh config-derived id sets only — never with snapshot * or in-memory cache keys, which do not prove registration. + * + * `eligibleIds` must be a getTombstonedIds() snapshot captured BEFORE the + * registration evidence was gathered: clearing is sound only when the + * evidence postdates the tombstone. A tombstone published while the + * evidence reads were in flight (same-process removal during the activity + * list's authoritative enumeration await) would otherwise be cleared by a + * stale pre-removal view, un-suppressing late writers and letting the + * removed entry ride back into the renderer. */ - clearTombstonesForRegisteredIds(registeredIds: ReadonlySet): void { - for (const workspaceId of this.deletedWorkspaceIds) { + clearTombstonesForRegisteredIds( + registeredIds: ReadonlySet, + eligibleIds: ReadonlySet + ): void { + for (const workspaceId of eligibleIds) { if (registeredIds.has(workspaceId)) { this.deletedWorkspaceIds.delete(workspaceId); } } } + /** + * Snapshot of the ids currently write-tombstoned in this process. Capture + * it before gathering registration evidence and pass it back to + * clearTombstonesForRegisteredIds as the set of clearable tombstones. + */ + getTombstonedIds(): ReadonlySet { + return new Set(this.deletedWorkspaceIds); + } + /** * Delete metadata for a workspace. * Call this when a workspace is deleted. @@ -909,15 +929,67 @@ export class ExtensionMetadataService { * Merge sidecar-only entries into the main file (main wins per key — its * writes are newer; ids this process write-tombstoned stay out) and consume * the sidecar. Only same-schema (version 1) sidecars can be merged: a - * corrupt or newer-schema sidecar is left as the bounded fixed-name - * leftover. Must run inside withSerializedMutation. Returns false (no - * empty reset happened). + * corrupt sidecar is left as the bounded fixed-name leftover, while a + * newer-schema sidecar is restored to the canonical path (superseding the + * recreated file, preserved as its own leftover). Must run inside + * withSerializedMutation. Returns false (no empty reset happened). */ private async reconcileRecreatedMainWithSidecar(quarantinePath: string): Promise { let sidecarParsed: unknown; try { sidecarParsed = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; - } catch { + } catch (readError) { + // Sidecar gone: a concurrent recovery consumed it and the recreated + // main is all there is — nothing left to reconcile. + if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { + return false; + } + // Transient I/O failure (EACCES/EIO/...): the sidecar cannot be + // verified, and reporting success would make the caller accept the + // possibly-partial recreated main while the healthy sidecar is never + // inspected again (loads stop probing it once the main path exists). + // Propagate so the strict read stays retryable; only deterministic + // parse corruption below stays as the bounded fixed-name leftover. + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + return false; + } + if (ExtensionMetadataService.isUnsupportedVersion(sidecarParsed)) { + // A newer build's schema stranded in the sidecar while an older-schema + // backend re-created the main path (usually a partial ENOENT self-heal + // holding one freshly mutated entry). Accepting the recreated file + // would lose the newer data permanently: nothing re-inspects the + // sidecar once the main path exists, so even re-upgrading reads the + // partial file — and a later quarantine's rename would destroy the + // sidecar bytes. Restore the newer bytes to the canonical path + // (upgrade↔downgrade preservation; this build's readers then see the + // same non-destructive retryable unsupported-version signal as any + // downgrade overlap) and preserve the recreated file as a bounded + // fixed-name leftover. A crash between the two steps leaves the + // resumable missing-main + sidecar state, which restores the + // unsupported sidecar via completeQuarantineRecovery. + await rename(this.filePath, `${this.filePath}.recreated`); + try { + await link(quarantinePath, this.filePath); + } catch (linkError) { + if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + // Yet another writer re-created the main path mid-swap: re-enter + // with the new file (the leftover keeps the latest superseded one). + return this.reconcileRecreatedMainWithSidecar(quarantinePath); + } + try { + await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + return this.reconcileRecreatedMainWithSidecar(quarantinePath); + } + // Main path missing with the newer bytes still in the sidecar: + // rethrow (retryable) — the resumed recovery restores them. + throw copyError; + } + } + await unlink(quarantinePath).catch(() => undefined); return false; } if (!ExtensionMetadataService.isValidMetadataFileShape(sidecarParsed)) { @@ -932,7 +1004,17 @@ export class ExtensionMetadataService { return false; } main = parsed; - } catch { + } catch (readError) { + // Main vanished again mid-reconcile: back to the resumable + // missing-main state the normal read paths already handle. + if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { + return false; + } + // Same transient-I/O contract as the sidecar read above: an + // unverifiable main must stay retryable, not report success. + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } return false; } let modified = false; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 67df47cd1d..7ccf8e22d4 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -143,6 +143,7 @@ const mockInitStateManager: Partial = { const mockExtensionMetadataService: Partial = { isWorkspaceDeleted: mock(() => false), clearTombstonesForRegisteredIds: mock(() => undefined), + getTombstonedIds: mock((): ReadonlySet => new Set()), setStreaming: mock(() => Promise.resolve({ recency: Date.now(), @@ -3659,6 +3660,76 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList bootstraps workflow runs for workspaces merged mid-list", async () => { + // A workspace admitted only by the fresh revalidation re-reads never went + // through the per-id loop, so its on-disk active workflow runs are not in + // the process-local cache. The merge must probe disk for them — a + // cached-only merge would omit activeWorkflowRunCount for exactly the + // cross-process registrations it exists to bootstrap, and the + // process-local activity subscription can never deliver that delta. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "brand-new-workflow-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + // The other backend registers the workspace, persists its first + // activity, AND starts a workflow run before the revalidation + // re-read. + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(workspaceId, 888); + const runStore = new WorkflowRunStore({ + sessionDir: config.getSessionDir(workspaceId), + }); + await runStore.createRun({ + id: "wfr_midlist", + workspaceId, + workflow: { + name: "demo", + description: "Demo workflow", + scope: "global" as const, + executable: true, + }, + source: "export default function workflow() { return {}; }", + args: {}, + now: "2026-06-17T00:00:00.000Z", + }); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[workspaceId]?.recency).toBe(888); + expect(activityList?.[workspaceId]?.activeWorkflowRunCount).toBe(1); + expect(activityList?.[workspaceId]?.activeWorkflowRunIds).toEqual(["wfr_midlist"]); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("getActivityList drops snapshotless legacy entries removed mid-list", async () => { // A legacy id-less config entry's stable id is resolved authoritatively // during enumeration and can never appear in the raw config-id baseline, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 93f579acb0..4997afefac 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13024,6 +13024,17 @@ export class WorkspaceService extends EventEmitter { } const freshPersistedIds: ReadonlySet | null = freshSnapshots != null ? new Set(freshSnapshots.keys()) : null; + // Tombstones the registration evidence below may legitimately clear: + // only ones that already exist HERE, before the evidence is captured + // (the raw superset read is synchronous with this snapshot, and the + // authoritative enumeration runs later still). A same-process removal + // landing during the enumeration await publishes its tombstone after + // the evidence reads began — stale evidence still showing the id + // registered must not clear that fresh tombstone, or the pre-removal + // snapshot rides back into the renderer and late producers persist the + // entry again. Such a tombstone stays clearable by the NEXT list's + // fresh evidence if the id really is re-registered. + const clearableTombstoneIds = this.extensionMetadata.getTombstonedIds(); let freshConfigIds: ReadonlySet | null = null; try { freshConfigIds = this.config.readPersistedWorkspaceIdSuperset(); @@ -13101,7 +13112,10 @@ export class WorkspaceService extends EventEmitter { for (const workspaceId of authoritativeIds ?? []) { registeredIds.add(workspaceId); } - this.extensionMetadata.clearTombstonesForRegisteredIds(registeredIds); + this.extensionMetadata.clearTombstonesForRegisteredIds( + registeredIds, + clearableTombstoneIds + ); } const activityById = Object.fromEntries( entries.filter( @@ -13164,13 +13178,20 @@ export class WorkspaceService extends EventEmitter { ) { continue; } - // Same sync overlay path emitWorkspaceActivity uses; the per-id - // disk workflow probe already ran for this in-scope id above. + // Same overlay path emitWorkspaceActivity uses, except workflow + // runs come from the bootstrapping probe rather than the bare + // cache: a candidate admitted by the fresh config/snapshot re-reads + // never went through the per-id loop above, so its on-disk active + // workflow runs are not cached yet — a cached-only merge would omit + // activeWorkflowRunCount for exactly the cross-process + // registrations this merge exists to bootstrap, and the process- + // local subscription can never deliver the missing delta. Ids the + // per-id loop already probed resolve from the shared cached Set. const merged = this.mergeCurrentActiveBashMonitorCount( workspaceId, - this.mergeCachedActiveWorkflowRuns( - workspaceId, - this.overlayPendingGoal(workspaceId, lateSnapshot) + mergeActiveWorkflowRuns( + this.overlayPendingGoal(workspaceId, lateSnapshot), + await this.getActiveWorkflowRunIds(workspaceId) ) ); if (merged != null) { From 09006de1eb7303db403deda57d5ba13649715583 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 19:32:22 +0000 Subject: [PATCH 34/72] review: address round-31 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the mid-list merge probes workflow runs BEFORE the final guard views and re-reads the metadata file plus the raw config superset after the probes, so no await separates guard evaluation from insertion: a removal landing while the probe awaited disk (cross-process key deletion, same-process tombstone, raw deregistration) can no longer ride the stale late snapshot back into the authoritative response - raw-invisible legacy ids (stable ids living only in session metadata.json) registered by a downgraded backend mid-list are admitted into the merge through the authoritative enumeration, which now also triggers when a fresh-snapshot id is outside the per-id scope and the raw view cannot vouch for it — without that the workspace stayed absent until reconnect - the prune's mid-pass re-registration recheck reads the raw config id evidence instead of repeating the strict per-workspace enumeration: readPersistedWorkspaceIdEvidence reports whether any persisted workspace entry lacks an inline id, and only then (id-less legacy entries can be registered raw-invisibly) is the enumeration repeated — stale-heavy modern deployments no longer pay the walk twice while the metadata mutation queue blocks live recency/status writes --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- src/node/config.test.ts | 35 +++ src/node/config.ts | 41 +++- src/node/services/ExtensionMetadataService.ts | 12 +- src/node/services/workspaceService.test.ts | 126 ++++++++++- src/node/services/workspaceService.ts | 201 ++++++++++++------ 5 files changed, 347 insertions(+), 68 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index afdafcf455..5a69fd3931 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -929,6 +929,41 @@ describe("Config", () => { fs.writeFileSync(path.join(tempDir, "config.json"), JSON.stringify(["array-root"])); expect(() => new Config(tempDir).readPersistedWorkspaceIdSuperset()).toThrow(); }); + + it("reports whether any workspace entry lacks an inline id", () => { + // Completeness signal for registration evidence: with inline ids + // everywhere, the raw view is complete and callers may skip the + // per-workspace authoritative enumeration; a single id-less (legacy) + // entry means a raw-invisible stable id may exist. + const configPath = path.join(tempDir, "config.json"); + fs.writeFileSync( + configPath, + JSON.stringify({ + projects: [["/repo", { workspaces: [{ id: "modern", path: "/repo/ws" }] }]], + }) + ); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(["modern"]), + hasWorkspaceEntriesWithoutIds: false, + }); + fs.writeFileSync( + configPath, + JSON.stringify({ + projects: [ + ["/repo", { workspaces: [{ id: "modern", path: "/repo/ws" }, { path: "/repo/old" }] }], + ], + }) + ); + const evidence = new Config(tempDir).readPersistedWorkspaceIdEvidence(); + expect(evidence.hasWorkspaceEntriesWithoutIds).toBe(true); + expect(evidence.ids.has("modern")).toBe(true); + // Missing file: healthy empty evidence (fresh install). + fs.rmSync(configPath); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(), + hasWorkspaceEntriesWithoutIds: false, + }); + }); }); describe("legacy task variant compatibility", () => { diff --git a/src/node/config.ts b/src/node/config.ts index f5139c5c20..c175224f8f 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1107,7 +1107,27 @@ export class Config { * a plain object; a missing file resolves as an empty set (fresh install). */ readPersistedWorkspaceIdSuperset(): Set { + return this.readPersistedWorkspaceIdEvidence().ids; + } + + /** + * readPersistedWorkspaceIdSuperset plus a completeness signal: whether any + * persisted workspace entry lacks an inline string id. Only such id-less + * (legacy) entries can be registered "raw-invisibly" — their stable id + * lives in the session metadata.json, which only the authoritative + * enumeration resolves. When this reports false, the raw id set is + * complete registration evidence and callers can skip that per-workspace + * enumeration. Detection is conservative in the destructive direction: + * any workspaces container whose entries cannot be verified to carry ids + * reports true (over-reporting merely costs an extra enumeration, while + * under-reporting could let a pruner delete a live legacy workspace). + */ + readPersistedWorkspaceIdEvidence(): { + ids: Set; + hasWorkspaceEntriesWithoutIds: boolean; + } { const ids = new Set(); + let hasWorkspaceEntriesWithoutIds = false; let raw: string; try { raw = fs.readFileSync(this.configFile, "utf-8"); @@ -1117,7 +1137,7 @@ export class Config { // set and let destructive callers treat every workspace as removed. // Only a genuinely missing file is a healthy empty set (fresh install). if (isEnoentError(error)) { - return ids; + return { ids, hasWorkspaceEntriesWithoutIds }; } throw error; } @@ -1125,6 +1145,13 @@ export class Config { if (!parsedValue || typeof parsedValue !== "object" || Array.isArray(parsedValue)) { throw new Error("Config root must be a JSON object"); } + const hasInlineStringId = (value: unknown): boolean => { + if (value === null || typeof value !== "object") { + return false; + } + const id = (value as { id?: unknown }).id; + return typeof id === "string" && id.length > 0; + }; const collect = (value: unknown): void => { if (Array.isArray(value)) { for (const entry of value) { @@ -1137,13 +1164,23 @@ export class Config { if (typeof id === "string" && id.length > 0) { ids.add(id); } + const workspaces = (value as { workspaces?: unknown }).workspaces; + if (Array.isArray(workspaces)) { + if (workspaces.some((entry) => !hasInlineStringId(entry))) { + hasWorkspaceEntriesWithoutIds = true; + } + } else if (workspaces !== null && typeof workspaces === "object") { + // A workspaces container in an uninterpretable shape may still + // describe registered workspaces; stay conservative. + hasWorkspaceEntriesWithoutIds = true; + } for (const nested of Object.values(value)) { collect(nested); } } }; collect((parsedValue as { projects?: unknown }).projects); - return ids; + return { ids, hasWorkspaceEntriesWithoutIds }; } loadConfigOrDefault(options?: { throwOnError?: boolean }): ProjectsConfig { diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 723b32b6c3..81bd017b7c 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -628,7 +628,15 @@ export class ExtensionMetadataService { * on-disk format is unchanged. */ async pruneMissingWorkspaces( - getKnownWorkspaceIds: () => Promise> + getKnownWorkspaceIds: () => Promise>, + // Optional cheaper view for the mid-prune re-registration recheck: it + // only needs to answer "is this stale-classified id registered NOW", so + // callers whose full enumeration is expensive (per-workspace filesystem + // walks) can substitute an equally-complete but cheaper read. It must be + // COMPLETE (contain every currently registered id) or throw — resolving + // with a lossy set would let the prune delete a re-registered + // workspace's data. Defaults to getKnownWorkspaceIds. + recheckKnownWorkspaceIds?: () => Promise> ): Promise { return this.withSerializedMutation(async () => { const data = await this.load(); @@ -654,7 +662,7 @@ export class ExtensionMetadataService { // data. A re-registered id is dropped from the deletion set and its // write tombstone lifted. If the recheck fails, abort the prune (throw // to the caller's catch) rather than deleting on stale knowledge. - const recheckedKnownIds = await getKnownWorkspaceIds(); + const recheckedKnownIds = await (recheckKnownWorkspaceIds ?? getKnownWorkspaceIds)(); let prunedCount = 0; for (const workspaceId of staleWorkspaceIds) { if (recheckedKnownIds.has(workspaceId)) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7ccf8e22d4..c5e2c07686 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3242,10 +3242,14 @@ describe("WorkspaceService activity list scoping", () => { expect(activityList).not.toBeNull(); expect(activityList?.[workspaceId]?.recency).toBe(100); expect(activityList?.["removed-workspace"]).toBeUndefined(); - // Both walks belong to the prune itself (enumeration + the fresh - // pre-deletion recheck that spares concurrently re-registered ids); - // the list's SCOPING reuses the prune's ids instead of paying a third. - expect(metadataSpy).toHaveBeenCalledTimes(2); + // The single walk belongs to the prune's initial enumeration: the + // list's SCOPING reuses the prune's ids, and the prune's mid-pass + // re-registration recheck uses the raw config view (complete evidence + // here — every persisted workspace id is inline) instead of repeating + // the per-workspace walk while the metadata queue blocks live writes. + expect(metadataSpy).toHaveBeenCalledTimes(1); + // The stale entry really was reclaimed on disk, not merely filtered. + expect((await extensionMetadata.getAllSnapshots()).has("removed-workspace")).toBe(false); } finally { await cleanup(); } @@ -3730,6 +3734,120 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList admits raw-invisible legacy workspaces registered mid-list", async () => { + // A downgraded backend can register a legacy (id-less config entry) + // workspace mid-list: its stable id lives only in session metadata.json, + // so the fresh raw config re-read can never vouch for it. The merge must + // resolve such fresh-snapshot ids through the authoritative identity + // path instead of excluding them until reconnect. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "late-legacy-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + await extensionMetadata.updateRecency(stableId, 777); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[stableId]?.recency).toBe(777); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList drops mid-list additions removed during the workflow probe", async () => { + // The workflow-run bootstrap for late merge candidates awaits disk; a + // cross-process removal landing during that probe is invisible to every + // guard view captured before it. The final post-probe snapshot re-read + // must drop the entry instead of riding the deleted id back into the + // renderer (the process-local subscription cannot correct it). + const { config, historyService, cleanup } = await createTestHistoryService(); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const workspaceId = "late-then-removed"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(workspaceId, 888); + } + return realGetAllSnapshots(options); + } + ); + listStatusSnapshotsSpy.mockImplementation(async () => { + // Another backend removes the workspace while the probe is awaited: + // its persisted metadata entry disappears, unseen by this process's + // tombstones. + await new ExtensionMetadataService(metadataPath).deleteWorkspace(workspaceId); + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + test("getActivityList drops snapshotless legacy entries removed mid-list", async () => { // A legacy id-less config entry's stable id is resolved authoritatively // during enumeration and can never appear in the raw config-id baseline, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 4997afefac..59767d90ad 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12846,29 +12846,53 @@ export class WorkspaceService extends EventEmitter { this.prunedStaleExtensionMetadata = true; try { let normalizedIds: Set | null = null; - const prunedCount = await this.extensionMetadata.pruneMissingWorkspaces(async () => { - // Invoked inside the file's serialized mutation AFTER the file load, - // reading config fresh from disk (see pruneMissingWorkspaces), so a - // concurrently created workspace — even in another backend process — - // cannot lose its just-written entry. - // - // Union of two views, both of which throw (aborting the prune, caught - // below) rather than resolving with a silently lossy id set: - // - the raw persisted superset covers entries loadConfigOrDefault's - // validation/normalization would filter or discard (see - // readPersistedWorkspaceIdSuperset), so a live workspace with a - // malformed config entry is never treated as removed; - // - the strict normalized view covers ids produced by in-memory - // config migrations that are not yet persisted verbatim. - // A missing config file resolves as a healthy empty set in both. - const knownIds = this.config.readPersistedWorkspaceIdSuperset(); - const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); - normalizedIds = new Set(allMetadata.map((metadata) => metadata.id)); - for (const workspaceId of normalizedIds) { - knownIds.add(workspaceId); - } - return knownIds; - }); + const prunedCount = await this.extensionMetadata.pruneMissingWorkspaces( + async () => { + // Invoked inside the file's serialized mutation AFTER the file load, + // reading config fresh from disk (see pruneMissingWorkspaces), so a + // concurrently created workspace — even in another backend process — + // cannot lose its just-written entry. + // + // Union of two views, both of which throw (aborting the prune, caught + // below) rather than resolving with a silently lossy id set: + // - the raw persisted superset covers entries loadConfigOrDefault's + // validation/normalization would filter or discard (see + // readPersistedWorkspaceIdSuperset), so a live workspace with a + // malformed config entry is never treated as removed; + // - the strict normalized view covers ids produced by in-memory + // config migrations that are not yet persisted verbatim. + // A missing config file resolves as a healthy empty set in both. + const knownIds = this.config.readPersistedWorkspaceIdSuperset(); + const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); + normalizedIds = new Set(allMetadata.map((metadata) => metadata.id)); + for (const workspaceId of normalizedIds) { + knownIds.add(workspaceId); + } + return knownIds; + }, + async () => { + // Mid-prune re-registration recheck. The first callback's strict + // enumeration walks a session lookup per configured workspace; + // rerunning it doubles that cost on exactly the stale-heavy + // deployments this prune exists to fix, while the serialized + // metadata queue blocks live recency/status writes. The raw id view + // is complete registration evidence whenever every persisted + // workspace entry carries its id inline — only id-less legacy + // entries (whose stable id lives in session metadata.json) can be + // registered raw-invisibly, so the enumeration is repeated only + // when such entries exist. Both reads throw on failure, aborting + // the prune rather than deleting on lossy evidence. + const evidence = this.config.readPersistedWorkspaceIdEvidence(); + if (!evidence.hasWorkspaceEntriesWithoutIds) { + return evidence.ids; + } + const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); + for (const metadata of allMetadata) { + evidence.ids.add(metadata.id); + } + return evidence.ids; + } + ); if (prunedCount > 0) { log.info(`Pruned ${prunedCount} stale extension metadata entries`); } @@ -13068,12 +13092,26 @@ export class WorkspaceService extends EventEmitter { // (unreadable/id-less metadata.json throws in strict mode) skips the // recheck and conservatively retains every raw-invisible id. Computed // only when a retained entry is actually missing from the raw - // baseline, so modern deployments (every workspace id persisted in - // config) never pay the extra walk. + // baseline (or a raw-invisible late snapshot needs admission below), + // so modern deployments (every workspace id persisted in config) + // never pay the extra walk. + // Second trigger: a fresh-snapshot id outside the per-id scope that + // the raw view cannot vouch for. It is either a raw-invisible legacy + // id a downgraded backend registered mid-list — which the merge below + // must ADMIT, and only the authoritative enumeration can prove + // registered — or noise the merge must keep excluding; either way the + // enumeration is the only view that can tell. + const hasRawInvisibleLateSnapshotId = + freshSnapshots != null && + Array.from(freshSnapshots.keys()).some( + (workspaceId) => + !workspaceIds.has(workspaceId) && !(freshConfigIds?.has(workspaceId) ?? false) + ); let authoritativeIds: ReadonlySet | null = null; if ( initialConfigIds != null && - entries.some((entry) => entry != null && !initialConfigIds.has(entry[0])) + (entries.some((entry) => entry != null && !initialConfigIds.has(entry[0])) || + hasRawInvisibleLateSnapshotId) ) { try { authoritativeIds = new Set( @@ -13154,48 +13192,91 @@ export class WorkspaceService extends EventEmitter { // re-read, subject to the same removal guards as retained entries. if (freshSnapshots != null) { // Merge scope: the (possibly stale) per-id scope PLUS fresh-snapshot - // ids the fresh raw config view proves registered — a workspace + // ids a fresh config-derived view proves registered — a workspace // registered and written between the scope reads and the fresh - // re-read is in both fresh views but in neither stale one. + // re-read is in both fresh views but in neither stale one. Raw- + // invisible ids (legacy stable ids resolved from session + // metadata.json) can never appear in the raw view, so they are + // admitted through the authoritative enumeration instead — without + // that, a legacy workspace registered by a downgraded backend + // mid-list would stay absent until reconnect (the process-local + // subscription cannot deliver the cross-process snapshot). const mergeCandidateIds = new Set(workspaceIds); - if (freshConfigIds != null) { - for (const workspaceId of freshSnapshots.keys()) { - if (freshConfigIds.has(workspaceId)) { - mergeCandidateIds.add(workspaceId); - } + for (const workspaceId of freshSnapshots.keys()) { + if ( + (freshConfigIds?.has(workspaceId) ?? false) || + (authoritativeIds?.has(workspaceId) ?? false) + ) { + mergeCandidateIds.add(workspaceId); } } + // Workflow-run bootstrap for candidates the per-id loop never saw: + // their on-disk active runs are not in the process-local cache, and + // a cached-only merge would omit activeWorkflowRunCount for exactly + // the cross-process registrations this merge exists to bootstrap. + // Probed BEFORE the final guard views below so no await separates + // guard evaluation from insertion (ids the per-id loop already + // probed resolve synchronously from the shared cached Set). + const probedWorkflowRunIds = new Map>(); for (const workspaceId of mergeCandidateIds) { - if (workspaceId in activityById) { + if (workspaceId in activityById || !freshSnapshots.has(workspaceId)) { continue; } - const lateSnapshot = freshSnapshots.get(workspaceId) ?? null; - if ( - lateSnapshot == null || - this.extensionMetadata.isWorkspaceDeleted(workspaceId) || - isRemovedFromConfig(workspaceId) || - isRemovedPerAuthoritativeIdentity(workspaceId) - ) { - continue; + probedWorkflowRunIds.set(workspaceId, await this.getActiveWorkflowRunIds(workspaceId)); + } + if (probedWorkflowRunIds.size > 0) { + // Final post-probe views: the probes awaited disk, so a removal + // landing during them is invisible to every view captured above — + // inserting on those alone would ride the deleted id back into the + // renderer. Re-read the (post-prune bounded) metadata file and the + // raw config superset once more, then evaluate every guard with no + // awaits before insertion. A removed workspace's metadata key is + // deleted on removal, so the snapshot re-read also covers legacy + // ids the raw view cannot see; same-process removals are covered + // by the tombstone check. Best-effort like the other fresh + // re-reads: an unreadable view falls back to the pre-probe one. + let finalSnapshots: ReadonlyMap | null = null; + try { + finalSnapshots = await this.extensionMetadata.getAllSnapshots({ throwOnError: true }); + } catch { + finalSnapshots = null; } - // Same overlay path emitWorkspaceActivity uses, except workflow - // runs come from the bootstrapping probe rather than the bare - // cache: a candidate admitted by the fresh config/snapshot re-reads - // never went through the per-id loop above, so its on-disk active - // workflow runs are not cached yet — a cached-only merge would omit - // activeWorkflowRunCount for exactly the cross-process - // registrations this merge exists to bootstrap, and the process- - // local subscription can never deliver the missing delta. Ids the - // per-id loop already probed resolve from the shared cached Set. - const merged = this.mergeCurrentActiveBashMonitorCount( - workspaceId, - mergeActiveWorkflowRuns( - this.overlayPendingGoal(workspaceId, lateSnapshot), - await this.getActiveWorkflowRunIds(workspaceId) - ) - ); - if (merged != null) { - activityById[workspaceId] = merged; + let finalConfigIds: ReadonlySet | null = null; + try { + finalConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + } catch { + finalConfigIds = null; + } + for (const [workspaceId, activeWorkflowRunIds] of probedWorkflowRunIds) { + const lateSnapshot = + finalSnapshots != null + ? (finalSnapshots.get(workspaceId) ?? null) + : (freshSnapshots.get(workspaceId) ?? null); + if ( + lateSnapshot == null || + this.extensionMetadata.isWorkspaceDeleted(workspaceId) || + isRemovedFromConfig(workspaceId) || + isRemovedPerAuthoritativeIdentity(workspaceId) || + // Verifiably deregistered from the raw config during the + // probes (post-probe counterpart of isRemovedFromConfig). + (initialConfigIds != null && + finalConfigIds != null && + initialConfigIds.has(workspaceId) && + !finalConfigIds.has(workspaceId)) + ) { + continue; + } + // Same overlay path emitWorkspaceActivity uses. + const merged = this.mergeCurrentActiveBashMonitorCount( + workspaceId, + mergeActiveWorkflowRuns( + this.overlayPendingGoal(workspaceId, lateSnapshot), + activeWorkflowRunIds + ) + ); + if (merged != null) { + activityById[workspaceId] = merged; + } } } } From e25a2ca151711ddec06399c3ac69e2b8d9f9f851 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 19:49:20 +0000 Subject: [PATCH 35/72] review: address round-32 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - workflow-only late registrations are admitted into the mid-list merge: every fresh raw config id outside the stale per-id scope becomes a merge candidate (a workspace registered after the scope reads can have active workflow runs with no metadata snapshot at all, so fresh-snapshot keys alone cannot admit it); the guard chain replaces the null-snapshot early-drop with an explicit emptiness check plus a snapshot-vanished check so a missing snapshot is only removal evidence when the pre-probe re-read actually had one - the post-probe deregistration guard also covers candidates registered after the initial raw baseline: an id visible in the fresh raw view that admitted it (not just the initial baseline) and gone from the post-probe view is verifiably deregistered — during the normal gap between config deregistration and metadata cleanup the snapshot still exists, so the vanish check cannot catch this case - getAllSnapshots probes the fixed quarantine sidecar path once per process after a successful load and reconciles a leftover: a crash between quarantine's rename and completion followed by another backend recreating a valid partial main leaves no ENOENT or corruption for the existing recovery triggers, stranding the full snapshot in the sidecar forever; failures reset the latch and propagate so the read stays retryable --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- .../services/ExtensionMetadataService.test.ts | 34 +++++ src/node/services/ExtensionMetadataService.ts | 29 ++++ src/node/services/workspaceService.test.ts | 124 ++++++++++++++++++ src/node/services/workspaceService.ts | 50 ++++++- 4 files changed, 230 insertions(+), 7 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 8be88cfb6b..b537a0023f 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -905,6 +905,40 @@ describe("ExtensionMetadataService", () => { expect(snapshots.get("ws-partial")?.recency).toBe(300); }); + test("a strict read reconciles a leftover sidecar next to a recreated valid main", async () => { + // Crash between quarantine's rename and its completion, then another + // backend recreates a VALID partial main from the missing-main window + // before this process starts: no read ever sees ENOENT or corruption, + // so without the once-per-process sidecar check the full snapshot would + // stay stranded in the sidecar forever. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 300, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }) + ); + // Fresh instance: models the restarted process. + const restarted = new ExtensionMetadataService(filePath); + + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-new")?.recency).toBe(300); + expect(snapshots.get("ws-other")?.recency).toBe(42); + // Sidecar consumed by the reconcile. + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("clearTombstonesForRegisteredIds only clears tombstones the evidence postdates", async () => { await service.updateRecency("ws-1", 100); // Evidence snapshot captured BEFORE the removal: a tombstone published diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 81bd017b7c..e7adbbe771 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -71,6 +71,11 @@ export class ExtensionMetadataService { */ private readonly deletedWorkspaceIds = new Set(); + // Once-per-process latch for the leftover-sidecar reconcile in + // getAllSnapshots (see the comment there). Reset on failure so the next + // strict read retries instead of permanently skipping the recovery. + private checkedLeftoverQuarantineSidecar = false; + /** * Serialize all mutating operations on the shared metadata file. * Prevents cross-workspace read-modify-write races since all workspaces @@ -1079,6 +1084,30 @@ export class ExtensionMetadataService { } data = await this.load(options); } + // Once per process: a crash between quarantineCorruptFile's rename and + // its completion can strand the full snapshot in the sidecar while + // another backend recreates a VALID (typically partial, single-entry) + // main file from the missing-main window before this process starts. + // Every other recovery path triggers only on ENOENT or corruption, so a + // valid recreated main would otherwise hide the stranded healthy or + // newer-version data forever. Probe the fixed sidecar path once and + // reconcile before answering; a probe/reconcile failure resets the + // latch and propagates so the read stays retryable rather than + // presenting the partial file as authoritative. Deterministically + // corrupt sidecar leftovers stay put (reconcile leaves them), costing + // one no-op reconcile attempt per process. + if (!this.checkedLeftoverQuarantineSidecar) { + this.checkedLeftoverQuarantineSidecar = true; + try { + if (await this.probeQuarantineSidecar()) { + await this.resumeQuarantineRecovery(); + data = await this.load(options); + } + } catch (error) { + this.checkedLeftoverQuarantineSidecar = false; + throw error; + } + } const map = new Map(); for (const [workspaceId, entry] of Object.entries(data.workspaces)) { const snapshot = this.toSnapshot(entry); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c5e2c07686..f6f11aca44 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3791,6 +3791,130 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList bootstraps workflow-only workspaces registered mid-list", async () => { + // A backend can register a workspace after the scope reads and start a + // workflow WITHOUT writing extension metadata: the fresh snapshot re-read + // never contains the id, so admission must come from the fresh raw + // config view alone — otherwise the workflow-only activity is missing + // from the authoritative response until reconnect. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "workflow-only-late-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + // Registered + workflow started, but NO metadata write. + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const runStore = new WorkflowRunStore({ + sessionDir: config.getSessionDir(workspaceId), + }); + await runStore.createRun({ + id: "wfr_workflow_only", + workspaceId, + workflow: { + name: "demo", + description: "Demo workflow", + scope: "global" as const, + executable: true, + }, + source: "export default function workflow() { return {}; }", + args: {}, + now: "2026-06-17T00:00:00.000Z", + }); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[workspaceId]?.activeWorkflowRunCount).toBe(1); + expect(activityList?.[workspaceId]?.activeWorkflowRunIds).toEqual(["wfr_workflow_only"]); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList drops late additions deregistered during the workflow probe", async () => { + // A workspace registered AFTER the initial raw baseline and removed + // while the workflow probe awaits sits in the normal gap between config + // deregistration and extension-metadata cleanup: its snapshot still + // exists, so only the post-probe raw config re-read (compared against + // the fresh view that admitted it — the initial baseline never saw it) + // can prove the removal. + const { config, historyService, cleanup } = await createTestHistoryService(); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const workspaceId = "late-then-deregistered"; + const projectPath = path.join(config.rootDir, "project"); + const configPath = path.join(config.rootDir, "config.json"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(workspaceId, 888); + } + return realGetAllSnapshots(options); + } + ); + listStatusSnapshotsSpy.mockImplementation(async () => { + // Another backend deregisters the workspace mid-probe; its metadata + // entry intentionally survives (cleanup has not run yet). + await fsPromises.writeFile(configPath, JSON.stringify({ projects: [] })); + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + test("getActivityList drops mid-list additions removed during the workflow probe", async () => { // The workflow-run bootstrap for late merge candidates awaits disk; a // cross-process removal landing during that probe is invisible to every diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 59767d90ad..054fc8ebf9 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13210,6 +13210,19 @@ export class WorkspaceService extends EventEmitter { mergeCandidateIds.add(workspaceId); } } + // Workflow-only late registrations: a workspace registered after the + // scope reads can have active workflow runs but no metadata snapshot + // at all, so admission cannot come from fresh-snapshot keys alone — + // every fresh raw id outside the stale per-id scope is a candidate. + // Ids with neither a snapshot nor live activity cost one workflow + // probe and are dropped by the emptiness check below. + if (freshConfigIds != null) { + for (const workspaceId of freshConfigIds) { + if (!workspaceIds.has(workspaceId)) { + mergeCandidateIds.add(workspaceId); + } + } + } // Workflow-run bootstrap for candidates the per-id loop never saw: // their on-disk active runs are not in the process-local cache, and // a cached-only merge would omit activeWorkflowRunCount for exactly @@ -13219,7 +13232,13 @@ export class WorkspaceService extends EventEmitter { // probed resolve synchronously from the shared cached Set). const probedWorkflowRunIds = new Map>(); for (const workspaceId of mergeCandidateIds) { - if (workspaceId in activityById || !freshSnapshots.has(workspaceId)) { + if ( + workspaceId in activityById || + // In-scope ids without a late snapshot were fully decided by the + // per-id loop (probe + zero-tombstone logic); re-probing them + // would only trigger the final re-reads below on every list. + (workspaceIds.has(workspaceId) && !freshSnapshots.has(workspaceId)) + ) { continue; } probedWorkflowRunIds.set(workspaceId, await this.getActiveWorkflowRunIds(workspaceId)); @@ -13253,16 +13272,33 @@ export class WorkspaceService extends EventEmitter { ? (finalSnapshots.get(workspaceId) ?? null) : (freshSnapshots.get(workspaceId) ?? null); if ( - lateSnapshot == null || + // Nothing to contribute: no persisted snapshot and no live + // counts (a workflow-only candidate legitimately has no + // snapshot, so its absence alone is not removal evidence). + (lateSnapshot == null && + activeWorkflowRunIds.size === 0 && + this.getActiveBashMonitorCount(workspaceId) === 0) || this.extensionMetadata.isWorkspaceDeleted(workspaceId) || isRemovedFromConfig(workspaceId) || isRemovedPerAuthoritativeIdentity(workspaceId) || + // Persisted snapshot vanished during the probes (metadata keys + // only disappear through removal) — also covers legacy ids the + // raw views cannot see. + (finalSnapshots != null && + freshSnapshots.has(workspaceId) && + !finalSnapshots.has(workspaceId)) || // Verifiably deregistered from the raw config during the - // probes (post-probe counterpart of isRemovedFromConfig). - (initialConfigIds != null && - finalConfigIds != null && - initialConfigIds.has(workspaceId) && - !finalConfigIds.has(workspaceId)) + // probes: the id was visible in an EARLIER raw view — the + // initial baseline or the fresh re-read that admitted it (a + // late registration is absent from the initial baseline by + // definition) — and is gone from the post-probe view. During + // the normal gap between config deregistration and metadata + // cleanup the snapshot still exists, so the vanish check + // above cannot catch this. + (finalConfigIds != null && + !finalConfigIds.has(workspaceId) && + ((initialConfigIds?.has(workspaceId) ?? false) || + (freshConfigIds?.has(workspaceId) ?? false))) ) { continue; } From 2e239fa3db5fb6ddc8e1978d784e1b48234706e7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 20:06:53 +0000 Subject: [PATCH 36/72] review: address round-33 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - resumeQuarantineRecovery re-probes the sidecar with ENOENT-only absence semantics (probeQuarantineSidecar) instead of a bare access() that mapped EACCES/EIO to "absent": a transiently unprobeable sidecar now propagates so the once-per-process leftover check resets its latch and retries, rather than accepting a recreated partial main for the process lifetime - writes hitting a tombstoned id revalidate registration through an injected probe (wired in coreServices to the shared config: raw evidence first, authoritative enumeration only when id-less legacy entries exist): a deterministic legacy id re-registered by a downgraded backend writes through immediately and its broadcasts resume, instead of staying suppressed until an activity bootstrap happens to run; negative, missing, or failing probes keep the tombstone (suppression stays the safe default) - the mid-list merge discovers workflow-only legacy registrations: the fresh raw read now returns id evidence, and id-less entries (or an unreadable raw view) trigger the authoritative enumeration whose ids outside the stale scope become merge candidates — covering a downgraded backend registering a raw-invisible workspace with workflow-only activity mid-list - a post-probe authoritative re-enumeration (only when a probed candidate is raw-invisible) drops legacy candidates deregistered while the workflow probe awaited: every raw view is blind to them and the metadata snapshot survives the deregistration gap; the final raw superset read moves after all awaits so the raw guard sees the freshest view --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- .../services/ExtensionMetadataService.test.ts | 72 +++++++++ src/node/services/ExtensionMetadataService.ts | 79 ++++++++-- src/node/services/coreServices.ts | 21 +++ src/node/services/workspaceService.test.ts | 139 ++++++++++++++++++ src/node/services/workspaceService.ts | 73 ++++++++- 5 files changed, 366 insertions(+), 18 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index b537a0023f..727555f9f8 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -939,6 +939,78 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("an unprobeable sidecar during resumed recovery keeps the strict read retryable", async () => { + // The once-per-process leftover check found the sidecar, but the + // re-probe INSIDE the queued recovery transiently fails (EACCES/EIO + // class): reporting it absent would accept the recreated partial main + // and never look at the sidecar again for the process lifetime. The + // failure must propagate (latch reset) so a later read reconciles. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 300, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }) + ); + const restarted = new ExtensionMetadataService(filePath); + const internals = restarted as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(restarted); + let probeCalls = 0; + internals.probeQuarantineSidecar = async () => { + probeCalls += 1; + if (probeCalls === 2) { + // Second probe = the one inside resumeQuarantineRecovery's queue. + internals.probeQuarantineSidecar = originalProbe; + const error = new Error("probe blocked") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + } + return originalProbe(); + }; + + let strictError: unknown = null; + try { + await restarted.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect((strictError as NodeJS.ErrnoException | null)?.code).toBe("EACCES"); + // Latch was reset: the retry probes again and reconciles the sidecar. + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-new")?.recency).toBe(300); + expect(snapshots.get("ws-other")?.recency).toBe(42); + }); + + test("a registration probe clears a stale tombstone and lets the write persist", async () => { + await service.updateRecency("ws-1", 100); + await service.deleteWorkspace("ws-1"); + // No probe wired: tombstones stay strictly write-suppressing. + await service.updateRecency("ws-1", 200); + expect((await service.getAllSnapshots()).has("ws-1")).toBe(false); + // Probe reporting the id NOT registered: still suppressed. + let registered = false; + service.setRegistrationProbe(() => Promise.resolve(registered)); + await service.updateRecency("ws-1", 300); + expect((await service.getAllSnapshots()).has("ws-1")).toBe(false); + expect(service.isWorkspaceDeleted("ws-1")).toBe(true); + // Re-registered (e.g. by a downgraded concurrent backend): the write + // must persist without waiting for an activity bootstrap, and the + // tombstone lifts so broadcasts resume too. + registered = true; + await service.updateRecency("ws-1", 400); + expect((await service.getAllSnapshots()).get("ws-1")?.recency).toBe(400); + expect(service.isWorkspaceDeleted("ws-1")).toBe(false); + }); + test("clearTombstonesForRegisteredIds only clears tombstones the evidence postdates", async () => { await service.updateRecency("ws-1", 100); // Evidence snapshot captured BEFORE the removal: a tombstone published diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index e7adbbe771..9cbd3d922f 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -76,6 +76,20 @@ export class ExtensionMetadataService { // strict read retries instead of permanently skipping the recovery. private checkedLeftoverQuarantineSidecar = false; + /** + * Optional registration probe for writes hitting a tombstoned id (see + * recheckTombstonedRegistration). Wired by the owner that can read the + * shared config (coreServices); absent in bare constructions, where + * tombstones stay strictly write-suppressing. Must answer with CURRENT + * registration evidence or throw — a stale positive would resurrect a + * removed workspace's entry. + */ + private registrationProbe: ((workspaceId: string) => Promise) | null = null; + + setRegistrationProbe(probe: (workspaceId: string) => Promise): void { + this.registrationProbe = probe; + } + /** * Serialize all mutating operations on the shared metadata file. * Prevents cross-workspace read-modify-write races since all workspaces @@ -141,12 +155,46 @@ export class ExtensionMetadataService { return toWorkspaceActivitySnapshot(transient); } + /** + * A write arrived for a tombstoned id: decide whether the tombstone is + * stale. Tombstones are process-local removal knowledge and the shared + * config is the authority — a downgraded concurrent backend can + * legitimately re-register a deterministic legacy id this process pruned, + * and the renderer only calls the activity bootstrap (which also clears + * stale tombstones) when its subscription needs repair, so a healthy + * long-lived process would otherwise suppress the revived workspace's + * writes and broadcasts indefinitely. The probe reads registration + * evidence NOW, strictly after the tombstone was published, so clearing on + * a positive answer is sound (same ordering contract as + * clearTombstonesForRegisteredIds). A negative, missing, or failing probe + * keeps the tombstone: suppression self-heals on a later successful probe + * or bootstrap, while a wrongly persisted write would resurrect a removed + * workspace's entry on disk. + */ + private async recheckTombstonedRegistration(workspaceId: string): Promise { + if (this.registrationProbe == null) { + return false; + } + try { + if (!(await this.registrationProbe(workspaceId))) { + return false; + } + } catch { + return false; + } + this.deletedWorkspaceIds.delete(workspaceId); + return true; + } + private async mutateWorkspaceSnapshot( workspaceId: string, recency: number, mutate: (workspace: ExtensionMetadata) => void ): Promise { - if (this.deletedWorkspaceIds.has(workspaceId)) { + if ( + this.deletedWorkspaceIds.has(workspaceId) && + !(await this.recheckTombstonedRegistration(workspaceId)) + ) { return this.buildTransientSnapshot(workspaceId, recency, mutate); } return this.withSerializedMutation(async () => { @@ -154,7 +202,10 @@ export class ExtensionMetadataService { // tombstones only while its queued mutation runs, so a writer that // passed the pre-queue check and enqueued behind the prune must not // recreate an entry the prune just reclaimed. - if (this.deletedWorkspaceIds.has(workspaceId)) { + if ( + this.deletedWorkspaceIds.has(workspaceId) && + !(await this.recheckTombstonedRegistration(workspaceId)) + ) { return this.buildTransientSnapshot(workspaceId, recency, mutate); } const data = await this.load(); @@ -419,13 +470,21 @@ export class ExtensionMetadataService { options: { skipIfRecencyAdvancedSince?: number | null; inputHash?: string | null } = {} ): Promise { // See deletedWorkspaceIds: never resurrect a removed workspace's entry. - if (this.deletedWorkspaceIds.has(workspaceId)) { + // A stale tombstone (id re-registered by a concurrent backend) is + // cleared via the registration recheck, same as mutateWorkspaceSnapshot. + if ( + this.deletedWorkspaceIds.has(workspaceId) && + !(await this.recheckTombstonedRegistration(workspaceId)) + ) { return null; } return this.withSerializedMutation(async () => { // Re-check inside the queue (see mutateWorkspaceSnapshot): tombstones // published by an already-enqueued prune must be honored here too. - if (this.deletedWorkspaceIds.has(workspaceId)) { + if ( + this.deletedWorkspaceIds.has(workspaceId) && + !(await this.recheckTombstonedRegistration(workspaceId)) + ) { return null; } const data = await this.load(); @@ -909,13 +968,13 @@ export class ExtensionMetadataService { private async resumeQuarantineRecovery(): Promise { await this.withSerializedMutation(async () => { // Re-check inside the queue: a concurrent writer save or a sibling - // strict read may already have recovered the main path. + // strict read may already have recovered the main path. Probe with + // ENOENT-only absence semantics: a transiently unprobeable sidecar + // (EACCES/EIO) must propagate — reporting it absent would let the + // caller accept a recreated partial main and (for the once-per-process + // getAllSnapshots check) never look at the sidecar again. const quarantinePath = `${this.filePath}.corrupt`; - const sidecarExists = await access(quarantinePath).then( - () => true, - () => false - ); - if (!sidecarExists) { + if (!(await this.probeQuarantineSidecar())) { return; } const mainExists = await access(this.filePath).then( diff --git a/src/node/services/coreServices.ts b/src/node/services/coreServices.ts index 5e8e019188..d005d7f88d 100644 --- a/src/node/services/coreServices.ts +++ b/src/node/services/coreServices.ts @@ -98,6 +98,27 @@ export function createCoreServices(opts: CoreServicesOptions): CoreServices { providerService.getConfig() ); const extensionMetadata = new ExtensionMetadataService(extensionMetadataPath); + // Write tombstones are process-local removal knowledge; the shared config + // is the authority (with XUM_ALLOW_MULTIPLE_INSTANCES a downgraded backend + // can legitimately re-register a deterministic legacy id this process + // pruned). Without this probe, a tombstoned id that becomes active again + // would have every metadata write and broadcast suppressed until an + // activity bootstrap happens to run. Raw view first (cheap; complete when + // every persisted entry carries an inline id); only id-less legacy entries + // require the authoritative enumeration. Throws propagate: unknowable + // registration keeps the tombstone. + extensionMetadata.setRegistrationProbe(async (workspaceId) => { + const evidence = config.readPersistedWorkspaceIdEvidence(); + if (evidence.ids.has(workspaceId)) { + return true; + } + if (!evidence.hasWorkspaceEntriesWithoutIds) { + return false; + } + return (await config.getAllWorkspaceMetadata({ throwOnError: true })).some( + (metadata) => metadata.id === workspaceId + ); + }); const workspaceGoalService = new WorkspaceGoalService( config, historyService, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index f6f11aca44..a17821df74 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3856,6 +3856,145 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList bootstraps workflow-only legacy workspaces registered mid-list", async () => { + // Combined raw-invisible + snapshotless case: a downgraded backend + // registers an id-less legacy workspace mid-list and starts a workflow + // WITHOUT writing extension metadata. The stable id appears in neither + // the fresh raw view nor the fresh snapshots, so discovery must come + // from the authoritative enumeration triggered by the raw evidence's + // id-less-entry signal. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "late-legacy-workflow-only"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + const runStore = new WorkflowRunStore({ + sessionDir: config.getSessionDir(stableId), + }); + await runStore.createRun({ + id: "wfr_legacy_only", + workspaceId: stableId, + workflow: { + name: "demo", + description: "Demo workflow", + scope: "global" as const, + executable: true, + }, + source: "export default function workflow() { return {}; }", + args: {}, + now: "2026-06-17T00:00:00.000Z", + }); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[stableId]?.activeWorkflowRunCount).toBe(1); + expect(activityList?.[stableId]?.activeWorkflowRunIds).toEqual(["wfr_legacy_only"]); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList drops legacy additions deregistered during the workflow probe", async () => { + // A raw-invisible legacy workspace admitted mid-list and deregistered + // while the workflow probe awaits: every raw view is blind to it and its + // metadata snapshot survives the deregistration gap, so only the + // post-probe authoritative re-enumeration can prove the removal. + const { config, historyService, cleanup } = await createTestHistoryService(); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const stableId = "late-legacy-then-removed"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + await extensionMetadata.updateRecency(stableId, 999); + } + return realGetAllSnapshots(options); + } + ); + listStatusSnapshotsSpy.mockImplementation(async () => { + // Another backend deregisters the legacy workspace mid-probe; its + // metadata snapshot intentionally survives (cleanup gap). + await fsPromises.writeFile(configPath, JSON.stringify({ projects: [] })); + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[stableId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + test("getActivityList drops late additions deregistered during the workflow probe", async () => { // A workspace registered AFTER the initial raw baseline and removed // while the workflow probe awaits sits in the normal gap between config diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 054fc8ebf9..073eb5cbc0 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13060,10 +13060,19 @@ export class WorkspaceService extends EventEmitter { // fresh evidence if the id really is re-registered. const clearableTombstoneIds = this.extensionMetadata.getTombstonedIds(); let freshConfigIds: ReadonlySet | null = null; + // Whether the fresh raw view is COMPLETE registration evidence: id-less + // legacy entries register raw-invisibly (their stable id lives in + // session metadata.json), so their presence — or an unreadable raw + // view — forces the authoritative enumeration below for admission and + // removal decisions the raw comparison cannot make. + let freshConfigHasRawInvisibleEntries = false; try { - freshConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + const evidence = this.config.readPersistedWorkspaceIdEvidence(); + freshConfigIds = evidence.ids; + freshConfigHasRawInvisibleEntries = evidence.hasWorkspaceEntriesWithoutIds; } catch { freshConfigIds = null; + freshConfigHasRawInvisibleEntries = true; } // Like-for-like raw-superset comparison only: an id that WAS persisted // in config before the awaits and is gone from the fresh raw view was @@ -13111,7 +13120,13 @@ export class WorkspaceService extends EventEmitter { if ( initialConfigIds != null && (entries.some((entry) => entry != null && !initialConfigIds.has(entry[0])) || - hasRawInvisibleLateSnapshotId) + hasRawInvisibleLateSnapshotId || + // Third trigger: id-less legacy entries exist, so a downgraded + // backend may have registered a raw-invisible workspace with + // workflow-only activity (no snapshot) mid-list — only the + // enumeration can discover it for the merge admission below. + // Modern deployments (every id inline) never pay this walk. + freshConfigHasRawInvisibleEntries) ) { try { authoritativeIds = new Set( @@ -13213,11 +13228,14 @@ export class WorkspaceService extends EventEmitter { // Workflow-only late registrations: a workspace registered after the // scope reads can have active workflow runs but no metadata snapshot // at all, so admission cannot come from fresh-snapshot keys alone — - // every fresh raw id outside the stale per-id scope is a candidate. - // Ids with neither a snapshot nor live activity cost one workflow - // probe and are dropped by the emptiness check below. - if (freshConfigIds != null) { - for (const workspaceId of freshConfigIds) { + // every fresh config-derived id outside the stale per-id scope is a + // candidate. The authoritative enumeration contributes the ids the + // raw view can never carry (id-less legacy registrations by a + // downgraded backend). Ids with neither a snapshot nor live activity + // cost one workflow probe and are dropped by the emptiness check + // below. + for (const lateIdSource of [freshConfigIds, authoritativeIds]) { + for (const workspaceId of lateIdSource ?? []) { if (!workspaceIds.has(workspaceId)) { mergeCandidateIds.add(workspaceId); } @@ -13260,6 +13278,36 @@ export class WorkspaceService extends EventEmitter { } catch { finalSnapshots = null; } + // Post-probe authoritative recheck, only when a probed candidate is + // raw-invisible (in no raw view): the raw deregistration guard + // below is blind to a legacy workspace removed during the probes, + // and in the normal gap between config deregistration and metadata + // cleanup its snapshot still exists — the pre-probe authoritative + // set is the view that ADMITTED it, so only a fresh enumeration + // can prove the removal. Modern deployments never pay this walk. + let finalAuthoritativeIds: ReadonlySet | null = null; + if ( + Array.from(probedWorkflowRunIds.keys()).some( + (workspaceId) => + !(initialConfigIds?.has(workspaceId) ?? false) && + !(freshConfigIds?.has(workspaceId) ?? false) + ) + ) { + try { + finalAuthoritativeIds = new Set( + (await this.config.getAllWorkspaceMetadata({ throwOnError: true })).map( + (metadata) => metadata.id + ) + ); + } catch (error) { + log.debug("Failed to re-enumerate authoritative ids after workflow probes", { + error, + }); + finalAuthoritativeIds = null; + } + } + // Read last (synchronous), after every await above, so the raw + // deregistration guard sees the freshest possible view. let finalConfigIds: ReadonlySet | null = null; try { finalConfigIds = this.config.readPersistedWorkspaceIdSuperset(); @@ -13298,7 +13346,16 @@ export class WorkspaceService extends EventEmitter { (finalConfigIds != null && !finalConfigIds.has(workspaceId) && ((initialConfigIds?.has(workspaceId) ?? false) || - (freshConfigIds?.has(workspaceId) ?? false))) + (freshConfigIds?.has(workspaceId) ?? false))) || + // Raw-invisible candidates: post-probe authoritative + // counterpart of the raw guard above — a legacy workspace + // deregistered during the probes is invisible to every raw + // view, and its snapshot may outlive the deregistration. + (!(initialConfigIds?.has(workspaceId) ?? false) && + !(freshConfigIds?.has(workspaceId) ?? false) && + !(finalConfigIds?.has(workspaceId) ?? false) && + finalAuthoritativeIds != null && + !finalAuthoritativeIds.has(workspaceId)) ) { continue; } From 906d0e66fabe0e8405faf40c3455c8e0c7ddb2d8 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 20:24:03 +0000 Subject: [PATCH 37/72] review: address round-34 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sidecar consumption after a successful restore/reconcile propagates non-ENOENT unlink failures instead of swallowing them: a sidecar left behind with recovery reported successful would be reconciled again by a later process after the main file moved on, repeatedly re-merging entries that pruning or removal already reclaimed (ENOENT still means a concurrent recovery consumed it) - reconcile merges per FIELD instead of per key: the recreated main entry is commonly a partial self-heal (a recency write initializes every other field to its default), so key-level main-wins discarded the sidecar entry's model/goal/status and unknown newer-build fields; main still wins every field it carries a non-null value for, null/absent fields fill from the sidecar's complete pre-crash entry - crash-stranded streaming flags never come back from the sidecar: initialize()'s stale-streaming cleanup runs against the main file before the once-per-process reconcile, so merging a sidecar streaming flag verbatim would show an idle workspace as streaming forever; sidecar-only entries are copied with streaming forced off and the streaming field is never filled into existing entries (a genuinely streaming workspace re-asserts the flag with its next write) --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- .../services/ExtensionMetadataService.test.ts | 57 +++++++++++++ src/node/services/ExtensionMetadataService.ts | 84 +++++++++++++++++-- 2 files changed, 132 insertions(+), 9 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 727555f9f8..0c50d3d0fc 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -939,6 +939,63 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("reconcile preserves sidecar fields a partial recreated entry did not supersede", async () => { + // The recreated main entry is commonly a partial self-heal (a recency + // write initializes every other field to its default). Key-level + // main-wins would discard the sidecar entry's model/goal/unknown fields; + // the merge must be per field. Crash-stranded streaming flags must not + // leak back either: initialize()'s stale-streaming cleanup ran against + // the main file before this reconcile. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "ws-1": { recency: 300, streaming: false, lastModel: null, lastThinkingLevel: null }, + }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-1": { + recency: 42, + streaming: true, + lastModel: "anthropic:claude", + lastThinkingLevel: "high", + futureField: { fromNewerBuild: true }, + }, + "ws-2": { recency: 7, streaming: true, lastModel: null, lastThinkingLevel: null }, + }, + }) + ); + const restarted = new ExtensionMetadataService(filePath); + + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-1")?.recency).toBe(300); + expect(snapshots.get("ws-2")?.recency).toBe(7); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record>; + }; + // Main's newer write wins the fields it actually carries... + expect(persisted.workspaces["ws-1"]?.recency).toBe(300); + // ...while unaffected sidecar fields (including unknown newer-build + // fields) survive the reconcile instead of being discarded. + expect(persisted.workspaces["ws-1"]?.lastModel).toBe("anthropic:claude"); + expect(persisted.workspaces["ws-1"]?.lastThinkingLevel).toBe("high"); + expect(persisted.workspaces["ws-1"]?.futureField).toEqual({ fromNewerBuild: true }); + // Crash-leftover streaming flags never come back from the sidecar. + expect(persisted.workspaces["ws-1"]?.streaming).toBe(false); + expect(persisted.workspaces["ws-2"]?.streaming).toBe(false); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("an unprobeable sidecar during resumed recovery keeps the strict read retryable", async () => { // The once-per-process leftover check found the sidecar, but the // re-probe INSIDE the queued recovery transiently fails (EACCES/EIO diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 9cbd3d922f..0cdbd3dbee 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -250,6 +250,24 @@ export class ExtensionMetadataService { } } + /** + * Consume the quarantine sidecar once every surviving entry is represented + * at the main path. ENOENT means a concurrent recovery already consumed + * it; any other failure must propagate (retryable) — reporting success + * with the sidecar still present would let a later process reconcile the + * stale sidecar again after the main file has moved on, repeatedly + * re-merging entries that pruning or removal already reclaimed. + */ + private static async consumeQuarantineSidecar(quarantinePath: string): Promise { + try { + await unlink(quarantinePath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + } + /** * Seam for load()'s missing-main handling (and deterministic TOCTOU tests): * reports whether the quarantine sidecar currently exists. @@ -914,7 +932,7 @@ export class ExtensionMetadataService { throw copyError; } } - await unlink(quarantinePath).catch(() => undefined); + await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath); return false; } // Replace the quarantined main file with a valid EMPTY file instead of @@ -998,9 +1016,11 @@ export class ExtensionMetadataService { * snapshot an older backend self-healed from the missing-main window; * treating it as authoritative would permanently hide every other * workspace's recency/goal/status while the full data sits in the sidecar. - * Merge sidecar-only entries into the main file (main wins per key — its - * writes are newer; ids this process write-tombstoned stay out) and consume - * the sidecar. Only same-schema (version 1) sidecars can be merged: a + * Merge sidecar entries into the main file (per FIELD: main wins fields it + * carries non-null values for — its writes are newer — while null/absent + * fields fill from the sidecar's complete pre-crash entry; ids this + * process write-tombstoned stay out) and consume the sidecar. Only + * same-schema (version 1) sidecars can be merged: a * corrupt sidecar is left as the bounded fixed-name leftover, while a * newer-schema sidecar is restored to the canonical path (superseding the * recreated file, preserved as its own leftover). Must run inside @@ -1061,7 +1081,7 @@ export class ExtensionMetadataService { throw copyError; } } - await unlink(quarantinePath).catch(() => undefined); + await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath); return false; } if (!ExtensionMetadataService.isValidMetadataFileShape(sidecarParsed)) { @@ -1091,18 +1111,64 @@ export class ExtensionMetadataService { } let modified = false; for (const [workspaceId, entry] of Object.entries(sidecarParsed.workspaces)) { - if (workspaceId in main.workspaces || this.deletedWorkspaceIds.has(workspaceId)) { + if (this.deletedWorkspaceIds.has(workspaceId)) { + continue; + } + if (!(workspaceId in main.workspaces)) { + // Sidecar-only entry: by definition no writer touched it since the + // quarantine (a live workspace's writes land in the recreated main), + // so a truthy streaming flag is a crash leftover. initialize()'s + // clearStaleStreaming already ran against the main file before this + // reconcile, so merging the flag verbatim would leave the workspace + // "streaming" forever; a genuinely streaming workspace re-asserts + // the flag with its next write. + main.workspaces[workspaceId] = + entry !== null && + typeof entry === "object" && + !Array.isArray(entry) && + (entry as { streaming?: unknown }).streaming === true + ? { ...entry, streaming: false } + : entry; + modified = true; continue; } - main.workspaces[workspaceId] = entry; - modified = true; + // Same id on both sides: the recreated main entry is commonly a + // PARTIAL self-heal (e.g. a recency write initializes every other + // field to its default), so treating it as wholly authoritative would + // discard the sidecar entry's goal/status/model fields. Field-level + // merge instead: main wins every field it carries a non-null value + // for (its writes are newer); null/absent fields fill from the + // sidecar. An explicit pre-crash clear (null) can be re-filled with a + // stale value — the lesser loss than dropping every unaffected field + // (upgrade↔downgrade data preservation). `streaming` is never filled + // from the sidecar for the crash-leftover reason above. + const target: unknown = main.workspaces[workspaceId]; + if ( + target !== null && + typeof target === "object" && + !Array.isArray(target) && + entry !== null && + typeof entry === "object" && + !Array.isArray(entry) + ) { + const targetRecord = target as unknown as Record; + for (const [field, value] of Object.entries(entry as unknown as Record)) { + if (field === "streaming" || value == null) { + continue; + } + if (targetRecord[field] == null) { + targetRecord[field] = value; + modified = true; + } + } + } } if (modified) { await this.save(main); } // Consumed either way: every surviving sidecar entry is now represented // at the main path. - await unlink(quarantinePath).catch(() => undefined); + await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath); return false; } From 0577f8ea4816e304295d80e9dee4464b40fc58ea Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 20:27:42 +0000 Subject: [PATCH 38/72] review: satisfy no-chained-type-assertions in reconcile field merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Narrow via unknown-typed aliases instead of chained assertions. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- src/node/services/ExtensionMetadataService.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 0cdbd3dbee..3f68fee9c3 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1143,16 +1143,17 @@ export class ExtensionMetadataService { // (upgrade↔downgrade data preservation). `streaming` is never filled // from the sidecar for the crash-leftover reason above. const target: unknown = main.workspaces[workspaceId]; + const sidecarEntry: unknown = entry; if ( target !== null && typeof target === "object" && !Array.isArray(target) && - entry !== null && - typeof entry === "object" && - !Array.isArray(entry) + sidecarEntry !== null && + typeof sidecarEntry === "object" && + !Array.isArray(sidecarEntry) ) { - const targetRecord = target as unknown as Record; - for (const [field, value] of Object.entries(entry as unknown as Record)) { + const targetRecord = target as Record; + for (const [field, value] of Object.entries(sidecarEntry)) { if (field === "streaming" || value == null) { continue; } From 2dd398b02d3888e6c6df562f254870a5ba390c1b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 20:43:33 +0000 Subject: [PATCH 39/72] review: address round-35 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the leftover-sidecar probe runs on every authoritative read instead of once per process: quarantines are cross-process, so a healthy sidecar can be stranded (crash between rename and completion, partial main recreated) at any point in this process's lifetime — a lifetime latch would hide it until restart; the probe is one access() syscall next to the full-file read getAllSnapshots already does, and the reconcile only runs when a sidecar exists - reconcile revalidates tombstoned sidecar entries against current registration before dropping them: the sidecar is consumed afterwards, so a wrong drop is permanent (unlike write suppression, which self-heals) — a re-registered legacy id's only copy now survives; without a probe the local removal knowledge stands, and a FAILING probe aborts the reconcile (retryable) rather than consuming the sidecar on unknowable evidence - tombstones carry per-process generations: clearing paths that await between reading registration evidence and deleting (the write-path registration recheck and the reconcile revalidation) only clear the exact tombstone the evidence preceded — a same-process removal republishing the tombstone mid-probe survives a stale positive, so a parked writer can no longer recreate the entry right after the removal's queued deletion --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- .../services/ExtensionMetadataService.test.ts | 93 ++++++++++++++++++- src/node/services/ExtensionMetadataService.ts | 93 ++++++++++++------- 2 files changed, 149 insertions(+), 37 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 0c50d3d0fc..42a9160910 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -996,12 +996,95 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("a strict read reconciles a sidecar stranded after earlier successful reads", async () => { + // Quarantines are cross-process: another backend can crash mid-quarantine + // (stranding a healthy sidecar) at any point in this process's lifetime, + // not just before startup. A process-lifetime latch would hide that + // sidecar until restart; the probe must run on every authoritative read. + await service.updateRecency("ws-main", 100); + expect((await service.getAllSnapshots({ throwOnError: true })).has("ws-main")).toBe(true); + // Stranded AFTER the first successful read. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }) + ); + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-main")?.recency).toBe(100); + expect(snapshots.get("ws-other")?.recency).toBe(42); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("reconcile revalidates tombstoned sidecar entries against registration", async () => { + // A tombstone may be stale (id re-registered by a downgraded backend) + // and the sidecar holds that workspace's ONLY copy — dropping is + // permanent, unlike write suppression. With the probe denying + // registration the entry is dropped; with the probe confirming it, the + // entry is merged and the tombstone lifted. + await service.updateRecency("ws-legacy", 100); + await service.deleteWorkspace("ws-legacy"); + let registered = false; + service.setRegistrationProbe(() => Promise.resolve(registered)); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-legacy": { recency: 42, streaming: false } }, + }) + ); + expect((await service.getAllSnapshots({ throwOnError: true })).has("ws-legacy")).toBe(false); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(true); + // Re-registered: a re-stranded sidecar entry must survive the reconcile. + registered = true; + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-legacy": { recency: 43, streaming: false } }, + }) + ); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-legacy")?.recency).toBe(43); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(false); + }); + + test("a tombstone published during the registration probe survives a stale positive", async () => { + await service.updateRecency("ws-1", 100); + await service.deleteWorkspace("ws-1"); + let resolveProbe: ((registered: boolean) => void) | null = null; + service.setRegistrationProbe( + () => + new Promise((resolve) => { + resolveProbe = resolve; + }) + ); + // The write parks on the probe (invoked synchronously up to the await). + const write = service.updateRecency("ws-1", 200); + expect(resolveProbe).not.toBeNull(); + // While the probe is in flight, the workspace is removed AGAIN: a newer + // tombstone generation is published. The probe's positive answer is + // stale (it read config before the deregistration) and must not clear + // the newer tombstone — otherwise the parked write recreates the entry + // right after the removal's queued deletion. + await service.deleteWorkspace("ws-1"); + resolveProbe!(true); + await write; + expect(service.isWorkspaceDeleted("ws-1")).toBe(true); + expect((await service.getAllSnapshots()).has("ws-1")).toBe(false); + }); + test("an unprobeable sidecar during resumed recovery keeps the strict read retryable", async () => { - // The once-per-process leftover check found the sidecar, but the - // re-probe INSIDE the queued recovery transiently fails (EACCES/EIO - // class): reporting it absent would accept the recreated partial main - // and never look at the sidecar again for the process lifetime. The - // failure must propagate (latch reset) so a later read reconciles. + // The per-read leftover check found the sidecar, but the re-probe + // INSIDE the queued recovery transiently fails (EACCES/EIO class): + // reporting it absent would silently accept the recreated partial main. + // The failure must propagate (retryable) so a later read reconciles. await writeFile( filePath, JSON.stringify({ diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 3f68fee9c3..19fa3ddbf8 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -69,12 +69,17 @@ export class ExtensionMetadataService { * so a tombstone never blocks a legitimate new workspace; recreations of * legacy fixed-id workspaces happen in a different (downgraded) process. */ - private readonly deletedWorkspaceIds = new Set(); + // Map value is the tombstone's GENERATION (monotonic per process): + // clearing paths that await between reading registration evidence and + // deleting must only clear the exact tombstone the evidence preceded — a + // same-process removal republishing the tombstone mid-probe must survive + // the stale positive (see recheckTombstonedRegistration). + private readonly deletedWorkspaceIds = new Map(); + private tombstoneGeneration = 0; - // Once-per-process latch for the leftover-sidecar reconcile in - // getAllSnapshots (see the comment there). Reset on failure so the next - // strict read retries instead of permanently skipping the recovery. - private checkedLeftoverQuarantineSidecar = false; + private publishTombstone(workspaceId: string): void { + this.deletedWorkspaceIds.set(workspaceId, ++this.tombstoneGeneration); + } /** * Optional registration probe for writes hitting a tombstoned id (see @@ -175,6 +180,13 @@ export class ExtensionMetadataService { if (this.registrationProbe == null) { return false; } + // Capture the tombstone's generation before the probe: a same-process + // removal can republish the tombstone while the probe awaits, and a + // probe that read config just before that deregistration returns a + // stale positive that must not clear the NEWER tombstone (the removal's + // queued deletion may run before the caller's queued write, which would + // then recreate the removed entry). + const generationBefore = this.deletedWorkspaceIds.get(workspaceId); try { if (!(await this.registrationProbe(workspaceId))) { return false; @@ -182,6 +194,9 @@ export class ExtensionMetadataService { } catch { return false; } + if (this.deletedWorkspaceIds.get(workspaceId) !== generationBefore) { + return false; + } this.deletedWorkspaceIds.delete(workspaceId); return true; } @@ -651,7 +666,7 @@ export class ExtensionMetadataService { * clearTombstonesForRegisteredIds as the set of clearable tombstones. */ getTombstonedIds(): ReadonlySet { - return new Set(this.deletedWorkspaceIds); + return new Set(this.deletedWorkspaceIds.keys()); } /** @@ -661,7 +676,7 @@ export class ExtensionMetadataService { async deleteWorkspace(workspaceId: string): Promise { // Synchronously, before the queued mutation: any writer enqueued from now // on must see the tombstone (see deletedWorkspaceIds). - this.deletedWorkspaceIds.add(workspaceId); + this.publishTombstone(workspaceId); await this.withSerializedMutation(async () => { const data = await this.load(); @@ -729,7 +744,7 @@ export class ExtensionMetadataService { for (const workspaceId of staleWorkspaceIds) { // Same guard as deleteWorkspace: a late in-process writer must not // resurrect an entry this pass reclaims. - this.deletedWorkspaceIds.add(workspaceId); + this.publishTombstone(workspaceId); } if (staleWorkspaceIds.length === 0) { return 0; @@ -1112,7 +1127,25 @@ export class ExtensionMetadataService { let modified = false; for (const [workspaceId, entry] of Object.entries(sidecarParsed.workspaces)) { if (this.deletedWorkspaceIds.has(workspaceId)) { - continue; + // The tombstone may be stale: a downgraded backend can re-register a + // deterministic legacy id after this process pruned it, and the + // sidecar may hold that workspace's ONLY copy (consumed below, so a + // wrong drop is permanent — unlike write suppression, which + // self-heals). Revalidate registration first. Without a probe the + // local removal knowledge stands; a FAILING probe aborts the + // reconcile (retryable) rather than consuming the sidecar on + // unknowable evidence. Same generation contract as + // recheckTombstonedRegistration: a tombstone republished mid-probe + // survives the stale positive. + if (this.registrationProbe == null) { + continue; + } + const generationBefore = this.deletedWorkspaceIds.get(workspaceId); + const registered = await this.registrationProbe(workspaceId); + if (!registered || this.deletedWorkspaceIds.get(workspaceId) !== generationBefore) { + continue; + } + this.deletedWorkspaceIds.delete(workspaceId); } if (!(workspaceId in main.workspaces)) { // Sidecar-only entry: by definition no writer touched it since the @@ -1210,29 +1243,25 @@ export class ExtensionMetadataService { } data = await this.load(options); } - // Once per process: a crash between quarantineCorruptFile's rename and - // its completion can strand the full snapshot in the sidecar while - // another backend recreates a VALID (typically partial, single-entry) - // main file from the missing-main window before this process starts. - // Every other recovery path triggers only on ENOENT or corruption, so a - // valid recreated main would otherwise hide the stranded healthy or - // newer-version data forever. Probe the fixed sidecar path once and - // reconcile before answering; a probe/reconcile failure resets the - // latch and propagates so the read stays retryable rather than - // presenting the partial file as authoritative. Deterministically - // corrupt sidecar leftovers stay put (reconcile leaves them), costing - // one no-op reconcile attempt per process. - if (!this.checkedLeftoverQuarantineSidecar) { - this.checkedLeftoverQuarantineSidecar = true; - try { - if (await this.probeQuarantineSidecar()) { - await this.resumeQuarantineRecovery(); - data = await this.load(options); - } - } catch (error) { - this.checkedLeftoverQuarantineSidecar = false; - throw error; - } + // A crash between quarantineCorruptFile's rename and its completion can + // strand the full snapshot in the sidecar while another backend + // recreates a VALID (typically partial, single-entry) main file from + // the missing-main window — leaving no ENOENT or corruption for the + // other recovery triggers. Quarantines are cross-process, so this can + // happen at ANY point in this process's lifetime, not just before + // startup: probe the fixed sidecar path on every authoritative read (a + // process-lifetime latch would hide a sidecar stranded after its first + // read until restart). The probe is one access() syscall next to the + // full-file read this method already does; the reconcile only runs when + // a sidecar actually exists. Failures propagate so the read stays + // retryable rather than presenting a partial file as authoritative. A + // deterministically corrupt leftover (kept for inspection by design) + // costs one no-op reconcile per read while it exists — corruption + // incidents are rare and the leftover is consumed by the next + // quarantine or manual cleanup. + if (await this.probeQuarantineSidecar()) { + await this.resumeQuarantineRecovery(); + data = await this.load(options); } const map = new Map(); for (const [workspaceId, entry] of Object.entries(data.workspaces)) { From 55ced7c02c7b7560ddfd94016287547e773ccb07 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 20:53:49 +0000 Subject: [PATCH 40/72] review: address round-36 finding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - clearTombstonesForRegisteredIds uses the same generation contract as the write-path registration recheck: getTombstonedIds now snapshots id -> generation, and the clear skips tombstones whose generation changed after the snapshot — a same-process removal republishing a tombstone while getActivityList's registration evidence was being gathered survives the stale positive, so a queued late writer can no longer recreate and rebroadcast the removed entry --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- .../services/ExtensionMetadataService.test.ts | 8 ++++++ src/node/services/ExtensionMetadataService.ts | 26 +++++++++++++------ src/node/services/workspaceService.test.ts | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 42a9160910..1c95746ceb 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1160,6 +1160,14 @@ describe("ExtensionMetadataService", () => { await service.deleteWorkspace("ws-1"); service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), preEvidence); expect(service.isWorkspaceDeleted("ws-1")).toBe(true); + // A tombstone REPUBLISHED while the evidence was being gathered must + // also survive: the snapshot carries the old generation, so the stale + // positive clears only the exact tombstone it captured. + const staleGenerationEvidence = service.getTombstonedIds(); + await service.updateRecency("ws-1", 150); // suppressed (still tombstoned) + await service.deleteWorkspace("ws-1"); // republish: newer generation + service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), staleGenerationEvidence); + expect(service.isWorkspaceDeleted("ws-1")).toBe(true); // The next bootstrap snapshots the tombstone before its evidence reads, // so a genuinely re-registered id is cleared then. service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), service.getTombstonedIds()); diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 19fa3ddbf8..1f8d189f19 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -651,22 +651,32 @@ export class ExtensionMetadataService { */ clearTombstonesForRegisteredIds( registeredIds: ReadonlySet, - eligibleIds: ReadonlySet + eligibleIds: ReadonlyMap ): void { - for (const workspaceId of eligibleIds) { - if (registeredIds.has(workspaceId)) { + for (const [workspaceId, generation] of eligibleIds) { + // Generation compare (see recheckTombstonedRegistration): a same- + // process removal can republish the tombstone while the caller's + // registration evidence was being gathered — the stale positive must + // clear only the exact tombstone the snapshot captured, never the + // newer removal's. + if ( + registeredIds.has(workspaceId) && + this.deletedWorkspaceIds.get(workspaceId) === generation + ) { this.deletedWorkspaceIds.delete(workspaceId); } } } /** - * Snapshot of the ids currently write-tombstoned in this process. Capture - * it before gathering registration evidence and pass it back to - * clearTombstonesForRegisteredIds as the set of clearable tombstones. + * Snapshot of the ids currently write-tombstoned in this process, with + * their generations. Capture it before gathering registration evidence and + * pass it back to clearTombstonesForRegisteredIds as the set of clearable + * tombstones — the generation lets the clear skip tombstones republished + * after the snapshot. */ - getTombstonedIds(): ReadonlySet { - return new Set(this.deletedWorkspaceIds.keys()); + getTombstonedIds(): ReadonlyMap { + return new Map(this.deletedWorkspaceIds); } /** diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index a17821df74..bb49abcfb4 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -143,7 +143,7 @@ const mockInitStateManager: Partial = { const mockExtensionMetadataService: Partial = { isWorkspaceDeleted: mock(() => false), clearTombstonesForRegisteredIds: mock(() => undefined), - getTombstonedIds: mock((): ReadonlySet => new Set()), + getTombstonedIds: mock((): ReadonlyMap => new Map()), setStreaming: mock(() => Promise.resolve({ recency: Date.now(), From d30f9e73535e8a5f02d4af20c7a131eb02f3f6cd Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 21:10:10 +0000 Subject: [PATCH 41/72] review: address round-37 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the prune's re-registration spare is generation-guarded: publishTombstone returns the generation, the prune records its own tombstones' generations, and the recheck's pre-removal positive clears only the prune's tombstone — a same-process removal republishing it while the recheck enumeration awaited keeps its newer generation, so a late writer cannot recreate the removed entry after the removal's queued deletion - deleteWorkspace revalidates registration inside the queued deletion via the injected probe: a downgraded backend re-registering a deterministic legacy id after the caller's deregistration checks (or while the deletion waited in the queue) keeps its freshly persisted snapshot, and the tombstone this call published lifts (generation-guarded); an unknowable probe keeps the tombstone but skips the disk deletion (a stale entry is recoverable, destroyed re-registered data is not); without a probe the caller's own deregistration evidence stands --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- .../services/ExtensionMetadataService.test.ts | 69 +++++++++++++++++-- src/node/services/ExtensionMetadataService.ts | 51 ++++++++++++-- 2 files changed, 109 insertions(+), 11 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 1c95746ceb..217f8eadc0 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -996,6 +996,58 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("prune's re-registration spare keeps tombstones republished mid-recheck", async () => { + // The recheck enumeration awaits disk; a same-process removal landing in + // that window republishes the tombstone with a newer generation. The + // enumeration's pre-removal positive must clear only the prune's own + // tombstone — clearing the newer one would let a late writer recreate + // the removed entry right after the removal's queued deletion. + await service.updateRecency("ws-stale", 100); + let removal: Promise | null = null; + await service.pruneMissingWorkspaces( + () => Promise.resolve(new Set()), + () => { + // Removed again in this process while the recheck awaited: + // deleteWorkspace publishes its newer tombstone synchronously and + // queues the deletion behind the running prune. + removal = service.deleteWorkspace("ws-stale"); + // The (stale) enumeration still reports the id registered. + return Promise.resolve(new Set(["ws-stale"])); + } + ); + await removal!; + expect(service.isWorkspaceDeleted("ws-stale")).toBe(true); + // A late writer stays suppressed instead of recreating the entry. + await service.updateRecency("ws-stale", 200); + expect((await service.getAllSnapshots()).has("ws-stale")).toBe(false); + }); + + test("deleteWorkspace revalidates registration inside the queued deletion", async () => { + await service.updateRecency("ws-legacy", 100); + // Probe models a downgraded backend re-registering the id between the + // caller's deregistration checks and the queued deletion: the fresh + // snapshot must survive and the tombstone lift. + service.setRegistrationProbe(() => Promise.resolve(true)); + await service.deleteWorkspace("ws-legacy"); + expect((await service.getAllSnapshots()).get("ws-legacy")?.recency).toBe(100); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(false); + // An unknowable probe keeps the tombstone but never deletes on lossy + // evidence: the entry stays on disk (recoverable) while writes and + // lists are suppressed. + service.setRegistrationProbe(() => Promise.reject(new Error("io"))); + await service.deleteWorkspace("ws-legacy"); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(true); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + expect("ws-legacy" in persisted.workspaces).toBe(true); + // A verified-unregistered id deletes as before (the normal removal). + service.setRegistrationProbe(() => Promise.resolve(false)); + await service.deleteWorkspace("ws-legacy"); + expect((await service.getAllSnapshots()).has("ws-legacy")).toBe(false); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(true); + }); + test("a strict read reconciles a sidecar stranded after earlier successful reads", async () => { // Quarantines are cross-process: another backend can crash mid-quarantine // (stranding a healthy sidecar) at any point in this process's lifetime, @@ -1059,12 +1111,19 @@ describe("ExtensionMetadataService", () => { await service.updateRecency("ws-1", 100); await service.deleteWorkspace("ws-1"); let resolveProbe: ((registered: boolean) => void) | null = null; - service.setRegistrationProbe( - () => - new Promise((resolve) => { + let probeCalls = 0; + service.setRegistrationProbe(() => { + probeCalls += 1; + if (probeCalls === 1) { + // The write's probe: parked until the test resolves it below. + return new Promise((resolve) => { resolveProbe = resolve; - }) - ); + }); + } + // The removal's own in-queue revalidation reads FRESH config + // (deregistered), unlike the write's stale parked probe. + return Promise.resolve(false); + }); // The write parks on the probe (invoked synchronously up to the await). const write = service.updateRecency("ws-1", 200); expect(resolveProbe).not.toBeNull(); diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 1f8d189f19..b491c26f06 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -77,8 +77,10 @@ export class ExtensionMetadataService { private readonly deletedWorkspaceIds = new Map(); private tombstoneGeneration = 0; - private publishTombstone(workspaceId: string): void { - this.deletedWorkspaceIds.set(workspaceId, ++this.tombstoneGeneration); + private publishTombstone(workspaceId: string): number { + const generation = ++this.tombstoneGeneration; + this.deletedWorkspaceIds.set(workspaceId, generation); + return generation; } /** @@ -686,8 +688,33 @@ export class ExtensionMetadataService { async deleteWorkspace(workspaceId: string): Promise { // Synchronously, before the queued mutation: any writer enqueued from now // on must see the tombstone (see deletedWorkspaceIds). - this.publishTombstone(workspaceId); + const publishedGeneration = this.publishTombstone(workspaceId); await this.withSerializedMutation(async () => { + // In-queue registration revalidation: a downgraded backend can + // re-register a deterministic legacy id after the caller's + // deregistration checks (or while this deletion waited in the queue) + // and persist a fresh snapshot the unconditional delete would destroy. + // A probe-confirmed registration aborts the deletion and lifts the + // tombstone THIS call published (generation-guarded — a newer + // removal's tombstone stays). An unknowable probe keeps the tombstone + // and skips the disk deletion: a stale entry is recoverable (filtered + // by the tombstone now, reclaimed by a later removal or process-start + // prune), destroyed re-registered data is not. Without a probe the + // caller's own deregistration evidence stands. + if (this.registrationProbe != null) { + let registered: boolean; + try { + registered = await this.registrationProbe(workspaceId); + } catch { + return; + } + if (registered) { + if (this.deletedWorkspaceIds.get(workspaceId) === publishedGeneration) { + this.deletedWorkspaceIds.delete(workspaceId); + } + return; + } + } const data = await this.load(); // Key presence, not truthiness: malformed falsy persisted entries @@ -751,10 +778,12 @@ export class ExtensionMetadataService { const staleWorkspaceIds = Object.keys(data.workspaces).filter( (workspaceId) => !knownWorkspaceIds.has(workspaceId) ); + const staleTombstoneGenerations = new Map(); for (const workspaceId of staleWorkspaceIds) { // Same guard as deleteWorkspace: a late in-process writer must not - // resurrect an entry this pass reclaims. - this.publishTombstone(workspaceId); + // resurrect an entry this pass reclaims. The generation scopes the + // re-registration spare below to THIS prune's tombstone. + staleTombstoneGenerations.set(workspaceId, this.publishTombstone(workspaceId)); } if (staleWorkspaceIds.length === 0) { return 0; @@ -773,7 +802,17 @@ export class ExtensionMetadataService { let prunedCount = 0; for (const workspaceId of staleWorkspaceIds) { if (recheckedKnownIds.has(workspaceId)) { - this.deletedWorkspaceIds.delete(workspaceId); + // Generation-guarded (see clearTombstonesForRegisteredIds): the + // recheck enumeration awaited disk, and a same-process removal can + // republish this tombstone mid-await — the enumeration's + // pre-removal positive must clear only the prune's own tombstone, + // never the newer removal's (a late writer would otherwise pass + // its in-queue check and recreate the removed entry). + if ( + this.deletedWorkspaceIds.get(workspaceId) === staleTombstoneGenerations.get(workspaceId) + ) { + this.deletedWorkspaceIds.delete(workspaceId); + } continue; } if (workspaceId in fresh.workspaces) { From 6eb75cfd6e978c5cddc939c13ed45025e4eb1b67 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 21:34:39 +0000 Subject: [PATCH 42/72] review: address round-38 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a present but non-array workspaces container (null/primitive/object) in the raw config counts as incomplete registration evidence: the original entries may have been mangled, so destructive callers fall through to the strict loader instead of treating the raw id set as complete - workspace removal and tombstone clearing evict the process-local workflow-run cache/bootstrap and bash-monitor seen set (liftTombstone is now the single clearing exit point and notifies an owner listener): a deterministic legacy id re-registered after removal re-probes disk instead of showing the removed incarnation's ghost activity counts - deleteWorkspace loads the deletion snapshot BEFORE the registration probe (same ordering as the prune): any entry visible in the loaded snapshot was persisted before the load, so a post-load "unregistered" answer postdates that write — closing the window where a re-registration landing between probe and load was deleted - getActivityList refreshes the raw config evidence after the authoritative enumeration await so the like-for-like removal comparison never compares two pre-removal reads - reconcile resolves tombstone revalidation probes BEFORE reading the main file, keeping the merge loop synchronous between the main read and the save so probe awaits cannot cause the save to clobber a concurrent backend's newer write --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- src/node/config.test.ts | 13 +++ src/node/config.ts | 8 +- src/node/services/ExtensionMetadataService.ts | 105 ++++++++++++------ src/node/services/workspaceService.test.ts | 48 ++++++++ src/node/services/workspaceService.ts | 48 ++++++++ 5 files changed, 183 insertions(+), 39 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 5a69fd3931..8bed4d022c 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -957,6 +957,19 @@ describe("Config", () => { const evidence = new Config(tempDir).readPersistedWorkspaceIdEvidence(); expect(evidence.hasWorkspaceEntriesWithoutIds).toBe(true); expect(evidence.ids.has("modern")).toBe(true); + // A PRESENT but malformed (non-array) container is uninterpretable: + // the original entries may have been mangled, so the id set must not + // be treated as complete evidence for destructive decisions. + for (const malformed of [null, "mangled", 7]) { + fs.writeFileSync( + configPath, + JSON.stringify({ projects: [["/repo", { workspaces: malformed }]] }) + ); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(), + hasWorkspaceEntriesWithoutIds: true, + }); + } // Missing file: healthy empty evidence (fresh install). fs.rmSync(configPath); expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ diff --git a/src/node/config.ts b/src/node/config.ts index c175224f8f..486afff1b5 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1169,9 +1169,11 @@ export class Config { if (workspaces.some((entry) => !hasInlineStringId(entry))) { hasWorkspaceEntriesWithoutIds = true; } - } else if (workspaces !== null && typeof workspaces === "object") { - // A workspaces container in an uninterpretable shape may still - // describe registered workspaces; stay conservative. + } else if (workspaces !== undefined) { + // A PRESENT workspaces container in any non-array shape (object, + // null, primitive) is uninterpretable evidence — the original + // entries may have been mangled, so the raw id set cannot be + // proven complete. Only an absent key means "no entries here". hasWorkspaceEntriesWithoutIds = true; } for (const nested of Object.values(value)) { diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index b491c26f06..b8f83431b5 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -83,6 +83,29 @@ export class ExtensionMetadataService { return generation; } + /** + * Single exit point for clearing a tombstone (re-registration revival). + * Notifies the owner so process-local activity caches bootstrapped for the + * REMOVED incarnation (active workflow-run ids, bash-monitor seen set) are + * evicted — workspace removal never evicts them, and the revived + * incarnation's session state on disk may differ, so a stale cache would + * otherwise show ghost activity counts indefinitely. + */ + private liftTombstone(workspaceId: string): void { + this.deletedWorkspaceIds.delete(workspaceId); + try { + this.tombstoneClearedListener?.(workspaceId); + } catch { + // Listener failures must not break the clearing path. + } + } + + private tombstoneClearedListener: ((workspaceId: string) => void) | null = null; + + setTombstoneClearedListener(listener: (workspaceId: string) => void): void { + this.tombstoneClearedListener = listener; + } + /** * Optional registration probe for writes hitting a tombstoned id (see * recheckTombstonedRegistration). Wired by the owner that can read the @@ -199,7 +222,7 @@ export class ExtensionMetadataService { if (this.deletedWorkspaceIds.get(workspaceId) !== generationBefore) { return false; } - this.deletedWorkspaceIds.delete(workspaceId); + this.liftTombstone(workspaceId); return true; } @@ -665,7 +688,7 @@ export class ExtensionMetadataService { registeredIds.has(workspaceId) && this.deletedWorkspaceIds.get(workspaceId) === generation ) { - this.deletedWorkspaceIds.delete(workspaceId); + this.liftTombstone(workspaceId); } } } @@ -690,17 +713,23 @@ export class ExtensionMetadataService { // on must see the tombstone (see deletedWorkspaceIds). const publishedGeneration = this.publishTombstone(workspaceId); await this.withSerializedMutation(async () => { - // In-queue registration revalidation: a downgraded backend can - // re-register a deterministic legacy id after the caller's - // deregistration checks (or while this deletion waited in the queue) - // and persist a fresh snapshot the unconditional delete would destroy. - // A probe-confirmed registration aborts the deletion and lifts the - // tombstone THIS call published (generation-guarded — a newer - // removal's tombstone stays). An unknowable probe keeps the tombstone - // and skips the disk deletion: a stale entry is recoverable (filtered - // by the tombstone now, reclaimed by a later removal or process-start - // prune), destroyed re-registered data is not. Without a probe the - // caller's own deregistration evidence stands. + const data = await this.load(); + // In-queue registration revalidation, strictly AFTER the load (same + // ordering as the prune): a workspace is durably registered before its + // first metadata write, so any entry visible in the loaded snapshot + // was persisted before the load — a post-load probe reporting + // "unregistered" therefore postdates that write and proves the entry + // belongs to the removed incarnation. A downgraded backend + // re-registering a deterministic legacy id (after the caller's + // deregistration checks, or while this deletion waited in the queue) + // keeps its fresh snapshot: the probe-confirmed registration aborts + // the deletion and lifts the tombstone THIS call published + // (generation-guarded — a newer removal's tombstone stays). An + // unknowable probe keeps the tombstone and skips the disk deletion: a + // stale entry is recoverable (filtered by the tombstone now, reclaimed + // by a later removal or process-start prune), destroyed re-registered + // data is not. Without a probe the caller's own deregistration + // evidence stands. if (this.registrationProbe != null) { let registered: boolean; try { @@ -710,13 +739,11 @@ export class ExtensionMetadataService { } if (registered) { if (this.deletedWorkspaceIds.get(workspaceId) === publishedGeneration) { - this.deletedWorkspaceIds.delete(workspaceId); + this.liftTombstone(workspaceId); } return; } } - const data = await this.load(); - // Key presence, not truthiness: malformed falsy persisted entries // (e.g. null) must be deleted too, or removal leaves a stale key // behind until the next process-start prune. @@ -811,7 +838,7 @@ export class ExtensionMetadataService { if ( this.deletedWorkspaceIds.get(workspaceId) === staleTombstoneGenerations.get(workspaceId) ) { - this.deletedWorkspaceIds.delete(workspaceId); + this.liftTombstone(workspaceId); } continue; } @@ -1151,6 +1178,28 @@ export class ExtensionMetadataService { if (!ExtensionMetadataService.isValidMetadataFileShape(sidecarParsed)) { return false; } + // Resolve tombstone revalidations BEFORE reading the main file: the + // probes await config, and holding a pre-probe main snapshot across + // those awaits would let the save below write stale data over a + // concurrent backend's newer write. The tombstone may be stale (a + // downgraded backend can re-register a deterministic legacy id after + // this process pruned it), and the sidecar may hold that workspace's + // ONLY copy — consumed below, so a wrong drop is permanent, unlike + // write suppression, which self-heals. Without a probe the local + // removal knowledge stands; a FAILING probe aborts the reconcile + // (retryable) rather than consuming the sidecar on unknowable evidence. + // Same generation contract as recheckTombstonedRegistration: a + // tombstone republished mid-probe survives the stale positive. + for (const workspaceId of Object.keys(sidecarParsed.workspaces)) { + if (!this.deletedWorkspaceIds.has(workspaceId) || this.registrationProbe == null) { + continue; + } + const generationBefore = this.deletedWorkspaceIds.get(workspaceId); + const registered = await this.registrationProbe(workspaceId); + if (registered && this.deletedWorkspaceIds.get(workspaceId) === generationBefore) { + this.liftTombstone(workspaceId); + } + } let main: ExtensionMetadataFile; try { const parsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; @@ -1176,25 +1225,9 @@ export class ExtensionMetadataService { let modified = false; for (const [workspaceId, entry] of Object.entries(sidecarParsed.workspaces)) { if (this.deletedWorkspaceIds.has(workspaceId)) { - // The tombstone may be stale: a downgraded backend can re-register a - // deterministic legacy id after this process pruned it, and the - // sidecar may hold that workspace's ONLY copy (consumed below, so a - // wrong drop is permanent — unlike write suppression, which - // self-heals). Revalidate registration first. Without a probe the - // local removal knowledge stands; a FAILING probe aborts the - // reconcile (retryable) rather than consuming the sidecar on - // unknowable evidence. Same generation contract as - // recheckTombstonedRegistration: a tombstone republished mid-probe - // survives the stale positive. - if (this.registrationProbe == null) { - continue; - } - const generationBefore = this.deletedWorkspaceIds.get(workspaceId); - const registered = await this.registrationProbe(workspaceId); - if (!registered || this.deletedWorkspaceIds.get(workspaceId) !== generationBefore) { - continue; - } - this.deletedWorkspaceIds.delete(workspaceId); + // Still tombstoned after the (pre-main-read) revalidation pass: + // the local removal knowledge stands. + continue; } if (!(workspaceId in main.workspaces)) { // Sidecar-only entry: by definition no writer touched it since the diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index bb49abcfb4..78d2fde30d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -144,6 +144,7 @@ const mockExtensionMetadataService: Partial = { isWorkspaceDeleted: mock(() => false), clearTombstonesForRegisteredIds: mock(() => undefined), getTombstonedIds: mock((): ReadonlyMap => new Map()), + setTombstoneClearedListener: mock(() => undefined), setStreaming: mock(() => Promise.resolve({ recency: Date.now(), @@ -3856,6 +3857,53 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("a re-registered id does not inherit workflow caches from its removed incarnation", async () => { + // Workspace removal deletes session state without producing terminal + // workflow events, and the process-local run cache was never evicted: + // a deterministic legacy id re-registered by a downgraded backend would + // show the removed incarnation's ghost activeWorkflowRunCount forever + // (the per-id bootstrap returns the cached set without re-probing disk). + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "revived-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Cache a live workflow run for the (unregistered) old incarnation. + await workspaceService.emitWorkflowRunActivity({ + workspaceId, + runId: "wfr_ghost", + status: "running", + }); + // Removal cleanup: deregistered (never in config here), so the entry + // is tombstoned and the process-local caches must be evicted. + await workspaceService.discardExtensionMetadataEntry(workspaceId); + // The downgraded backend re-registers the same id; its session dir has + // no workflow runs. + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + // No ghost count from the removed incarnation's cache: the revived id + // re-probes disk (empty) and stays absent from the list. + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + await cleanup(); + } + }); + test("getActivityList bootstraps workflow-only legacy workspaces registered mid-list", async () => { // Combined raw-invisible + snapshotless case: a downgraded backend // registers an id-less legacy workspace mid-list and starts a workflow diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 073eb5cbc0..3f7ecb0cae 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2300,6 +2300,19 @@ export class WorkspaceService extends EventEmitter { }); this.setupMetadataListeners(); this.setupInitMetadataListeners(); + // A cleared tombstone means a removed id was re-registered (downgraded + // concurrent backend re-creating a deterministic legacy id): evict the + // process-local activity caches bootstrapped for the REMOVED incarnation + // so the revived workspace re-probes disk instead of showing ghost + // workflow-run counts from session state that removal deleted. + // Guarded like the backgroundProcessManager subscriptions above: tests + // may construct WorkspaceService with a partial ExtensionMetadataService + // stub. + if (typeof this.extensionMetadata.setTombstoneClearedListener === "function") { + this.extensionMetadata.setTombstoneClearedListener((workspaceId) => { + this.evictWorkspaceActivityCaches(workspaceId); + }); + } // r63 startup self-heal: reclaim removal tombstones left behind by a // removal whose config deregistration AND tombstone rollback both failed // — otherwise that workspace stays registered but refused every mutation @@ -12812,6 +12825,11 @@ export class WorkspaceService extends EventEmitter { return; } await this.extensionMetadata.deleteWorkspace(workspaceId); + // Removal-side counterpart of the tombstone-cleared eviction: drop the + // process-local workflow/bash-monitor caches for the removed id so a + // later re-registration never inherits activity bootstrapped from + // session state that removal deleted. + this.evictWorkspaceActivityCaches(workspaceId); } catch (error) { log.debug("Failed to prune extension metadata after workspace deregistration", { workspaceId, @@ -12820,6 +12838,17 @@ export class WorkspaceService extends EventEmitter { } } + /** + * Evict process-local activity caches for a removed (or removed-then- + * revived) workspace id. The caches re-bootstrap from disk on next access; + * an in-flight bootstrap keeps populating its orphaned Set harmlessly. + */ + private evictWorkspaceActivityCaches(workspaceId: string): void { + this.activeWorkflowRunIdsByWorkspace.delete(workspaceId); + this.activeWorkflowRunIdBootstrapsByWorkspace.delete(workspaceId); + this.bashMonitorSeenWorkspaces.delete(workspaceId); + } + /** * One-time lazy cleanup for pre-existing deployments: drop * extensionMetadata.json entries whose workspace no longer exists in @@ -13138,6 +13167,25 @@ export class WorkspaceService extends EventEmitter { log.debug("Failed to enumerate authoritative ids for removal revalidation", { error }); authoritativeIds = null; } + // The enumeration awaited disk: refresh the raw view so the + // like-for-like removal comparison below never compares two + // pre-removal reads. An inline-id workspace removed by another + // backend DURING the enumeration is invisible to the pre-await + // freshConfigIds (and deliberately exempt from the authoritative + // check, which skips ids present in the initial baseline), so its + // stale entry would otherwise pass the retained-entry filter with no + // cross-process event to correct it. Fresher raw evidence is + // strictly better for every downstream consumer; the tombstone-clear + // eligibility snapshot was captured before ALL evidence reads, so + // ordering stays sound. + try { + const refreshedEvidence = this.config.readPersistedWorkspaceIdEvidence(); + freshConfigIds = refreshedEvidence.ids; + freshConfigHasRawInvisibleEntries = refreshedEvidence.hasWorkspaceEntriesWithoutIds; + } catch { + freshConfigIds = null; + freshConfigHasRawInvisibleEntries = true; + } } const isRemovedPerAuthoritativeIdentity = (workspaceId: string): boolean => initialConfigIds != null && From fc02661f24f3aa190cc0a934e1f6e2452d14ba96 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 21:49:02 +0000 Subject: [PATCH 43/72] review: address round-39 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the raw config scan collects ids ONLY from direct entries of workspaces arrays: workspace entries carry nested id-bearing objects (e.g. taskPendingGuidance items) whose ids can reference other — including removed — workspaces, and treating those as registered corrupted registration evidence (aborted deletions, lifted tombstones, ghost activity probes) and unbounded the activity scope; under-collection stays safe because unverifiable workspaces containers flip the incompleteness flag, routing callers to the strict enumeration (orphan session cleanup additionally unions the strict enumeration ids) - getSnapshot runs the same leftover-sidecar reconcile as getAllSnapshots (shared reconcileLeftoverSidecarIfPresent helper): live emissions read through getSnapshot after the subscription bootstraps and a healthy subscription never issues another list read, so a recreated partial main beside a stranded sidecar would otherwise feed emitted snapshots (clearing goal/status in the renderer) indefinitely; best-effort there — an unprobeable sidecar logs and falls through rather than blocking the emission, while the strict list read keeps propagating the same failure --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- src/node/config.test.ts | 32 +++++++++++++++++++ src/node/config.ts | 21 ++++++++---- .../services/ExtensionMetadataService.test.ts | 31 ++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 31 ++++++++++++++++-- 4 files changed, 107 insertions(+), 8 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 8bed4d022c..7d1990dd01 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -930,6 +930,38 @@ describe("Config", () => { expect(() => new Config(tempDir).readPersistedWorkspaceIdSuperset()).toThrow(); }); + it("collects only workspace-entry ids, not nested id-bearing objects", () => { + // Workspace entries carry nested id-bearing objects (e.g. + // taskPendingGuidance items) whose ids can reference OTHER — including + // removed — workspaces. Treating those as registered would corrupt + // registration evidence (aborted deletions, lifted tombstones) and + // unbound the activity scope. + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ + projects: [ + [ + "/repo", + { + workspaces: [ + { + id: "ws-live", + path: "/repo/ws", + taskPendingGuidance: [{ id: "removed-workspace-id", message: "hi" }], + parentWorkspaceId: "some-parent", + }, + ], + }, + ], + ], + }) + ); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(["ws-live"]), + hasWorkspaceEntriesWithoutIds: false, + }); + }); + it("reports whether any workspace entry lacks an inline id", () => { // Completeness signal for registration evidence: with inline ids // everywhere, the raw view is complete and callers may skip the diff --git a/src/node/config.ts b/src/node/config.ts index 486afff1b5..f64d740975 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1152,6 +1152,15 @@ export class Config { const id = (value as { id?: unknown }).id; return typeof id === "string" && id.length > 0; }; + // Collect ids ONLY from direct entries of `workspaces` arrays — never + // from arbitrary nested objects. Workspace entries carry nested id-bearing + // objects (e.g. taskPendingGuidance items) whose ids can reference OTHER + // (including removed) workspaces; a whole-subtree scan would report those + // as registered, corrupting registration evidence (aborted deletions, + // lifted tombstones, ghost activity probes) and unbounding the activity + // scope. Under-collection stays safe: any workspaces container whose + // entries cannot be verified (id-less entry, non-array container) flips + // the incompleteness flag, routing callers to the strict enumeration. const collect = (value: unknown): void => { if (Array.isArray(value)) { for (const entry of value) { @@ -1160,14 +1169,14 @@ export class Config { return; } if (value !== null && typeof value === "object") { - const id = (value as { id?: unknown }).id; - if (typeof id === "string" && id.length > 0) { - ids.add(id); - } const workspaces = (value as { workspaces?: unknown }).workspaces; if (Array.isArray(workspaces)) { - if (workspaces.some((entry) => !hasInlineStringId(entry))) { - hasWorkspaceEntriesWithoutIds = true; + for (const entry of workspaces) { + if (hasInlineStringId(entry)) { + ids.add((entry as { id: string }).id); + } else { + hasWorkspaceEntriesWithoutIds = true; + } } } else if (workspaces !== undefined) { // A PRESENT workspaces container in any non-array shape (object, diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 217f8eadc0..e0538b901e 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1048,6 +1048,37 @@ describe("ExtensionMetadataService", () => { expect(service.isWorkspaceDeleted("ws-legacy")).toBe(true); }); + test("live snapshot reads reconcile a stranded sidecar too", async () => { + // After the activity subscription bootstraps, live metadata/workflow + // emissions read via getSnapshot — a healthy subscription never issues + // another list read, so getSnapshot must run the same leftover-sidecar + // reconcile or a recreated partial main would feed emitted snapshots + // indefinitely. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 300, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }) + ); + const restarted = new ExtensionMetadataService(filePath); + + const snapshot = await restarted.getSnapshot("ws-other"); + expect(snapshot?.recency).toBe(42); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("a strict read reconciles a sidecar stranded after earlier successful reads", async () => { // Quarantines are cross-process: another backend can crash mid-quarantine // (stranding a healthy sidecar) at any point in this process's lifetime, diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index b8f83431b5..b6f1599992 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -308,6 +308,22 @@ export class ExtensionMetadataService { } } + /** + * Shared leftover-sidecar recovery for the read paths (authoritative list + * and per-workspace snapshot reads): probe the fixed sidecar path and run + * the resumable recovery when one exists. Returns true when a sidecar was + * found (callers should re-read). Failures propagate; each caller decides + * whether its read contract is strict (list) or best-effort (live + * emissions). + */ + private async reconcileLeftoverSidecarIfPresent(): Promise { + if (!(await this.probeQuarantineSidecar())) { + return false; + } + await this.resumeQuarantineRecovery(); + return true; + } + /** * Seam for load()'s missing-main handling (and deterministic TOCTOU tests): * reports whether the quarantine sidecar currently exists. @@ -642,6 +658,18 @@ export class ExtensionMetadataService { } async getSnapshot(workspaceId: string): Promise { + // Same leftover-sidecar reconcile as getAllSnapshots (see the comment + // there): live emissions read through this path after the subscription + // bootstraps, so without it a recreated partial main would feed emitted + // snapshots (clearing goal/status in the renderer) while the healthy + // subscription never triggers another list read. Best-effort here: an + // unprobeable sidecar must not block a live emission — the strict list + // read propagates the same failure and keeps hydration retryable. + try { + await this.reconcileLeftoverSidecarIfPresent(); + } catch (error) { + log.debug("Leftover sidecar reconcile failed during snapshot read", { error }); + } const data = await this.load(); return this.toSnapshot(data.workspaces[workspaceId]); } @@ -1341,8 +1369,7 @@ export class ExtensionMetadataService { // costs one no-op reconcile per read while it exists — corruption // incidents are rare and the leftover is consumed by the next // quarantine or manual cleanup. - if (await this.probeQuarantineSidecar()) { - await this.resumeQuarantineRecovery(); + if (await this.reconcileLeftoverSidecarIfPresent()) { data = await this.load(options); } const map = new Map(); From 145f18c8daf831036c67790f77efd25195b67869 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 22:05:04 +0000 Subject: [PATCH 44/72] review: address round-40 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - isUnsupportedVersion accepts only structurally plausible forward versions (finite number != 1): a malformed version value (null/string/object) now classifies as deterministic corruption and quarantines/self-heals instead of failing every read and write forever on a file no build can read - the raw evidence scan validates the OUTER config structure: a present non-array projects container, non-pair element, non-object project config, or project config with no workspaces key marks the evidence incomplete (a mangled remnant of real registrations must not read as a definitive negative during sidecar reconciliation) - the mid-list merge is no longer gated on the metadata re-read succeeding: config-proven workflow-only late ids are probed even when freshSnapshots is null (the response is still authoritative and the process-local subscription cannot supply the foreign event); snapshot guards degrade to the views that are available - reconcile restores a healthy sidecar entry over an uncoercible (null/primitive/array) recreated main entry, streaming cleared, instead of consuming the sidecar with no repair - recheckTombstonedRegistration treats an already-lifted tombstone (cleared by the bootstrap or reconcile mid-probe) as "proceed and persist": returning a transient snapshot with the tombstone gone let WorkspaceService broadcast unpersisted state ahead of disk - getActiveWorkflowRunIds re-verifies after every await that the Set it is about to return is still the installed cache, retrying when eviction (removal / tombstone-lift revival) raced the bootstrap — waiters can no longer serve the removed incarnation's detached run set; pathological churn falls back to one detached disk probe --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `high`_ --- src/node/config.test.ts | 23 ++++ src/node/config.ts | 26 +++- .../services/ExtensionMetadataService.test.ts | 92 +++++++++++++ src/node/services/ExtensionMetadataService.ts | 49 +++++-- src/node/services/workspaceService.test.ts | 125 ++++++++++++++++++ src/node/services/workspaceService.ts | 73 ++++++---- 6 files changed, 356 insertions(+), 32 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 7d1990dd01..14dae741f0 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -1008,6 +1008,29 @@ describe("Config", () => { ids: new Set(), hasWorkspaceEntriesWithoutIds: false, }); + // Malformed OUTER structure is incomplete evidence too: a mangled + // projects container / pair / project config may be the remnant of + // real registrations. Only an absent projects key is healthy empty. + for (const projects of [ + null, + {}, + "mangled", + [null], + ["not-a-pair"], + [["/repo", null]], + [["/repo", ["array-config"]]], + [["/repo", {}]], // project config with no workspaces key at all + ]) { + fs.writeFileSync(configPath, JSON.stringify({ projects })); + expect( + new Config(tempDir).readPersistedWorkspaceIdEvidence().hasWorkspaceEntriesWithoutIds + ).toBe(true); + } + fs.writeFileSync(configPath, JSON.stringify({ defaultProjectDir: "/tmp" })); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(), + hasWorkspaceEntriesWithoutIds: false, + }); }); }); diff --git a/src/node/config.ts b/src/node/config.ts index f64d740975..b43a12bbed 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1190,7 +1190,31 @@ export class Config { } } }; - collect((parsedValue as { projects?: unknown }).projects); + // The OUTER structure must be interpretable too, or the id set cannot be + // proven complete: a present non-array `projects`, a non-pair element, a + // non-object project config, or a project config with no workspaces key + // at all may be a mangled remnant of real registrations. Only an absent + // projects key is healthy emptiness (the strict loader accepts it). + const projects = (parsedValue as { projects?: unknown }).projects; + if (projects !== undefined) { + if (!Array.isArray(projects)) { + hasWorkspaceEntriesWithoutIds = true; + } else { + for (const pair of projects) { + const projectConfig: unknown = Array.isArray(pair) ? pair[1] : undefined; + if ( + projectConfig === null || + typeof projectConfig !== "object" || + Array.isArray(projectConfig) || + (projectConfig as { workspaces?: unknown }).workspaces === undefined + ) { + hasWorkspaceEntriesWithoutIds = true; + break; + } + } + } + } + collect(projects); return { ids, hasWorkspaceEntriesWithoutIds }; } diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index e0538b901e..c0aef4237c 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -996,6 +996,98 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("a malformed version value quarantines instead of masquerading as a newer schema", async () => { + // Only a structurally plausible forward version (finite number != 1) + // earns non-destructive preservation. Corruption that mangles the + // version field (null/string/object) must quarantine and self-heal — + // preserving it would fail every read and write forever on a file no + // build can read. + for (const version of [null, "two", {}]) { + await writeFile( + filePath, + JSON.stringify({ version, workspaces: { "ws-1": { recency: 1, streaming: false } } }) + ); + const restarted = new ExtensionMetadataService(filePath); + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + // Quarantined to empty (bytes preserved in the sidecar), not stuck. + expect(snapshots.size).toBe(0); + await rm(`${filePath}.corrupt`, { force: true }); + } + // A plausible forward version stays preserved (non-destructive signal). + const newerFile = JSON.stringify({ version: 2, workspaces: {} }); + await writeFile(filePath, newerFile); + const restarted = new ExtensionMetadataService(filePath); + let strictError: unknown = null; + try { + await restarted.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + expect(await readFile(filePath, "utf-8")).toBe(newerFile); + }); + + test("a write proceeds when another path lifts the tombstone mid-probe", async () => { + await service.updateRecency("ws-1", 100); + await service.deleteWorkspace("ws-1"); + let resolveProbe: ((registered: boolean) => void) | null = null; + service.setRegistrationProbe( + () => + new Promise((resolve) => { + resolveProbe = resolve; + }) + ); + const write = service.updateRecency("ws-1", 200); + expect(resolveProbe).not.toBeNull(); + // The activity bootstrap clears the (re-registered) tombstone while the + // write's probe is still in flight. + service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), service.getTombstonedIds()); + expect(service.isWorkspaceDeleted("ws-1")).toBe(false); + resolveProbe!(true); + await write; + // The write must have PERSISTED: with the tombstone gone, the caller's + // snapshot is broadcast, and a transient (unpersisted) result would + // leave renderer state ahead of disk until restart. + expect((await service.getAllSnapshots()).get("ws-1")?.recency).toBe(200); + }); + + test("reconcile restores a healthy sidecar entry over an uncoercible main entry", async () => { + // The recreated main can carry a malformed value (null/primitive/array) + // under the same key as a healthy sidecar entry: the field merge cannot + // repair it, and the sidecar is consumed — leaving it would permanently + // lose the only valid copy. + await writeFile( + filePath, + JSON.stringify({ version: 1, workspaces: { "ws-1": null, "ws-2": 7 } }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-1": { recency: 42, streaming: true, lastModel: "m", lastThinkingLevel: null }, + "ws-2": { recency: 43, streaming: false, lastModel: null, lastThinkingLevel: null }, + }, + }) + ); + const restarted = new ExtensionMetadataService(filePath); + + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-1")?.recency).toBe(42); + expect(snapshots.get("ws-2")?.recency).toBe(43); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record>; + }; + expect(persisted.workspaces["ws-1"]?.lastModel).toBe("m"); + // Crash-leftover streaming stays cleared on restore. + expect(persisted.workspaces["ws-1"]?.streaming).toBe(false); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("prune's re-registration spare keeps tombstones republished mid-recheck", async () => { // The recheck enumeration awaits disk; a same-process removal landing in // that window republishes the tombstone with a newer generation. The diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index b6f1599992..d5a73fb128 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -219,7 +219,18 @@ export class ExtensionMetadataService { } catch { return false; } - if (this.deletedWorkspaceIds.get(workspaceId) !== generationBefore) { + const generationAfter = this.deletedWorkspaceIds.get(workspaceId); + if (generationAfter === undefined) { + // Another path (activity bootstrap, reconcile) lifted the tombstone + // while the probe was in flight — the id is verifiably registered and + // no newer removal exists, so the write must PERSIST. Returning false + // here would hand the caller an unpersisted transient snapshot that + // WorkspaceService still broadcasts (the tombstone is gone), leaving + // renderer state ahead of disk until restart. + return true; + } + if (generationAfter !== generationBefore) { + // Republished mid-probe: a newer same-process removal wins. return false; } this.liftTombstone(workspaceId); @@ -927,13 +938,17 @@ export class ExtensionMetadataService { * classifies it as non-quarantinable and load() refuses to self-heal it. */ private static isUnsupportedVersion(parsed: unknown): boolean { - return ( - typeof parsed === "object" && - parsed !== null && - !Array.isArray(parsed) && - "version" in parsed && - (parsed as { version?: unknown }).version !== 1 - ); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return false; + } + const version = (parsed as { version?: unknown }).version; + // Only a structurally PLAUSIBLE forward version (a finite number other + // than 1) earns the non-destructive preservation path. A malformed value + // (null, string, object — corruption or a mangled manual edit) must + // classify as deterministic corruption instead: preserving it would + // leave every strict read and lenient writer failing forever on a file + // no build can ever read, rather than quarantining and self-healing. + return typeof version === "number" && Number.isFinite(version) && version !== 1; } private static unsupportedVersionError(): NodeJS.ErrnoException { @@ -1287,6 +1302,24 @@ export class ExtensionMetadataService { // from the sidecar for the crash-leftover reason above. const target: unknown = main.workspaces[workspaceId]; const sidecarEntry: unknown = entry; + if ( + sidecarEntry !== null && + typeof sidecarEntry === "object" && + !Array.isArray(sidecarEntry) && + (target === null || typeof target !== "object" || Array.isArray(target)) + ) { + // Uncoercible main entry (null/primitive/array) shadowing a healthy + // sidecar entry: no field merge is possible, and the sidecar is + // consumed below — leaving the malformed value would permanently + // lose the only valid copy. Restore the sidecar entry (streaming + // cleared, same crash-leftover rule as sidecar-only entries). + main.workspaces[workspaceId] = + (sidecarEntry as { streaming?: unknown }).streaming === true + ? { ...entry, streaming: false } + : entry; + modified = true; + continue; + } if ( target !== null && typeof target === "object" && diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 78d2fde30d..a6bbc6716f 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3857,6 +3857,59 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("a workflow bootstrap evicted mid-flight is retried instead of served detached", async () => { + // Cache eviction (removal / tombstone-lift revival) can race an + // in-flight bootstrap: waiters that captured the pre-eviction Set would + // return the removed incarnation's runs — ghost counts with no terminal + // event to clear them. The read must detect the eviction and re-probe. + const { config, historyService, cleanup } = await createTestHistoryService(); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const workspaceId = "evicted-mid-bootstrap"; + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + }); + const internals = workspaceService as unknown as { + getActiveWorkflowRunIds(id: string): Promise>; + evictWorkspaceActivityCaches(id: string): void; + }; + const releaseFirstScan = createDeferred(); + let scanCalls = 0; + listStatusSnapshotsSpy.mockImplementation(async () => { + scanCalls += 1; + if (scanCalls === 1) { + // Old-incarnation bootstrap: parked, then reports a ghost run. + await releaseFirstScan.promise; + return [ + { + id: "wfr_ghost", + workspaceId, + status: "running" as const, + createdAt: "2026-06-17T00:00:00.000Z", + updatedAt: "2026-06-17T00:00:00.000Z", + }, + ]; + } + // Post-revival probe: the new incarnation has no runs. + return []; + }); + + const read = internals.getActiveWorkflowRunIds(workspaceId); + // Removal + re-registration land while the bootstrap is parked. + internals.evictWorkspaceActivityCaches(workspaceId); + releaseFirstScan.resolve(); + const runIds = await read; + expect(runIds.size).toBe(0); + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + test("a re-registered id does not inherit workflow caches from its removed incarnation", async () => { // Workspace removal deletes session state without producing terminal // workflow events, and the process-local run cache was never evicted: @@ -3904,6 +3957,78 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList bootstraps workflow-only late ids even when the metadata reread fails", async () => { + // The fresh raw config re-read can discover a workflow-only late + // registration while the metadata re-read transiently fails. The list + // still returns an authoritative (non-null) response, and the + // process-local subscription cannot supply the foreign workflow event — + // so the config-proven id must be probed regardless of the failed + // snapshot view. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "workflow-only-late-reread-fails"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 1) { + return realGetAllSnapshots(options); + } + if (snapshotCalls === 2) { + // Registration + workflow start land before the (failing) + // revalidation re-read. + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const runStore = new WorkflowRunStore({ + sessionDir: config.getSessionDir(workspaceId), + }); + await runStore.createRun({ + id: "wfr_reread_fail", + workspaceId, + workflow: { + name: "demo", + description: "Demo workflow", + scope: "global" as const, + executable: true, + }, + source: "export default function workflow() { return {}; }", + args: {}, + now: "2026-06-17T00:00:00.000Z", + }); + } + // Every re-read after the initial one fails transiently. + throw new Error("transient metadata read failure"); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.activeWorkflowRunCount).toBe(1); + expect(activityList?.[workspaceId]?.activeWorkflowRunIds).toEqual(["wfr_reread_fail"]); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("getActivityList bootstraps workflow-only legacy workspaces registered mid-list", async () => { // Combined raw-invisible + snapshotless case: a downgraded backend // registers an id-less legacy workspace mid-list and starts a workflow diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 3f7ecb0cae..90e1ccb6e2 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3814,28 +3814,47 @@ export class WorkspaceService extends EventEmitter { private async getActiveWorkflowRunIds(workspaceId: string): Promise> { assert(workspaceId.length > 0, "getActiveWorkflowRunIds requires workspaceId"); - const cached = this.activeWorkflowRunIdsByWorkspace.get(workspaceId); - if (cached != null) { - const bootstrap = this.activeWorkflowRunIdBootstrapsByWorkspace.get(workspaceId); - if (bootstrap != null) { - await bootstrap; + // Bounded retry: evictWorkspaceActivityCaches (removal, or a tombstone + // lifted for re-registration) can race an in-flight bootstrap. A waiter + // that captured the pre-eviction Set would otherwise return the removed + // incarnation's runs — ghost counts with no future terminal event to + // clear them — so after every await the Set is re-verified as still the + // installed cache and the read restarts when it was evicted. + for (let attempt = 0; attempt < 3; attempt++) { + const cached = this.activeWorkflowRunIdsByWorkspace.get(workspaceId); + if (cached != null) { + const bootstrap = this.activeWorkflowRunIdBootstrapsByWorkspace.get(workspaceId); + if (bootstrap != null) { + await bootstrap; + } + if (this.activeWorkflowRunIdsByWorkspace.get(workspaceId) !== cached) { + continue; + } + return cached; } - return cached; - } - // Install the shared Set before awaiting disk so parallel workflow status events - // mutate the same cache instead of racing to replace each other after bootstrap. - const activeRunIds = new Set(); - this.activeWorkflowRunIdsByWorkspace.set(workspaceId, activeRunIds); - const bootstrap = this.populateActiveWorkflowRunIds(workspaceId, activeRunIds); - this.activeWorkflowRunIdBootstrapsByWorkspace.set(workspaceId, bootstrap); - try { - return await bootstrap; - } finally { - if (this.activeWorkflowRunIdBootstrapsByWorkspace.get(workspaceId) === bootstrap) { - this.activeWorkflowRunIdBootstrapsByWorkspace.delete(workspaceId); + // Install the shared Set before awaiting disk so parallel workflow status events + // mutate the same cache instead of racing to replace each other after bootstrap. + const activeRunIds = new Set(); + this.activeWorkflowRunIdsByWorkspace.set(workspaceId, activeRunIds); + const bootstrap = this.populateActiveWorkflowRunIds(workspaceId, activeRunIds); + this.activeWorkflowRunIdBootstrapsByWorkspace.set(workspaceId, bootstrap); + try { + await bootstrap; + } finally { + if (this.activeWorkflowRunIdBootstrapsByWorkspace.get(workspaceId) === bootstrap) { + this.activeWorkflowRunIdBootstrapsByWorkspace.delete(workspaceId); + } } + if (this.activeWorkflowRunIdsByWorkspace.get(workspaceId) !== activeRunIds) { + continue; + } + return activeRunIds; } + // Eviction churn exhausted the retries (pathological): probe disk once + // more DETACHED so the caller still gets current durable state without + // installing a cache that may itself be mid-eviction. + return this.populateActiveWorkflowRunIds(workspaceId, new Set()); } private async updateActiveWorkflowRunCount(event: { @@ -13253,7 +13272,15 @@ export class WorkspaceService extends EventEmitter { // (it is backed by this process's EventEmitter, so cross-process // writes produce no delta). Merge in-scope additions from the fresh // re-read, subject to the same removal guards as retained entries. - if (freshSnapshots != null) { + // NOT gated on the snapshot re-read succeeding: config-proven late ids + // (workflow-only registrations) must be probed even when the metadata + // re-read transiently failed — the method still returns an + // authoritative (non-null) response in that case, and the + // process-local subscription can never supply the foreign workflow + // event, so skipping the probe would hide that activity until + // reconnect. Snapshot-derived candidates and snapshot guards simply + // degrade to the views that ARE available. + { // Merge scope: the (possibly stale) per-id scope PLUS fresh-snapshot // ids a fresh config-derived view proves registered — a workspace // registered and written between the scope reads and the fresh @@ -13265,7 +13292,7 @@ export class WorkspaceService extends EventEmitter { // mid-list would stay absent until reconnect (the process-local // subscription cannot deliver the cross-process snapshot). const mergeCandidateIds = new Set(workspaceIds); - for (const workspaceId of freshSnapshots.keys()) { + for (const workspaceId of freshSnapshots?.keys() ?? []) { if ( (freshConfigIds?.has(workspaceId) ?? false) || (authoritativeIds?.has(workspaceId) ?? false) @@ -13303,7 +13330,7 @@ export class WorkspaceService extends EventEmitter { // In-scope ids without a late snapshot were fully decided by the // per-id loop (probe + zero-tombstone logic); re-probing them // would only trigger the final re-reads below on every list. - (workspaceIds.has(workspaceId) && !freshSnapshots.has(workspaceId)) + (workspaceIds.has(workspaceId) && !(freshSnapshots?.has(workspaceId) ?? false)) ) { continue; } @@ -13366,7 +13393,7 @@ export class WorkspaceService extends EventEmitter { const lateSnapshot = finalSnapshots != null ? (finalSnapshots.get(workspaceId) ?? null) - : (freshSnapshots.get(workspaceId) ?? null); + : (freshSnapshots?.get(workspaceId) ?? null); if ( // Nothing to contribute: no persisted snapshot and no live // counts (a workflow-only candidate legitimately has no @@ -13381,7 +13408,7 @@ export class WorkspaceService extends EventEmitter { // only disappear through removal) — also covers legacy ids the // raw views cannot see. (finalSnapshots != null && - freshSnapshots.has(workspaceId) && + (freshSnapshots?.has(workspaceId) ?? false) && !finalSnapshots.has(workspaceId)) || // Verifiably deregistered from the raw config during the // probes: the id was visible in an EARLIER raw view — the From a5b39a5a1cca4a53bf6c2dd717bb4ab5a1cc604b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 22:38:48 +0000 Subject: [PATCH 45/72] review: reject missing workspaces key in strict loads; restrict unsupported-version to integers > 1 --- src/node/config.test.ts | 5 +++++ src/node/config.ts | 7 ++++++- src/node/services/ExtensionMetadataService.test.ts | 12 ++++++------ src/node/services/ExtensionMetadataService.ts | 13 +++++++------ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 14dae741f0..a399a0ce21 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -871,6 +871,11 @@ describe("Config", () => { ["non-pair projects entry", { projects: ["not-a-pair"] }], ["non-object project config", { projects: [["/repo", null]] }], ["non-array workspaces", { projects: [["/repo", { workspaces: "bogus" }]] }], + // ProjectConfigSchema always persists `workspaces`; a present project + // entry without the key is mangled state that raw evidence flags as + // incomplete — strict mode must not vouch "authoritatively empty" for + // it (the prune would delete every snapshot of that project). + ["missing workspaces key", { projects: [["/repo", {}]] }], ]; for (const [label, shape] of invalidShapes) { diff --git a/src/node/config.ts b/src/node/config.ts index b43a12bbed..42d0491eaa 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1418,7 +1418,12 @@ export class Config { throw new Error("Config project entries must be objects"); } const workspaces = (projectConfig as { workspaces?: unknown }).workspaces; - if (workspaces !== undefined && !Array.isArray(workspaces)) { + // ProjectConfigSchema persists `workspaces` as a REQUIRED array, + // so a present project entry without the key is mangled state + // (readPersistedWorkspaceIdEvidence flags it incomplete too). + // Accepting it here would hand destructive callers an + // authoritatively-empty workspace set for that project. + if (!Array.isArray(workspaces)) { throw new Error("Config project workspaces must be an array"); } } diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index c0aef4237c..ce839dae5b 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -997,12 +997,12 @@ describe("ExtensionMetadataService", () => { }); test("a malformed version value quarantines instead of masquerading as a newer schema", async () => { - // Only a structurally plausible forward version (finite number != 1) - // earns non-destructive preservation. Corruption that mangles the - // version field (null/string/object) must quarantine and self-heal — - // preserving it would fail every read and write forever on a file no - // build can read. - for (const version of [null, "two", {}]) { + // Only a structurally plausible forward version (integer > 1) earns + // non-destructive preservation. Corruption that mangles the version + // field (null/string/object, or a numeric no schema lineage can produce: + // 0/-1/2.5) must quarantine and self-heal — preserving it would fail + // every read and write forever on a file no build can read. + for (const version of [null, "two", {}, 0, -1, 2.5]) { await writeFile( filePath, JSON.stringify({ version, workspaces: { "ws-1": { recency: 1, streaming: false } } }) diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index d5a73fb128..2a740a42e1 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -942,13 +942,14 @@ export class ExtensionMetadataService { return false; } const version = (parsed as { version?: unknown }).version; - // Only a structurally PLAUSIBLE forward version (a finite number other + // Only a structurally PLAUSIBLE forward version (an integer greater // than 1) earns the non-destructive preservation path. A malformed value - // (null, string, object — corruption or a mangled manual edit) must - // classify as deterministic corruption instead: preserving it would - // leave every strict read and lenient writer failing forever on a file - // no build can ever read, rather than quarantining and self-healing. - return typeof version === "number" && Number.isFinite(version) && version !== 1; + // (null, string, object, or a numeric no schema lineage can produce — + // 0, -1, 2.5) must classify as deterministic corruption instead: + // preserving it would leave every strict read and lenient writer failing + // forever on a file no build can ever read, rather than quarantining and + // self-healing. + return typeof version === "number" && Number.isInteger(version) && version > 1; } private static unsupportedVersionError(): NodeJS.ErrnoException { From 9b3c1cce6383571e084bc8f925e051302655fafd Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 22:54:51 +0000 Subject: [PATCH 46/72] review: revalidate retained activity entries with post-probe final views --- src/node/services/workspaceService.test.ts | 73 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 49 +++++++++++++-- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index a6bbc6716f..c2b8e009ff 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -4284,6 +4284,79 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList drops retained entries removed during late workflow probes", async () => { + // The retained-entry filter runs before the late-candidate workflow + // probes await disk. A cross-process removal of an ALREADY-RETAINED + // workspace landing during those probes is invisible to every view the + // filter used — without the post-probe re-filter the removed id rides + // the response back into the renderer with no event to correct it. + const { config, historyService, cleanup } = await createTestHistoryService(); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const retainedId = "retained-then-removed"; + const lateId = "late-registered"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: retainedId, + name: retainedId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + await extensionMetadata.updateRecency(retainedId, 555); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + // Another backend registers a NEW workspace mid-list so the + // merge has a late candidate whose probe awaits disk. + await config.addWorkspace(projectPath, { + id: lateId, + name: lateId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(lateId, 777); + } + return realGetAllSnapshots(options); + } + ); + let probeCalls = 0; + listStatusSnapshotsSpy.mockImplementation(async () => { + probeCalls += 1; + if (probeCalls === 2) { + // The late candidate's probe is awaited: another backend removes + // the RETAINED workspace, deleting its persisted metadata entry + // unseen by this process's tombstones. + await new ExtensionMetadataService(metadataPath).deleteWorkspace(retainedId); + } + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[lateId]).toBeDefined(); + expect(activityList?.[retainedId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + test("getActivityList drops snapshotless legacy entries removed mid-list", async () => { // A legacy id-less config entry's stable id is resolved authoritatively // during enumeration and can never appear in the raw config-id baseline, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 90e1ccb6e2..591faa98df 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13361,12 +13361,16 @@ export class WorkspaceService extends EventEmitter { // set is the view that ADMITTED it, so only a fresh enumeration // can prove the removal. Modern deployments never pay this walk. let finalAuthoritativeIds: ReadonlySet | null = null; + const isRawInvisible = (workspaceId: string): boolean => + !(initialConfigIds?.has(workspaceId) ?? false) && + !(freshConfigIds?.has(workspaceId) ?? false); if ( - Array.from(probedWorkflowRunIds.keys()).some( - (workspaceId) => - !(initialConfigIds?.has(workspaceId) ?? false) && - !(freshConfigIds?.has(workspaceId) ?? false) - ) + Array.from(probedWorkflowRunIds.keys()).some(isRawInvisible) || + // Retained entries are re-filtered with the final views below + // (they were admitted before the probes awaited), and a + // raw-invisible retained id's removal is provable only through + // the same fresh enumeration. + Object.keys(activityById).some(isRawInvisible) ) { try { finalAuthoritativeIds = new Set( @@ -13389,6 +13393,41 @@ export class WorkspaceService extends EventEmitter { } catch { finalConfigIds = null; } + // The retained-entry filter above ran BEFORE the workflow probes, + // so a removal landing during those awaits is invisible to every + // view it used — the removed workspace would ride the response + // back into the renderer with no cross-process event to correct + // it. Re-apply the removal guards to retained entries with the + // post-probe views (final filtering must follow the last await). + // Removal guards only: zero-count tombstone entries legitimately + // have no snapshot and no live counts, so the probed candidates' + // emptiness check must not run here. + for (const workspaceId of Object.keys(activityById)) { + if ( + this.extensionMetadata.isWorkspaceDeleted(workspaceId) || + // Persisted snapshot vanished during the probes (metadata + // keys only disappear through removal) — also covers legacy + // ids the raw views cannot see. + (finalSnapshots != null && + (snapshots.has(workspaceId) || (freshSnapshots?.has(workspaceId) ?? false)) && + !finalSnapshots.has(workspaceId)) || + // Verifiably deregistered from the raw config during the + // probes: visible in an earlier raw view, gone from the + // post-probe one. + (finalConfigIds != null && + !finalConfigIds.has(workspaceId) && + ((initialConfigIds?.has(workspaceId) ?? false) || + (freshConfigIds?.has(workspaceId) ?? false))) || + // Raw-invisible retained ids: post-probe authoritative + // counterpart of the raw guard above. + (isRawInvisible(workspaceId) && + !(finalConfigIds?.has(workspaceId) ?? false) && + finalAuthoritativeIds != null && + !finalAuthoritativeIds.has(workspaceId)) + ) { + delete activityById[workspaceId]; + } + } for (const [workspaceId, activeWorkflowRunIds] of probedWorkflowRunIds) { const lateSnapshot = finalSnapshots != null From d9627f5088a2c0e447cf1e182e9f852c45bfaa7a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 23:15:01 +0000 Subject: [PATCH 47/72] review: propagate failed sidecar reconcile on strict snapshot reads (emit paths retain last-known state) --- .../services/ExtensionMetadataService.test.ts | 33 +++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 22 ++++++++++--- src/node/services/workspaceGoalService.ts | 16 ++++++++- src/node/services/workspaceService.ts | 26 ++++++++++++--- 4 files changed, 86 insertions(+), 11 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index ce839dae5b..7c86647c3b 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -905,6 +905,39 @@ describe("ExtensionMetadataService", () => { expect(snapshots.get("ws-partial")?.recency).toBe(300); }); + test("a strict snapshot read propagates a failed sidecar reconcile instead of the partial main", async () => { + // Live emissions read per-workspace snapshots after the subscription + // bootstraps. With a sidecar stranded next to a recreated partial main + // and the reconcile failing transiently, emitting the partial main + // would clear goal/status in the renderer with no guaranteed + // strict-list retry — strict readers (the emit paths) must skip the + // emit by propagating, while lenient readers (settings/eligibility) + // keep availability. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-partial": { recency: 300, streaming: false } }, + }) + ); + // A directory at the sidecar path yields a deterministic errno (EISDIR) + // standing in for EACCES/EIO-class reconcile failures. + await mkdir(`${filePath}.corrupt`); + + let strictError: unknown = null; + try { + await service.getSnapshot("ws-partial", { throwOnError: true }); + } catch (error) { + strictError = error; + } + expect((strictError as NodeJS.ErrnoException | null)?.code).toBe("EISDIR"); + // Lenient readers keep availability on the same state. + expect((await service.getSnapshot("ws-partial"))?.recency).toBe(300); + // Once the sidecar becomes readable (here: gone), the strict read heals. + await rm(`${filePath}.corrupt`, { recursive: true }); + expect((await service.getSnapshot("ws-partial", { throwOnError: true }))?.recency).toBe(300); + }); + test("a strict read reconciles a leftover sidecar next to a recreated valid main", async () => { // Crash between quarantine's rename and its completion, then another // backend recreates a VALID partial main from the missing-main window diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 2a740a42e1..0854dbb394 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -668,20 +668,32 @@ export class ExtensionMetadataService { }); } - async getSnapshot(workspaceId: string): Promise { + async getSnapshot( + workspaceId: string, + options?: { throwOnError?: boolean } + ): Promise { // Same leftover-sidecar reconcile as getAllSnapshots (see the comment // there): live emissions read through this path after the subscription // bootstraps, so without it a recreated partial main would feed emitted // snapshots (clearing goal/status in the renderer) while the healthy - // subscription never triggers another list read. Best-effort here: an - // unprobeable sidecar must not block a live emission — the strict list - // read propagates the same failure and keeps hydration retryable. + // subscription never triggers another list read. + // Strict callers (the emit paths) must also propagate a FAILED + // reconcile instead of reading through it: with a sidecar present the + // main file is suspect (typically a partial recreation), and emitting + // it would clear goal/status in the renderer with no guaranteed + // strict-list retry to repair — skipping the emit retains the + // renderer's last-known snapshot until the reconcile succeeds. Lenient + // callers (settings/eligibility readers) keep availability: a stranded + // sidecar must never block message sending or heartbeats. try { await this.reconcileLeftoverSidecarIfPresent(); } catch (error) { + if (options?.throwOnError) { + throw error; + } log.debug("Leftover sidecar reconcile failed during snapshot read", { error }); } - const data = await this.load(); + const data = await this.load(options); return this.toSnapshot(data.workspaces[workspaceId]); } diff --git a/src/node/services/workspaceGoalService.ts b/src/node/services/workspaceGoalService.ts index 5b220db50f..b1b74e9b0f 100644 --- a/src/node/services/workspaceGoalService.ts +++ b/src/node/services/workspaceGoalService.ts @@ -1281,7 +1281,21 @@ export class WorkspaceGoalService { workspaceId: string, snapshot: GoalSnapshot ): Promise { - const activity = await this.extensionMetadata.getSnapshot(workspaceId); + let activity: WorkspaceActivitySnapshot | null; + try { + activity = await this.extensionMetadata.getSnapshot(workspaceId, { throwOnError: true }); + } catch (error) { + // A suspect baseline (failed sidecar reconcile / unreadable main) + // must not feed an emitted overlay: partial-main fields would clear + // status in the renderer. Report non-delivery instead — callers that + // must guarantee delivery fall back to pushSnapshot, whose lenient + // write path self-heals. + log.debug("Skipping transient goal emit after failed snapshot read", { + workspaceId, + error, + }); + return false; + } if (!activity) { // No baseline activity snapshot to overlay the transient goal on // (extensionMetadata has no entry for this workspace yet). Callers diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 591faa98df..cea5659bd2 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2042,7 +2042,10 @@ export class WorkspaceService extends EventEmitter { // the clear. Cost: an occasional duplicate emit, which renderers apply idempotently. this.lastEmittedBashMonitorCounts.delete(workspaceId); void this.extensionMetadata - .getSnapshot(workspaceId) + // Strict: emitting a suspect (partial-main) snapshot after a failed + // sidecar reconcile would clear goal/status in the renderer; the + // catch below already retains "unknown" and re-emits on next change. + .getSnapshot(workspaceId, { throwOnError: true }) .then((snapshot) => { this.emitWorkspaceActivity(workspaceId, snapshot); // Record only after a successful emit. Re-read the count because the emit merges @@ -3920,10 +3923,23 @@ export class WorkspaceService extends EventEmitter { assert(event.workspaceId.length > 0, "emitWorkflowRunActivity requires workspaceId"); assert(event.runId.length > 0, "emitWorkflowRunActivity requires runId"); await this.updateActiveWorkflowRunCount(event); - this.emitWorkspaceActivity( - event.workspaceId, - await this.extensionMetadata.getSnapshot(event.workspaceId) - ); + let snapshot: WorkspaceActivitySnapshot | null; + try { + snapshot = await this.extensionMetadata.getSnapshot(event.workspaceId, { + throwOnError: true, + }); + } catch (error) { + // Emitting a suspect (partial-main) snapshot after a failed sidecar + // reconcile would clear goal/status in the renderer with no repair + // event. Retention is recoverable: the run count is already cached, + // so the next emit or list read delivers it. + log.debug("Skipping workflow-run activity emit after failed snapshot read", { + workspaceId: event.workspaceId, + error, + }); + return; + } + this.emitWorkspaceActivity(event.workspaceId, snapshot); } /** From a7bf4f1490d85623ecf376e98e7cdc1be60b98af Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 23:29:06 +0000 Subject: [PATCH 48/72] review: use observed nonzero workflow activity, not cache presence, as the zero-count tombstone signal --- src/node/services/workspaceService.test.ts | 37 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 30 ++++++++++++++++-- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c2b8e009ff..7761966cd9 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3213,6 +3213,43 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("repeat lists keep omitting idle workspaces after the first list installs caches", async () => { + // The first list's workflow probe installs an empty run cache for every + // scoped id. Cache initialization must not read as activity: treating it + // as the zero-count tombstone signal would emit a fabricated recency:0 + // entry for every idle config-known workspace from the second list on, + // re-bloating exactly the payload this scoping trims. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "activity-scoping-idle"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const firstList = await workspaceService.getActivityList(); + expect(firstList).not.toBeNull(); + expect(firstList?.[workspaceId]).toBeUndefined(); + const secondList = await workspaceService.getActivityList(); + expect(secondList).not.toBeNull(); + expect(secondList?.[workspaceId]).toBeUndefined(); + } finally { + await cleanup(); + } + }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { // getAllWorkspaceMetadata walks every workspace with per-workspace disk // probes; the latency-sensitive first bootstrap must not pay it twice. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index cea5659bd2..5b9e7ae845 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2070,6 +2070,13 @@ export class WorkspaceService extends EventEmitter { Promise> >(); private readonly activeWorkflowRunIdsByWorkspace = new Map>(); + // Workspaces where NONZERO workflow-run activity was actually observed + // (bootstrap probe, run-status event, or list read). This — not cache + // presence — is the zero-count tombstone signal: the activity list's own + // probe installs an (empty) cache for every scoped id, so cache existence + // would fabricate recency:0 entries for every idle config-known workspace + // from the second list on, re-bloating the payload this scoping trims. + private readonly workflowRunSeenWorkspaces = new Set(); // Debounce post-compaction metadata refreshes (file_edit_* can fire rapidly) private readonly postCompactionRefreshTimers = new Map>(); @@ -3816,6 +3823,16 @@ export class WorkspaceService extends EventEmitter { } private async getActiveWorkflowRunIds(workspaceId: string): Promise> { + const activeRunIds = await this.resolveActiveWorkflowRunIds(workspaceId); + if (activeRunIds.size > 0) { + // A list- or event-delivered nonzero count is what makes a later zero + // meaningful as a tombstone (see workflowRunSeenWorkspaces). + this.workflowRunSeenWorkspaces.add(workspaceId); + } + return activeRunIds; + } + + private async resolveActiveWorkflowRunIds(workspaceId: string): Promise> { assert(workspaceId.length > 0, "getActiveWorkflowRunIds requires workspaceId"); // Bounded retry: evictWorkspaceActivityCaches (removal, or a tombstone // lifted for re-registration) can race an in-flight bootstrap. A waiter @@ -3868,6 +3885,7 @@ export class WorkspaceService extends EventEmitter { const activeRunIds = await this.getActiveWorkflowRunIds(event.workspaceId); if (isActiveWorkflowRunStatus(event.status)) { activeRunIds.add(event.runId); + this.workflowRunSeenWorkspaces.add(event.workspaceId); } else { activeRunIds.delete(event.runId); } @@ -12882,6 +12900,7 @@ export class WorkspaceService extends EventEmitter { this.activeWorkflowRunIdsByWorkspace.delete(workspaceId); this.activeWorkflowRunIdBootstrapsByWorkspace.delete(workspaceId); this.bashMonitorSeenWorkspaces.delete(workspaceId); + this.workflowRunSeenWorkspaces.delete(workspaceId); } /** @@ -13043,6 +13062,9 @@ export class WorkspaceService extends EventEmitter { for (const workspaceId of this.bashMonitorSeenWorkspaces) { workspaceIds.add(workspaceId); } + for (const workspaceId of this.workflowRunSeenWorkspaces) { + workspaceIds.add(workspaceId); + } } } @@ -13051,7 +13073,11 @@ export class WorkspaceService extends EventEmitter { workspaceIds, async (workspaceId): Promise => { const snapshot = snapshots.get(workspaceId) ?? null; - const hadWorkflowActivityCache = this.activeWorkflowRunIdsByWorkspace.has(workspaceId); + // Nonzero-observation signal, NOT cache presence: the probe below + // installs an empty cache for every scoped id, which would turn + // every idle config-known workspace into a fabricated recency:0 + // entry on the next list. + const hadWorkflowActivity = this.workflowRunSeenWorkspaces.has(workspaceId); // Bash-monitor counterpart of the workflow tombstone: a monitor that stopped // while the renderer was disconnected (or whose stop emit failed) must still // surface a zero-count entry here, otherwise the renderer's last-known @@ -13071,7 +13097,7 @@ export class WorkspaceService extends EventEmitter { if ( snapshot == null && activeWorkflowRunCount === 0 && - !hadWorkflowActivityCache && + !hadWorkflowActivity && activeBashMonitorCount === 0 && !hadBashMonitorActivityCache ) { From 5ef0e9b3f21ad4c67ae0006ec0bfdda6ef47b3f4 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 26 Aug 2026 23:51:38 +0000 Subject: [PATCH 49/72] review: guard seen marker against eviction races, retry config baseline, enumerate basename-backed legacy ids --- src/node/config.test.ts | 36 ++++++ src/node/config.ts | 38 +++--- src/node/services/workspaceService.test.ts | 127 +++++++++++++++++++++ src/node/services/workspaceService.ts | 58 ++++++++-- 4 files changed, 238 insertions(+), 21 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index a399a0ce21..f988e6b450 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -3173,6 +3173,42 @@ describe("Config", () => { expect(workspace.name).toBe(workspaceName); expect(workspace.createdAt).toBe("2025-01-01T00:00:00.000Z"); }); + + it("enumerates basename-backed legacy stable ids like findWorkspace", async () => { + // Oldest layout: an id-less config entry whose stable id lives in + // sessions//metadata.json. findWorkspace resolves + // it (basename candidate first), so the enumeration must report the + // same identity — destructive callers (the extension-metadata prune) + // classify ids as stale against the enumeration, and a mismatch would + // delete the live workspace's activity data. + const projectPath = "/fake/project"; + const workspaceName = "old-feature"; + const workspacePath = path.join(config.srcDir, "project", workspaceName); + fs.mkdirSync(workspacePath, { recursive: true }); + + const sessionDir = config.getSessionDir(workspaceName); + fs.mkdirSync(sessionDir, { recursive: true }); + fs.writeFileSync( + path.join(sessionDir, "metadata.json"), + JSON.stringify({ + id: "stable-basename-id", + name: workspaceName, + projectName: "project", + projectPath, + createdAt: "2025-01-01T00:00:00.000Z", + }) + ); + + await config.editConfig((cfg) => { + cfg.projects.set(projectPath, { + workspaces: [{ path: workspacePath }], + }); + return cfg; + }); + + const allMetadata = await config.getAllWorkspaceMetadata({ throwOnError: true }); + expect(allMetadata.map((metadata) => metadata.id)).toContain("stable-basename-id"); + }); }); describe("transcriptOnly derivation", () => { diff --git a/src/node/config.ts b/src/node/config.ts index 42d0491eaa..98a11a5a6b 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2640,23 +2640,35 @@ export class Config { continue; // Skip metadata file lookup } - // LEGACY FORMAT: Fall back to reading metadata.json - // Try legacy ID format first (project-workspace) - used by E2E tests and old workspaces + // LEGACY FORMAT: Fall back to reading metadata.json. + // findWorkspace resolves an id-less entry's stable id from EITHER + // sessions//metadata.json (old layout, checked + // first there) or sessions//metadata.json. + // Enumerate the same candidates in the same order: destructive + // callers (the extension-metadata prune) classify ids as stale + // against THIS enumeration, so a basename-backed stable id that + // findWorkspace can resolve but this walk cannot would be deleted + // as unknown. const legacyId = this.generateLegacyId(projectPath, workspace.path); - const metadataPath = path.join(this.getSessionDir(legacyId), "metadata.json"); let metadataFound = false; + let metadataPath = ""; let legacyMetadataRaw: string | undefined; - try { - legacyMetadataRaw = fs.readFileSync(metadataPath, "utf-8"); - } catch (readError) { - // Missing is normal (most entries never had a legacy - // metadata.json). Any other failure means the workspace's - // authoritative stable id is unknowable right now — surface it - // to the catch below instead of silently substituting the - // generated legacy path id via the !metadataFound branch. - if (!isEnoentError(readError)) { - throw readError; + for (const candidateId of [workspaceBasename, legacyId]) { + const candidatePath = path.join(this.getSessionDir(candidateId), "metadata.json"); + try { + legacyMetadataRaw = fs.readFileSync(candidatePath, "utf-8"); + metadataPath = candidatePath; + break; + } catch (readError) { + // Missing is normal (most entries never had a legacy + // metadata.json). Any other failure means the workspace's + // authoritative stable id is unknowable right now — surface it + // to the catch below instead of silently substituting the + // generated legacy path id via the !metadataFound branch. + if (!isEnoentError(readError)) { + throw readError; + } } } if (legacyMetadataRaw !== undefined) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7761966cd9..0360b9de1b 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3250,6 +3250,133 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("a run-status event racing cache eviction does not strand the seen marker", async () => { + // An eviction (removal, or a tombstone lifted for revival) can land in + // the microtask gap after the run cache resolves. The status event must + // retry against the freshly installed cache instead of mutating the + // detached set and marking the seen set — a stale marker would fabricate + // zero-count entries for the idle revived workspace on every later list. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "evict-race"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const internals = workspaceService as unknown as { + getActiveWorkflowRunIds(workspaceId: string): Promise>; + evictWorkspaceActivityCaches(workspaceId: string): void; + }; + const realGetActiveWorkflowRunIds = internals.getActiveWorkflowRunIds.bind(workspaceService); + let evicted = false; + internals.getActiveWorkflowRunIds = async (targetWorkspaceId: string) => { + const result = await realGetActiveWorkflowRunIds(targetWorkspaceId); + if (!evicted && targetWorkspaceId === workspaceId) { + evicted = true; + // Lands after the cache read resolved, before the caller's + // continuation — the exact revival-eviction window. + internals.evictWorkspaceActivityCaches(targetWorkspaceId); + } + return result; + }; + + await workspaceService.emitWorkflowRunActivity({ + workspaceId, + runId: "wfr_race", + status: "running", + }); + const activity = (await workspaceService.getActivityList())?.[workspaceId]; + // The retried update must land the run in the INSTALLED cache (not a + // detached pre-eviction set that leaves only the stale seen marker). + expect(activity?.activeWorkflowRunCount).toBe(1); + } finally { + await cleanup(); + } + }); + + test("getActivityList re-establishes the config baseline after a transient initial read failure", async () => { + // The pre-await baseline read can fail transiently while the strict + // scoping enumeration succeeds. Without a replacement baseline both + // cross-process removal guards stay disabled on an authoritative + // response: a workspace another backend deregisters during the workflow + // probes (its metadata entry still present in the normal cleanup gap) + // would ride back into the renderer with no event to correct it. + const { config, historyService, cleanup } = await createTestHistoryService(); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const workspaceId = "baseline-retry"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 321); + const realSuperset = config.readPersistedWorkspaceIdSuperset.bind(config); + let failedOnce = false; + const supersetSpy = spyOn(config, "readPersistedWorkspaceIdSuperset").mockImplementation( + () => { + if (!failedOnce) { + failedOnce = true; + throw new Error("transient config read failure"); + } + return realSuperset(); + } + ); + let removedFromConfig = false; + listStatusSnapshotsSpy.mockImplementation(async () => { + if (!removedFromConfig) { + removedFromConfig = true; + // Another backend deregisters the workspace while the per-id + // probe awaits; its metadata entry intentionally stays behind. + const configPath = path.join(config.rootDir, "config.json"); + const parsed = JSON.parse(await fsPromises.readFile(configPath, "utf-8")) as { + projects?: Array<[string, { workspaces?: Array<{ id?: string }> }]>; + }; + for (const [, projectConfig] of parsed.projects ?? []) { + projectConfig.workspaces = (projectConfig.workspaces ?? []).filter( + (workspace) => workspace.id !== workspaceId + ); + } + await fsPromises.writeFile(configPath, JSON.stringify(parsed)); + } + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + supersetSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { // getAllWorkspaceMetadata walks every workspace with per-workspace disk // probes; the latency-sensitive first bootstrap must not pay it twice. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 5b9e7ae845..787283f2f7 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3824,7 +3824,15 @@ export class WorkspaceService extends EventEmitter { private async getActiveWorkflowRunIds(workspaceId: string): Promise> { const activeRunIds = await this.resolveActiveWorkflowRunIds(workspaceId); - if (activeRunIds.size > 0) { + if ( + activeRunIds.size > 0 && + // Installation re-check in THIS continuation: an eviction (removal, or + // a tombstone lifted for revival) can land in the microtask gap after + // resolve's own final check. A detached set must not repopulate the + // seen marker — the evicted id's caches are empty, so a stale marker + // would fabricate zero-count entries for the idle revived workspace. + this.activeWorkflowRunIdsByWorkspace.get(workspaceId) === activeRunIds + ) { // A list- or event-delivered nonzero count is what makes a later zero // meaningful as a tombstone (see workflowRunSeenWorkspaces). this.workflowRunSeenWorkspaces.add(workspaceId); @@ -3882,14 +3890,32 @@ export class WorkspaceService extends EventEmitter { runId: string; status: WorkflowRunStatus; }): Promise { - const activeRunIds = await this.getActiveWorkflowRunIds(event.workspaceId); - if (isActiveWorkflowRunStatus(event.status)) { - activeRunIds.add(event.runId); - this.workflowRunSeenWorkspaces.add(event.workspaceId); - } else { - activeRunIds.delete(event.runId); + // Bounded retry (same reason as resolveActiveWorkflowRunIds): an + // eviction can land in the microtask gap after the cache read resolves, + // so the mutation below would hit a detached incarnation — and must not + // mark the seen set for an id whose caches were just evicted, or the + // idle revived workspace emits fabricated zero-count entries forever. + let detachedSize = 0; + for (let attempt = 0; attempt < 3; attempt++) { + const activeRunIds = await this.getActiveWorkflowRunIds(event.workspaceId); + if (isActiveWorkflowRunStatus(event.status)) { + activeRunIds.add(event.runId); + } else { + activeRunIds.delete(event.runId); + } + detachedSize = activeRunIds.size; + if (this.activeWorkflowRunIdsByWorkspace.get(event.workspaceId) !== activeRunIds) { + continue; + } + if (isActiveWorkflowRunStatus(event.status)) { + this.workflowRunSeenWorkspaces.add(event.workspaceId); + } + return activeRunIds.size; } - return activeRunIds.size; + // Eviction churn exhausted the retries (pathological): report the last + // detached mutation's size without seen-marking, mirroring the detached + // disk-probe fallback in resolveActiveWorkflowRunIds. + return detachedSize; } private mergeCachedActiveWorkflowRuns( @@ -13068,6 +13094,22 @@ export class WorkspaceService extends EventEmitter { } } + // Re-establish the raw baseline when the pre-await read failed: with a + // null baseline BOTH cross-process removal guards stay disabled while + // the response is still authoritative — a workspace another backend + // removes during the probes below would ride back into the renderer + // with no event to correct it. This retry still precedes every per-id + // probe await, so it remains a valid "registered at list start" + // baseline; ids it cannot see flow through the authoritative-identity + // path instead. If it fails again the config is genuinely unreadable + // and the fail-open scoping above already chose availability. + if (initialConfigIds == null) { + try { + initialConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + } catch { + initialConfigIds = null; + } + } const entries = await Promise.all( Array.from( workspaceIds, From 54b92af84ed228b281526c517d8a695cd8d3d1bf Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 06:33:01 +0000 Subject: [PATCH 50/72] test: de-flake goal kickoff-window test (same-ms consent-stamp collision on fast runners) --- src/node/services/workspaceGoalService.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/node/services/workspaceGoalService.test.ts b/src/node/services/workspaceGoalService.test.ts index 38c78384ec..1f539666ff 100644 --- a/src/node/services/workspaceGoalService.test.ts +++ b/src/node/services/workspaceGoalService.test.ts @@ -453,7 +453,12 @@ describe("WorkspaceGoalService", () => { // Reconciliation must not pause it — the in-memory kickoff candidate can be // lost (restart, eviction), and the next getGoal (heartbeat/wake tool // assembly) would otherwise silently pause the goal before it ever ran. - await appendUserHistoryMessage(historyService, workspaceId, "Set yourself a goal"); + // Explicitly pre-goal: the consent arm compares strictly (same-ms fails + // closed to pause), so a fast runner landing this append and the goal's + // activation stamp in the same millisecond flakes the keep-active path. + await appendUserHistoryMessage(historyService, workspaceId, "Set yourself a goal", { + timestamp: Date.now() - 1_000, + }); await setGoalOk(service, { workspaceId, objective: "Follow chat tail" }); const reconciled = await service.getGoal(workspaceId); From 94b9d422603901e632671f9737825d59c3c8d923 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 06:47:11 +0000 Subject: [PATCH 51/72] review: preserve every resolvable legacy identity (alias ids) in destructive known-id sets --- src/node/config.test.ts | 42 +++++++++++++++++ src/node/config.ts | 54 ++++++++++++++++++++-- src/node/services/coreServices.ts | 11 +++-- src/node/services/workspaceService.test.ts | 53 +++++++++++++++++++++ src/node/services/workspaceService.ts | 46 ++++++++++-------- 5 files changed, 179 insertions(+), 27 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 4b2f9910a6..3410ff2607 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -3251,6 +3251,48 @@ describe("Config", () => { const allMetadata = await config.getAllWorkspaceMetadata({ throwOnError: true }); expect(allMetadata.map((metadata) => metadata.id)).toContain("stable-basename-id"); }); + + it("surfaces the second resolvable compatibility file's id as a legacy alias", async () => { + // Both supported layouts exist with DIFFERENT ids (e.g. a stale + // basename-side file next to the live generated-legacy metadata). + // findWorkspace resolves either id, so destructive known-id sets must + // retain both — the enumeration returns the first candidate as the + // entry and reports the second through the legacyAliasIds + // out-parameter. + const projectPath = "/fake/project"; + const workspaceName = "aliased-feature"; + const workspacePath = path.join(config.srcDir, "project", workspaceName); + fs.mkdirSync(workspacePath, { recursive: true }); + + const basenameSessionDir = config.getSessionDir(workspaceName); + fs.mkdirSync(basenameSessionDir, { recursive: true }); + fs.writeFileSync( + path.join(basenameSessionDir, "metadata.json"), + JSON.stringify({ id: "stale-basename-id", name: workspaceName }) + ); + const legacyId = config.generateLegacyId(projectPath, workspacePath); + const legacySessionDir = config.getSessionDir(legacyId); + fs.mkdirSync(legacySessionDir, { recursive: true }); + fs.writeFileSync( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: "live-generated-id", name: workspaceName }) + ); + + await config.editConfig((cfg) => { + cfg.projects.set(projectPath, { + workspaces: [{ path: workspacePath }], + }); + return cfg; + }); + + const legacyAliasIds = new Set(); + const allMetadata = await config.getAllWorkspaceMetadata({ + throwOnError: true, + legacyAliasIds, + }); + expect(allMetadata.map((metadata) => metadata.id)).toContain("stale-basename-id"); + expect(legacyAliasIds.has("live-generated-id")).toBe(true); + }); }); describe("transcriptOnly derivation", () => { diff --git a/src/node/config.ts b/src/node/config.ts index 55bd757612..1ae38e0ed0 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2494,6 +2494,18 @@ export class Config { * fresh install), matching loadConfigOrDefault. */ throwOnError?: boolean; + /** + * Out-parameter collecting ADDITIONAL stable ids that findWorkspace() + * can resolve for id-less legacy entries but that are not the returned + * entry's primary id: when both compatibility files + * (sessions//metadata.json and + * sessions//metadata.json) exist with different + * ids, only the first becomes the metadata entry, yet the second + * identity remains registered for targeted lookups. Destructive callers + * building "known id" sets must include these aliases or they would + * delete activity data findWorkspace still vouches for. + */ + legacyAliasIds?: Set; }): Promise { const config = this.loadConfigOrDefault({ throwOnError: options?.throwOnError }); const workspaceMetadata: FrontendWorkspaceMetadata[] = []; @@ -2678,12 +2690,13 @@ export class Config { let metadataPath = ""; let legacyMetadataRaw: string | undefined; - for (const candidateId of [workspaceBasename, legacyId]) { + const candidateIds = + workspaceBasename === legacyId ? [workspaceBasename] : [workspaceBasename, legacyId]; + for (const candidateId of candidateIds) { const candidatePath = path.join(this.getSessionDir(candidateId), "metadata.json"); + let candidateRaw: string | undefined; try { - legacyMetadataRaw = fs.readFileSync(candidatePath, "utf-8"); - metadataPath = candidatePath; - break; + candidateRaw = fs.readFileSync(candidatePath, "utf-8"); } catch (readError) { // Missing is normal (most entries never had a legacy // metadata.json). Any other failure means the workspace's @@ -2693,6 +2706,39 @@ export class Config { if (!isEnoentError(readError)) { throw readError; } + continue; + } + if (legacyMetadataRaw === undefined) { + legacyMetadataRaw = candidateRaw; + metadataPath = candidatePath; + continue; + } + // A SECOND resolvable compatibility file: findWorkspace() tries + // every candidate, so its id stays registered for targeted + // lookups even though only the first file becomes the metadata + // entry. Surface it through the legacyAliasIds out-parameter so + // destructive known-id sets retain it. findWorkspace consults + // candidate files only for id-less entries; for a partially + // migrated entry (inline id, no name) the alias is + // over-inclusive, which errs toward retention, never deletion. + // A corrupt or id-less second file leaves the alias identity + // unknowable — fail closed in strict mode, mirroring the + // primary lookup guards. + let aliasMetadata: WorkspaceMetadata | undefined; + try { + aliasMetadata = JSON.parse(candidateRaw) as WorkspaceMetadata; + } catch (parseError) { + if (options?.throwOnError) { + throw parseError; + } + } + const aliasId = aliasMetadata?.id; + if (typeof aliasId === "string" && aliasId.length > 0) { + options?.legacyAliasIds?.add(aliasId); + } else if (aliasMetadata !== undefined && options?.throwOnError) { + throw new Error( + `Legacy workspace metadata at ${candidatePath} resolved without a usable id` + ); } } if (legacyMetadataRaw !== undefined) { diff --git a/src/node/services/coreServices.ts b/src/node/services/coreServices.ts index d005d7f88d..63045b49a7 100644 --- a/src/node/services/coreServices.ts +++ b/src/node/services/coreServices.ts @@ -115,9 +115,14 @@ export function createCoreServices(opts: CoreServicesOptions): CoreServices { if (!evidence.hasWorkspaceEntriesWithoutIds) { return false; } - return (await config.getAllWorkspaceMetadata({ throwOnError: true })).some( - (metadata) => metadata.id === workspaceId - ); + // Alias ids: a second resolvable compatibility file's identity stays + // registered for findWorkspace even though it is not any entry's + // primary id — refusing its writes/deletions requires knowing it here. + const legacyAliasIds = new Set(); + const registered = ( + await config.getAllWorkspaceMetadata({ throwOnError: true, legacyAliasIds }) + ).some((metadata) => metadata.id === workspaceId); + return registered || legacyAliasIds.has(workspaceId); }); const workspaceGoalService = new WorkspaceGoalService( config, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index bd22ae55e6..b5180bf4b6 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5477,6 +5477,59 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("prune spares both legacy identities when compatibility files disagree", async () => { + // An id-less legacy entry can have BOTH supported session layouts with + // different stable ids (stale basename-side file + live generated-legacy + // metadata). findWorkspace resolves either id, so the one-time prune must + // spare extension-metadata entries under both — classifying the second + // identity as stale would delete activity findWorkspace still vouches + // for. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "old-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]] }) + ); + const basenameSessionDir = config.getSessionDir("old-ws"); + await fsPromises.mkdir(basenameSessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(basenameSessionDir, "metadata.json"), + JSON.stringify({ id: "basename-stable-id", name: "old-ws" }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: "generated-live-id", name: "old-ws" }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("basename-stable-id", 100); + await extensionMetadata.updateRecency("generated-live-id", 200); + await extensionMetadata.updateRecency("truly-stale-id", 300); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + + const snapshots = await extensionMetadata.getAllSnapshots(); + expect(snapshots.get("basename-stable-id")?.recency).toBe(100); + expect(snapshots.get("generated-live-id")?.recency).toBe(200); + expect(snapshots.has("truly-stale-id")).toBe(false); + } finally { + await cleanup(); + } + }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { // getAllWorkspaceMetadata walks every workspace with per-workspace disk // probes; the latency-sensitive first bootstrap must not pay it twice. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index d6b7dc4d42..9ad69ea27e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13327,6 +13327,26 @@ export class WorkspaceService extends EventEmitter { } } + /** + * Strict authoritative enumeration for destructive/identity decisions: + * every primary workspace id PLUS legacy alias identities (a second + * resolvable compatibility file findWorkspace still vouches for — see + * Config.getAllWorkspaceMetadata's legacyAliasIds). Known-id sets built + * without the aliases would prune or drop live alias-keyed activity. + */ + private async enumerateAuthoritativeWorkspaceIds(): Promise> { + const legacyAliasIds = new Set(); + const ids = new Set( + (await this.config.getAllWorkspaceMetadata({ throwOnError: true, legacyAliasIds })).map( + (metadata) => metadata.id + ) + ); + for (const aliasId of legacyAliasIds) { + ids.add(aliasId); + } + return ids; + } + /** * Evict process-local activity caches for a removed (or removed-then- * revived) workspace id. The caches re-bootstrap from disk on next access; @@ -13382,8 +13402,7 @@ export class WorkspaceService extends EventEmitter { // config migrations that are not yet persisted verbatim. // A missing config file resolves as a healthy empty set in both. const knownIds = this.config.readPersistedWorkspaceIdSuperset(); - const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); - normalizedIds = new Set(allMetadata.map((metadata) => metadata.id)); + normalizedIds = await this.enumerateAuthoritativeWorkspaceIds(); for (const workspaceId of normalizedIds) { knownIds.add(workspaceId); } @@ -13405,9 +13424,8 @@ export class WorkspaceService extends EventEmitter { if (!evidence.hasWorkspaceEntriesWithoutIds) { return evidence.ids; } - const allMetadata = await this.config.getAllWorkspaceMetadata({ throwOnError: true }); - for (const metadata of allMetadata) { - evidence.ids.add(metadata.id); + for (const workspaceId of await this.enumerateAuthoritativeWorkspaceIds()) { + evidence.ids.add(workspaceId); } return evidence.ids; } @@ -13481,11 +13499,7 @@ export class WorkspaceService extends EventEmitter { // throwOnError so a corrupted config.json actually reaches the // fail-open fallback below instead of silently resolving as the // empty default and dropping every live entry from the list. - workspaceIds = new Set( - (await this.config.getAllWorkspaceMetadata({ throwOnError: true })).map( - (metadata) => metadata.id - ) - ); + workspaceIds = await this.enumerateAuthoritativeWorkspaceIds(); } catch (error) { // Fail open: without the config view, stale ids cannot be told apart // from live ones, and dropping live entries would strand renderer @@ -13671,11 +13685,7 @@ export class WorkspaceService extends EventEmitter { freshConfigHasRawInvisibleEntries) ) { try { - authoritativeIds = new Set( - (await this.config.getAllWorkspaceMetadata({ throwOnError: true })).map( - (metadata) => metadata.id - ) - ); + authoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); } catch (error) { log.debug("Failed to enumerate authoritative ids for removal revalidation", { error }); authoritativeIds = null; @@ -13867,11 +13877,7 @@ export class WorkspaceService extends EventEmitter { Object.keys(activityById).some(isRawInvisible) ) { try { - finalAuthoritativeIds = new Set( - (await this.config.getAllWorkspaceMetadata({ throwOnError: true })).map( - (metadata) => metadata.id - ) - ); + finalAuthoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); } catch (error) { log.debug("Failed to re-enumerate authoritative ids after workflow probes", { error, From cfb6c9f92767068465f4ceef0d83ba9967d9136b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 06:54:39 +0000 Subject: [PATCH 52/72] review: fail closed without a raw baseline; never quarantine over an existing sidecar --- .../services/ExtensionMetadataService.test.ts | 28 ++++++++++++ src/node/services/ExtensionMetadataService.ts | 15 +++++++ src/node/services/workspaceService.test.ts | 44 +++++++++++++++++++ src/node/services/workspaceService.ts | 18 +++++++- 4 files changed, 104 insertions(+), 1 deletion(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 7c86647c3b..91bf23b2b5 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -905,6 +905,34 @@ describe("ExtensionMetadataService", () => { expect(snapshots.get("ws-partial")?.recency).toBe(300); }); + test("quarantine preserves an existing healthy sidecar instead of clobbering it", async () => { + // Crash strands the full snapshot at .corrupt; another backend recreates + // the main file, which later becomes corrupt too. The next strict read's + // quarantine must not rename() over the healthy sidecar (POSIX rename + // replaces silently) — recovery would then reset the canonical file to + // empty and permanently destroy every stranded entry. The corrupt main + // moves aside as the bounded fixed-name leftover instead and the healthy + // sidecar restores. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + await writeFile(filePath, "{corrupt json"); + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-stranded")?.recency).toBe(700); + // Corrupt main preserved as the bounded leftover; sidecar consumed. + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{corrupt json"); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("a strict snapshot read propagates a failed sidecar reconcile instead of the partial main", async () => { // Live emissions read per-workspace snapshots after the subscription // bootstraps. With a sidecar stranded next to a recreated partial main diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 0854dbb394..ca02513994 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1005,6 +1005,21 @@ export class ExtensionMetadataService { } } const quarantinePath = `${this.filePath}.corrupt`; + // A crash-stranded sidecar may already occupy the quarantine path — + // typically the full healthy snapshot, with the corrupt main being a + // later re-corruption of a recreated file. On POSIX, rename() would + // silently REPLACE those bytes, and recovery would then reset the + // canonical file to empty, permanently destroying the recoverable + // data. Move the corrupt main aside as the bounded fixed-name + // leftover instead (same name the newer-schema swap uses; keeps the + // latest superseded file) and complete the EXISTING sidecar's + // recovery: healthy bytes restore to the canonical path, corrupt + // sidecar bytes reset it to empty exactly as before. A failing probe + // propagates so the read stays retryable on unknowable evidence. + if (await this.probeQuarantineSidecar()) { + await rename(this.filePath, `${this.filePath}.recreated`); + return this.completeQuarantineRecovery(quarantinePath); + } await rename(this.filePath, quarantinePath); return this.completeQuarantineRecovery(quarantinePath); }); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index b5180bf4b6..8eb7dedaf3 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5530,6 +5530,50 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("getActivityList fails closed when no raw baseline can be established", async () => { + // If every raw baseline read fails transiently while the strict scoping + // enumeration succeeds, both cross-process removal guards would stay + // disabled on a response the renderer applies as authoritative — a + // workspace another backend deregisters during the probes would ride + // back with no event to correct it. The list must fail (null → renderer + // keeps last-known state and retries) instead of serving guardless + // authoritative data; only the fail-open scope (config unreadable) may + // do that, and there the enumeration fails too. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "baseline-unavailable"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 111); + const supersetSpy = spyOn(config, "readPersistedWorkspaceIdSuperset").mockImplementation( + () => { + throw new Error("persistent raw read failure"); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + expect(await workspaceService.getActivityList()).toBeNull(); + } finally { + supersetSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { // getAllWorkspaceMetadata walks every workspace with per-workspace disk // probes; the latency-sensitive first bootstrap must not pay it twice. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 9ad69ea27e..bf6f309727 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13474,6 +13474,10 @@ export class WorkspaceService extends EventEmitter { // WITHOUT a snapshot still flow through the tombstone logic below — // scoping only drops ids that are not in config at all. let workspaceIds: Set; + // Whether scoping ended up on the fail-open legacy union (config + // unreadable): only that availability path may serve a response with + // the cross-process removal guards disabled. + let scopedFailOpen = false; if (prefetchedKnownIds != null) { workspaceIds = prefetchedKnownIds; // The prune enumerated config BEFORE the snapshot read above, so a @@ -13505,6 +13509,7 @@ export class WorkspaceService extends EventEmitter { // from live ones, and dropping live entries would strand renderer // activity state. Fall back to the legacy unscoped union. log.debug("Failed to scope activity list to known workspaces", { error }); + scopedFailOpen = true; workspaceIds = new Set(snapshots.keys()); for (const workspaceId of this.activeWorkflowRunIdsByWorkspace.keys()) { workspaceIds.add(workspaceId); @@ -13530,7 +13535,18 @@ export class WorkspaceService extends EventEmitter { if (initialConfigIds == null) { try { initialConfigIds = this.config.readPersistedWorkspaceIdSuperset(); - } catch { + } catch (error) { + // Authoritative scope but NO raw baseline even on retry: both + // cross-process removal guards would silently stay disabled on a + // response the renderer applies as authoritative — a workspace + // another backend deregisters during the probes below would ride + // back with no event to correct it. Fail the list instead (null → + // renderer keeps last-known state and retries). The fail-open + // scope keeps its availability contract: config is unreadable + // there by definition, and no baseline exists by design. + if (!scopedFailOpen) { + throw error; + } initialConfigIds = null; } } From 901d39db8518ee97d394fa32e1c17e7660ef782c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 07:15:41 +0000 Subject: [PATCH 53/72] review: initial-snapshot fallback for late candidates, reconcile sidecar before prune, distinct unavailable goal-push result --- src/node/services/ExtensionMetadataService.ts | 9 ++ .../services/workspaceGoalService.test.ts | 26 ++++ src/node/services/workspaceGoalService.ts | 29 +++-- src/node/services/workspaceService.test.ts | 113 ++++++++++++++++++ src/node/services/workspaceService.ts | 10 +- 5 files changed, 175 insertions(+), 12 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index ca02513994..cfea59327c 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -850,6 +850,15 @@ export class ExtensionMetadataService { // workspace's data. Defaults to getKnownWorkspaceIds. recheckKnownWorkspaceIds?: () => Promise> ): Promise { + // Reconcile a crash-stranded sidecar FIRST (outside the mutation queue — + // the resume path serializes itself): the prune classifies stale ids + // against the file it loads, so sidecar-only entries would dodge the + // one-time deletion set and merge back into the main file on the very + // next read — with the prune latched, exactly the stale entries this + // cleanup exists to remove would keep inflating every read and rewrite + // until restart. A failing reconcile propagates so the caller's + // fail-closed abort applies instead of pruning against a partial view. + await this.reconcileLeftoverSidecarIfPresent(); return this.withSerializedMutation(async () => { const data = await this.load(); const knownWorkspaceIds = await getKnownWorkspaceIds(); diff --git a/src/node/services/workspaceGoalService.test.ts b/src/node/services/workspaceGoalService.test.ts index cd7aab2c62..d4267a25d0 100644 --- a/src/node/services/workspaceGoalService.test.ts +++ b/src/node/services/workspaceGoalService.test.ts @@ -2298,6 +2298,32 @@ describe("WorkspaceGoalService", () => { }); }); + test("previewStreamAccounting skips the durable fallback when the strict baseline read is unavailable", async () => { + // "unavailable" (failed sidecar reconcile) must stay distinct from the + // authoritative "no baseline": the durable pushSnapshot fallback writes + // through the lenient load — accepting the suspect partial main the + // strict read refused — and emits it, clearing renderer goal/status + // state. The preview must resolve without delivering or writing. + await setGoalOk(service, { workspaceId, objective: "Preview goal" }); + const metadataFilePath = path.join(config.rootDir, "extensionMetadata.json"); + const before = await fs.readFile(metadataFilePath, "utf-8"); + // A directory at the sidecar path yields a deterministic errno (EISDIR) + // standing in for EACCES/EIO-class reconcile failures. + await fs.mkdir(`${metadataFilePath}.corrupt`); + try { + const activityUpdates = captureGoalActivity(service); + + const preview = await service.previewStreamAccounting({ workspaceId, costUsd: 1 }); + + expect(preview).toMatchObject({ objective: "Preview goal" }); + // No emit (renderer keeps last-known state) and no durable write. + expect(activityUpdates).toHaveLength(0); + expect(await fs.readFile(metadataFilePath, "utf-8")).toBe(before); + } finally { + await fs.rm(`${metadataFilePath}.corrupt`, { recursive: true }); + } + }); + test("successful no-op queued drains clear the pending snapshot", async () => { const created = await setGoalOk(service, { workspaceId, objective: "Existing goal" }); await extensionMetadata.setStreaming(workspaceId, true); diff --git a/src/node/services/workspaceGoalService.ts b/src/node/services/workspaceGoalService.ts index b1b74e9b0f..db8539a29b 100644 --- a/src/node/services/workspaceGoalService.ts +++ b/src/node/services/workspaceGoalService.ts @@ -1280,38 +1280,41 @@ export class WorkspaceGoalService { private async pushTransientGoalSnapshot( workspaceId: string, snapshot: GoalSnapshot - ): Promise { + ): Promise<"delivered" | "no_baseline" | "unavailable"> { let activity: WorkspaceActivitySnapshot | null; try { activity = await this.extensionMetadata.getSnapshot(workspaceId, { throwOnError: true }); } catch (error) { // A suspect baseline (failed sidecar reconcile / unreadable main) // must not feed an emitted overlay: partial-main fields would clear - // status in the renderer. Report non-delivery instead — callers that - // must guarantee delivery fall back to pushSnapshot, whose lenient - // write path self-heals. + // status in the renderer. "unavailable" is deliberately DISTINCT from + // the authoritative "no_baseline": the durable pushSnapshot fallback + // writes through the lenient load (accepting the same suspect partial + // main this strict read refused) and emits the result — converting + // this failure into that fallback would clear exactly the renderer + // state the strict read preserves. log.debug("Skipping transient goal emit after failed snapshot read", { workspaceId, error, }); - return false; + return "unavailable"; } if (!activity) { // No baseline activity snapshot to overlay the transient goal on // (extensionMetadata has no entry for this workspace yet). Callers // that must guarantee delivery — e.g. live cost previews — should - // observe this `false` return and fall back to `pushSnapshot`, which + // observe "no_baseline" and fall back to `pushSnapshot`, which // creates the entry and emits via the durable path. Pending-goal // publication does not retry here because it only fires after a // `setGoal` that already created the entry. - return false; + return "no_baseline"; } this.onActivityChange?.(workspaceId, { ...activity, goal: snapshot, transientGoalOnly: true, }); - return true; + return "delivered"; } private async pushLiveGoalPreviewOverlay( @@ -4128,13 +4131,17 @@ export class WorkspaceGoalService { }); const snapshot = toGoalSnapshot(preview); this.liveGoalPreviewSnapshots.set(input.workspaceId, snapshot); - const didEmitTransient = await this.pushTransientGoalSnapshot(input.workspaceId, snapshot); - if (!didEmitTransient) { + const transientResult = await this.pushTransientGoalSnapshot(input.workspaceId, snapshot); + if (transientResult === "no_baseline") { // If the baseline activity snapshot does not exist yet (for // example, extensionMetadata was reset or stream-start's // fire-and-forget metadata write has not finished), fall back to // the durable path so this preview is still delivered to Goals UI - // subscribers instead of being dropped. + // subscribers instead of being dropped. "unavailable" must NOT take + // this path: the durable write's lenient load would accept the + // suspect partial main the strict read refused and emit it, + // clearing renderer goal/status state — return the computed preview + // without delivery instead (renderer keeps last-known state). return this.pushSnapshot(input.workspaceId, preview); } return snapshot; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 8eb7dedaf3..b44c77d094 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5574,6 +5574,119 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("first prune also removes stale entries stranded in a sidecar", async () => { + // Crash strands the full snapshot in .corrupt while a valid partial main + // was recreated. The one-time prune must reconcile FIRST: sidecar-only + // stale entries would otherwise dodge the deletion set and merge back on + // the very next read — with the prune latched, they would keep inflating + // every read and rewrite until restart. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "sidecar-live"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + await fsPromises.writeFile( + metadataPath, + JSON.stringify({ + version: 1, + workspaces: { [workspaceId]: { recency: 100, streaming: false } }, + }) + ); + await fsPromises.writeFile( + `${metadataPath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + [workspaceId]: { recency: 90, streaming: false }, + "sidecar-stale": { recency: 80, streaming: false }, + }, + }) + ); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.["sidecar-stale"]).toBeUndefined(); + + const snapshots = await extensionMetadata.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get(workspaceId)?.recency).toBe(100); + expect(snapshots.has("sidecar-stale")).toBe(false); + } finally { + await cleanup(); + } + }); + + test("late-admitted raw-registered ids keep their initial snapshot when re-reads fail", async () => { + // A raw-registered entry outside the normalized scope (invalid project + // path) is admitted through the raw config view. When both mid-list + // snapshot re-reads fail transiently, the already-loaded initial + // snapshot must still supply its recency/goal/status — an authoritative + // response omitting the entry would clear that renderer state with no + // repair event. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "raw-only-live"; + const projectPath = path.join(config.rootDir, "project"); + // A null project path survives the raw id scan but is dropped by the + // normalized enumeration, keeping the id out of the per-id scope. + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [ + [null, { workspaces: [{ id: workspaceId, path: path.join(projectPath, "ws") }] }], + ], + }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 42); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + // List #1 (prune latch): initial + fresh reads stay real. List #2: + // the initial read (call 3) succeeds; the fresh and final re-reads + // fail transiently. + if (snapshotCalls > 3) { + throw new Error("transient snapshot re-read failure"); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const firstList = await workspaceService.getActivityList(); + expect(firstList?.[workspaceId]?.recency).toBe(42); + + const secondList = await workspaceService.getActivityList(); + expect(secondList).not.toBeNull(); + expect(secondList?.[workspaceId]?.recency).toBe(42); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { // getAllWorkspaceMetadata walks every workspace with per-workspace disk // probes; the latency-sensitive first bootstrap must not pay it twice. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index bf6f309727..caf75aa063 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13945,10 +13945,18 @@ export class WorkspaceService extends EventEmitter { } } for (const [workspaceId, activeWorkflowRunIds] of probedWorkflowRunIds) { + // Freshest available view, degrading to the INITIAL read: a raw- + // registered id outside the normalized scope (e.g. invalid + // project path) is admitted here, and when both re-reads failed + // transiently its already-loaded initial snapshot must still + // supply goal/status/recency — omitting it from an authoritative + // response would clear that renderer state with no repair event. const lateSnapshot = finalSnapshots != null ? (finalSnapshots.get(workspaceId) ?? null) - : (freshSnapshots?.get(workspaceId) ?? null); + : freshSnapshots != null + ? (freshSnapshots.get(workspaceId) ?? null) + : (snapshots.get(workspaceId) ?? null); if ( // Nothing to contribute: no persisted snapshot and no live // counts (a workflow-only candidate legitimately has no From 7908b0f3210c84c4fc25b0df175d51fcb55b1147 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 07:21:43 +0000 Subject: [PATCH 54/72] review: keep the generated-legacy record canonical when both compatibility files exist --- src/node/config.test.ts | 20 +++++++++++++++----- src/node/config.ts | 22 ++++++++++++++-------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 3410ff2607..59cd1c5ec6 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -3256,9 +3256,11 @@ describe("Config", () => { // Both supported layouts exist with DIFFERENT ids (e.g. a stale // basename-side file next to the live generated-legacy metadata). // findWorkspace resolves either id, so destructive known-id sets must - // retain both — the enumeration returns the first candidate as the - // entry and reports the second through the legacyAliasIds - // out-parameter. + // retain both — the GENERATED-LEGACY record stays canonical (its id + // feeds the read-time config migration; a stale basename-side primary + // would rewrite the persisted stable id on upgrade and orphan session + // history) while the basename id is reported through the + // legacyAliasIds out-parameter. const projectPath = "/fake/project"; const workspaceName = "aliased-feature"; const workspacePath = path.join(config.srcDir, "project", workspaceName); @@ -3290,8 +3292,16 @@ describe("Config", () => { throwOnError: true, legacyAliasIds, }); - expect(allMetadata.map((metadata) => metadata.id)).toContain("stale-basename-id"); - expect(legacyAliasIds.has("live-generated-id")).toBe(true); + expect(allMetadata.map((metadata) => metadata.id)).toContain("live-generated-id"); + expect(legacyAliasIds.has("stale-basename-id")).toBe(true); + // The read-time migration must persist the CANONICAL id — a stale + // basename-side id here would change the workspace's stable identity + // on upgrade and make its session history appear missing. + const persisted = config + .loadConfigOrDefault() + .projects.get(projectPath) + ?.workspaces.find((workspace) => workspace.path === workspacePath); + expect(persisted?.id).toBe("live-generated-id"); }); }); diff --git a/src/node/config.ts b/src/node/config.ts index 1ae38e0ed0..9bc96e4932 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2678,20 +2678,26 @@ export class Config { // LEGACY FORMAT: Fall back to reading metadata.json. // findWorkspace resolves an id-less entry's stable id from EITHER - // sessions//metadata.json (old layout, checked - // first there) or sessions//metadata.json. - // Enumerate the same candidates in the same order: destructive - // callers (the extension-metadata prune) classify ids as stale - // against THIS enumeration, so a basename-backed stable id that - // findWorkspace can resolve but this walk cannot would be deleted - // as unknown. + // sessions//metadata.json or + // sessions//metadata.json (old layout). + // Enumerate BOTH candidates: destructive callers (the + // extension-metadata prune) classify ids as stale against this + // enumeration, so a stable id that findWorkspace can resolve but + // this walk cannot would be deleted as unknown. + // The generated-legacy record stays CANONICAL when both exist: + // this walk's primary id feeds the read-time config migration, and + // it historically consulted only the generated-legacy file — making + // a (potentially stale) basename-side file primary would rewrite + // the workspace's persisted stable id on upgrade and orphan its + // session history. The basename id is surfaced as an alias below; + // it becomes primary only when it is the sole surviving record. const legacyId = this.generateLegacyId(projectPath, workspace.path); let metadataFound = false; let metadataPath = ""; let legacyMetadataRaw: string | undefined; const candidateIds = - workspaceBasename === legacyId ? [workspaceBasename] : [workspaceBasename, legacyId]; + workspaceBasename === legacyId ? [legacyId] : [legacyId, workspaceBasename]; for (const candidateId of candidateIds) { const candidatePath = path.join(this.getSessionDir(candidateId), "metadata.json"); let candidateRaw: string | undefined; From 03e14ddecef1a116e211e0ad09cece97d3578a3b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 07:45:30 +0000 Subject: [PATCH 55/72] review: reconcile sidecars before mutations, generation-scoped sidecar consumption, authoritative removal fallback --- .../services/ExtensionMetadataService.test.ts | 76 +++++++++++++ src/node/services/ExtensionMetadataService.ts | 107 +++++++++++++++++- src/node/services/workspaceService.test.ts | 102 +++++++++++++++++ src/node/services/workspaceService.ts | 38 +++++-- 4 files changed, 309 insertions(+), 14 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 91bf23b2b5..e676e6824b 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -905,6 +905,82 @@ describe("ExtensionMetadataService", () => { expect(snapshots.get("ws-partial")?.recency).toBe(300); }); + test("mutations reconcile a stranded sidecar before saving and emitting", async () => { + // Normal recency/status/goal writes save and BROADCAST their snapshot. + // With a valid partial main beside a healthy full sidecar, the mutation + // must reconcile first — otherwise it persists and emits the partial + // view, clearing goal/status in the renderer until some read recovers. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 1, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-live": { recency: 1, streaming: false }, + "ws-other": { recency: 5, streaming: false }, + }, + }) + ); + + await service.updateRecency("ws-live", 999); + + const main = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + expect(main.workspaces["ws-other"]?.recency).toBe(5); + expect(main.workspaces["ws-live"]?.recency).toBe(999); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("consumption is scoped to the sidecar generation that was reconciled", async () => { + // With multiple processes recovering the fixed .corrupt path, another + // backend can consume the generation this process read and strand a NEW + // snapshot at the same path before the unlink runs. A path-only unlink + // would destroy that unreconciled generation permanently. + const quarantinePath = `${filePath}.corrupt`; + const internals = ExtensionMetadataService as unknown as { + statQuarantineToken(path: string): Promise; + consumeQuarantineSidecar(path: string, token: unknown): Promise; + }; + await writeFile( + quarantinePath, + JSON.stringify({ version: 1, workspaces: { a: { recency: 1, streaming: false } } }) + ); + const staleToken = await internals.statQuarantineToken(quarantinePath); + // Concurrent recovery consumes that generation and quarantines a new + // snapshot at the same path (different content => different identity). + await rm(quarantinePath); + await writeFile( + quarantinePath, + JSON.stringify({ version: 1, workspaces: { b: { recency: 22222, streaming: false } } }) + ); + + await internals.consumeQuarantineSidecar(quarantinePath, staleToken); + // The newer generation survives for its own recovery pass. + const survivor = JSON.parse(await readFile(quarantinePath, "utf-8")) as { + workspaces: Record; + }; + expect(survivor.workspaces.b).toBeDefined(); + + const currentToken = await internals.statQuarantineToken(quarantinePath); + await internals.consumeQuarantineSidecar(quarantinePath, currentToken); + const gone = await readFile(quarantinePath, "utf-8").then( + () => false, + () => true + ); + expect(gone).toBe(true); + }); + test("quarantine preserves an existing healthy sidecar instead of clobbering it", async () => { // Crash strands the full snapshot at .corrupt; another backend recreates // the main file, which later becomes corrupt too. The next strict read's diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index cfea59327c..fd1a54d576 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1,5 +1,15 @@ import { dirname } from "path"; -import { mkdir, readFile, access, rename, link, unlink, copyFile, writeFile } from "fs/promises"; +import { + mkdir, + readFile, + access, + rename, + link, + unlink, + copyFile, + writeFile, + stat, +} from "fs/promises"; import { constants } from "fs"; import writeFileAtomic from "write-file-atomic"; import { @@ -16,6 +26,17 @@ import type { WorkspaceActivitySnapshot } from "@/common/types/workspace"; import type { GoalSnapshot } from "@/common/types/goal"; import { log } from "@/node/services/log"; +/** + * Identity of one quarantine-sidecar generation: with multiple processes + * recovering the fixed `.corrupt` path, consumption must be scoped to + * exactly the file generation a recovery read (see consumeQuarantineSidecar). + */ +interface QuarantineSidecarToken { + ino: bigint; + mtimeNs: bigint; + size: bigint; +} + /** * Marker code for "file written by a newer schema version" load failures. * Shaped like an errno code so the corruption/quarantine classification @@ -248,6 +269,15 @@ export class ExtensionMetadataService { ) { return this.buildTransientSnapshot(workspaceId, recency, mutate); } + // Write-produced snapshots are broadcast by WorkspaceService: reconcile a + // crash-stranded sidecar BEFORE mutating (outside the queue — the resume + // path serializes itself). Without this, a valid partial main next to a + // healthy full sidecar is saved and EMITTED, clearing goal/status in the + // renderer until some read triggers recovery. Failure propagates: + // failing one write beats persisting and broadcasting the partial view + // (same tradeoff as the unprobeable-sidecar contract in + // probeQuarantineSidecar). + await this.reconcileLeftoverSidecarIfPresent(); return this.withSerializedMutation(async () => { // Re-check inside the queue: pruneMissingWorkspaces publishes its // tombstones only while its queued mutation runs, so a writer that @@ -309,7 +339,45 @@ export class ExtensionMetadataService { * stale sidecar again after the main file has moved on, repeatedly * re-merging entries that pruning or removal already reclaimed. */ - private static async consumeQuarantineSidecar(quarantinePath: string): Promise { + private static async consumeQuarantineSidecar( + quarantinePath: string, + token: QuarantineSidecarToken | null + ): Promise { + // Generation check: with multiple processes recovering the fixed sidecar + // path, another backend can consume the generation this caller read and + // quarantine a NEW snapshot at the same path before this unlink runs — a + // path-only unlink would permanently destroy that unreconciled + // generation. Compare the identity captured when the caller READ the + // sidecar and leave a mismatched (newer) file for its own recovery pass. + // The residual stat→unlink window is the documented unlocked-shared-file + // boundary (see the PR's concurrency contract); closing it entirely + // requires an interprocess lock. + if (token == null) { + // The caller never observed a sidecar identity (it read the file only + // after a successful probe, so this is unreachable in practice) — + // fail closed by leaving the file rather than unlinking blind. + return; + } + let current: QuarantineSidecarToken | null; + try { + current = await ExtensionMetadataService.statQuarantineToken(quarantinePath); + } catch { + // Unprobeable: leave the sidecar (retryable) rather than guessing. + return; + } + if (current == null) { + return; // Already consumed by a concurrent recovery. + } + if ( + current.ino !== token.ino || + current.mtimeNs !== token.mtimeNs || + current.size !== token.size + ) { + log.debug("Leaving replaced quarantine sidecar for its own recovery pass", { + quarantinePath, + }); + return; + } try { await unlink(quarantinePath); } catch (unlinkError) { @@ -319,6 +387,26 @@ export class ExtensionMetadataService { } } + /** + * Identity of one sidecar generation (inode + mtime + size), captured when + * a recovery reads the sidecar and required by consumeQuarantineSidecar so + * consumption is scoped to exactly the generation that was reconciled. + * Resolves null on ENOENT; other stat failures propagate (retryable). + */ + private static async statQuarantineToken( + quarantinePath: string + ): Promise { + try { + const stats = await stat(quarantinePath, { bigint: true }); + return { ino: stats.ino, mtimeNs: stats.mtimeNs, size: stats.size }; + } catch (statError) { + if (ExtensionMetadataService.isErrnoCode(statError, "ENOENT")) { + return null; + } + throw statError; + } + } + /** * Shared leftover-sidecar recovery for the read paths (authoritative list * and per-workspace snapshot reads): probe the fixed sidecar path and run @@ -563,6 +651,9 @@ export class ExtensionMetadataService { ) { return null; } + // Same pre-mutation sidecar reconcile as mutateWorkspaceSnapshot: this + // path also saves and returns a snapshot the caller broadcasts. + await this.reconcileLeftoverSidecarIfPresent(); return this.withSerializedMutation(async () => { // Re-check inside the queue (see mutateWorkspaceSnapshot): tombstones // published by an already-enqueued prune must be honored here too. @@ -1053,6 +1144,9 @@ export class ExtensionMetadataService { // outcome here; a transient failure reading the sidecar means the // moved bytes cannot be verified, so it propagates (retryable) rather // than reporting a successful reset over possibly-healthy data. + // Generation identity captured before the read: consumption below is + // scoped to exactly these bytes (see consumeQuarantineSidecar). + const sidecarToken = await ExtensionMetadataService.statQuarantineToken(quarantinePath); let moved: unknown; let movedParses = true; try { @@ -1103,7 +1197,7 @@ export class ExtensionMetadataService { throw copyError; } } - await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath); + await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath, sidecarToken); return false; } // Replace the quarantined main file with a valid EMPTY file instead of @@ -1198,6 +1292,9 @@ export class ExtensionMetadataService { * withSerializedMutation. Returns false (no empty reset happened). */ private async reconcileRecreatedMainWithSidecar(quarantinePath: string): Promise { + // Generation identity captured before the read: consumption below is + // scoped to exactly these bytes (see consumeQuarantineSidecar). + const sidecarToken = await ExtensionMetadataService.statQuarantineToken(quarantinePath); let sidecarParsed: unknown; try { sidecarParsed = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; @@ -1252,7 +1349,7 @@ export class ExtensionMetadataService { throw copyError; } } - await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath); + await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath, sidecarToken); return false; } if (!ExtensionMetadataService.isValidMetadataFileShape(sidecarParsed)) { @@ -1382,7 +1479,7 @@ export class ExtensionMetadataService { } // Consumed either way: every surviving sidecar entry is now represented // at the main path. - await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath); + await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath, sidecarToken); return false; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index b44c77d094..448030b79d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5687,6 +5687,108 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("mid-list enumeration proves removal when the raw refresh fails", async () => { + // An inline-id workspace is removed by another backend while the + // mid-list authoritative enumeration awaits, and the post-enumeration + // raw refresh fails transiently. The raw comparison is disabled (fresh + // view null) and the id sits in the initial baseline, so without the + // enumeration fallback every removal guard passes and the stale entry + // rides the authoritative response with no event to repair the + // renderer. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "inline-removed-mid-enum"; + const projectPath = path.join(config.rootDir, "project"); + const configPath = path.join(config.rootDir, "config.json"); + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [ + [ + projectPath, + { + workspaces: [ + { id: workspaceId, path: path.join(projectPath, "ws") }, + // Id-less legacy entry whose stable id lives in session + // metadata.json: raw-INVISIBLE at the initial baseline, so + // its retained entry forces the mid-list authoritative + // enumeration this test exercises (the read-time migration + // may persist the id later, but the baseline predates it). + { path: path.join(projectPath, "legacy-ws") }, + ], + }, + ], + ], + }) + ); + const legacyStableId = "legacy-stable-mid-enum"; + const legacySessionDir = config.getSessionDir("legacy-ws"); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: legacyStableId, name: "legacy-ws" }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 77); + await extensionMetadata.updateRecency(legacyStableId, 55); + const realEnumerate = config.getAllWorkspaceMetadata.bind(config); + let enumerationCalls = 0; + let failEvidenceReads = false; + const enumerationSpy = spyOn(config, "getAllWorkspaceMetadata").mockImplementation( + async (options?: Parameters[0]) => { + enumerationCalls += 1; + if (enumerationCalls === 2) { + // Mid-list enumeration: another backend deregisters the inline-id + // workspace just before the config read, and every later raw + // view read fails transiently. + const parsed = JSON.parse(await fsPromises.readFile(configPath, "utf-8")) as { + projects: Array<[string, { workspaces: Array<{ id?: string }> }]>; + }; + for (const [, projectConfig] of parsed.projects) { + projectConfig.workspaces = projectConfig.workspaces.filter( + (workspace) => workspace.id !== workspaceId + ); + } + await fsPromises.writeFile(configPath, JSON.stringify(parsed)); + const result = await realEnumerate(options); + failEvidenceReads = true; + return result; + } + return realEnumerate(options); + } + ); + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + if (failEvidenceReads) { + throw new Error("transient raw config read failure"); + } + return realEvidence(); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + // The still-registered raw-invisible entry survives the fallback... + expect(activityList?.[legacyStableId]?.recency).toBe(55); + // ...while the enumeration-proven removal is dropped. + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + enumerationSpy.mockRestore(); + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { // getAllWorkspaceMetadata walks every workspace with per-workspace disk // probes; the latency-sensitive first bootstrap must not pay it twice. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index caf75aa063..2d5617df3e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13478,7 +13478,15 @@ export class WorkspaceService extends EventEmitter { // unreadable): only that availability path may serve a response with // the cross-process removal guards disabled. let scopedFailOpen = false; + // Ids the SCOPE's strict enumeration itself vouched for, captured + // before raw-view additions: the authoritative-removal fallback below + // may treat mid-list enumeration absence as removal evidence only for + // these ids — raw-registered ids the normalized view cannot see + // (invalid project path) are legitimately absent from every + // enumeration and must not read that absence as removal. + let scopeEnumerationIds: ReadonlySet | null = null; if (prefetchedKnownIds != null) { + scopeEnumerationIds = new Set(prefetchedKnownIds); workspaceIds = prefetchedKnownIds; // The prune enumerated config BEFORE the snapshot read above, so a // workspace registered in between would be missing here — and an @@ -13504,6 +13512,7 @@ export class WorkspaceService extends EventEmitter { // fail-open fallback below instead of silently resolving as the // empty default and dropping every live entry from the list. workspaceIds = await this.enumerateAuthoritativeWorkspaceIds(); + scopeEnumerationIds = workspaceIds; } catch (error) { // Fail open: without the config view, stale ids cannot be told apart // from live ones, and dropping live entries would strand renderer @@ -13727,15 +13736,26 @@ export class WorkspaceService extends EventEmitter { } } const isRemovedPerAuthoritativeIdentity = (workspaceId: string): boolean => - initialConfigIds != null && - !initialConfigIds.has(workspaceId) && - // An id visible in the FRESH raw view is verifiably registered - // regardless of what the (possibly earlier) authoritative - // enumeration saw — e.g. a workspace registered after that - // enumeration ran must not read as removed. - !(freshConfigIds?.has(workspaceId) ?? false) && - authoritativeIds != null && - !authoritativeIds.has(workspaceId); + (initialConfigIds != null && + !initialConfigIds.has(workspaceId) && + // An id visible in the FRESH raw view is verifiably registered + // regardless of what the (possibly earlier) authoritative + // enumeration saw — e.g. a workspace registered after that + // enumeration ran must not read as removed. + !(freshConfigIds?.has(workspaceId) ?? false) && + authoritativeIds != null && + !authoritativeIds.has(workspaceId)) || + // Raw view unavailable (initial evidence read or post-enumeration + // refresh failed): the mid-list authoritative enumeration is the + // only usable post-removal view — without this arm an inline-id + // workspace removed during that enumeration would ride the + // authoritative response with every raw guard disabled and no + // cross-process event to repair the renderer. Confined to ids the + // scope enumeration itself vouched for (see scopeEnumerationIds). + (freshConfigIds == null && + authoritativeIds != null && + (scopeEnumerationIds?.has(workspaceId) ?? false) && + !authoritativeIds.has(workspaceId)); // Tombstones are process-local removal knowledge; the shared config is // the authority. A downgraded concurrent backend can legitimately // re-register a deterministic legacy id this process pruned earlier — From 2d1eff48b68f61e218b7071324cf1b00ba6ad6c6 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 08:26:00 +0000 Subject: [PATCH 56/72] fix: replace stale .recreated leftovers portably and de-flake raw-config tests Codex round 50: rename() onto an existing .recreated leftover is not reliably a replace on Windows, so a second recovery pass could fail every strict activity read until the leftover was removed by hand. Move-aside now unlinks a stale leftover first (shared moveMainAsideAsRecreatedLeftover helper for both the quarantine and newer-schema swap sites). Also de-flakes two round-49 tests: the first config load schedules an async settings-migration persist that rewrites config.json through the parsed view, dropping null-path projects and attaching resolved legacy ids inline. Pre-seed the migration flags so the persist never fires mid-test. --- .../services/ExtensionMetadataService.test.ts | 23 +++++++++++++++ src/node/services/ExtensionMetadataService.ts | 28 +++++++++++++++++-- src/node/services/workspaceService.test.ts | 14 ++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index e676e6824b..ad2f2210c0 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1009,6 +1009,29 @@ describe("ExtensionMetadataService", () => { expect(sidecarGone).toBe(true); }); + test("recovery replaces a stale .recreated leftover from an earlier pass", async () => { + // A second recovery cycle can find the fixed-name leftover already + // occupied by an earlier pass. The move-aside must replace it portably + // (unlink first — Windows rename onto an existing file is not reliably + // a replace), keeping the LATEST superseded file: a recovery that fails + // on the occupied destination would keep every strict read failing + // until the leftover was removed by hand. + await writeFile(`${filePath}.recreated`, "stale leftover from an earlier recovery"); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + await writeFile(filePath, "{corrupt json"); + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-stranded")?.recency).toBe(700); + // The leftover now holds the LATEST superseded file, not the stale one. + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{corrupt json"); + }); + test("a strict snapshot read propagates a failed sidecar reconcile instead of the partial main", async () => { // Live emissions read per-workspace snapshots after the subscription // bootstraps. With a sidecar stranded next to a recreated partial main diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index fd1a54d576..60c7cef961 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1117,7 +1117,7 @@ export class ExtensionMetadataService { // sidecar bytes reset it to empty exactly as before. A failing probe // propagates so the read stays retryable on unknowable evidence. if (await this.probeQuarantineSidecar()) { - await rename(this.filePath, `${this.filePath}.recreated`); + await this.moveMainAsideAsRecreatedLeftover(); return this.completeQuarantineRecovery(quarantinePath); } await rename(this.filePath, quarantinePath); @@ -1125,6 +1125,30 @@ export class ExtensionMetadataService { }); } + /** + * Move the current main file aside as the bounded fixed-name `.recreated` + * leftover ("keeps the latest superseded file"). A stale leftover from an + * earlier recovery is unlinked first: POSIX rename() replaces the + * destination silently, but Windows rename onto an existing file is not + * reliably a replace (it can fail with EPERM/EEXIST), which would fail + * every strict activity read's recovery until the user removed the + * leftover by hand. A crash between the unlink and the rename only loses + * the STALE leftover; the main file stays in place and the recovery + * remains resumable. Non-ENOENT unlink failures propagate (retryable) — + * the rename would fail on the occupied destination anyway. + */ + private async moveMainAsideAsRecreatedLeftover(): Promise { + const leftoverPath = `${this.filePath}.recreated`; + try { + await unlink(leftoverPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + await rename(this.filePath, leftoverPath); + } + /** * Finish a quarantine whose main file was already moved to the sidecar: * restore the sidecar to the main path when its bytes turn out healthy, @@ -1329,7 +1353,7 @@ export class ExtensionMetadataService { // fixed-name leftover. A crash between the two steps leaves the // resumable missing-main + sidecar state, which restores the // unsupported sidecar via completeQuarantineRecovery. - await rename(this.filePath, `${this.filePath}.recreated`); + await this.moveMainAsideAsRecreatedLeftover(); try { await link(quarantinePath, this.filePath); } catch (linkError) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 448030b79d..352592cbc4 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5641,12 +5641,19 @@ describe("WorkspaceService activity list scoping", () => { const projectPath = path.join(config.rootDir, "project"); // A null project path survives the raw id scan but is dropped by the // normalized enumeration, keeping the id out of the per-id scope. + // Migration flags pre-seeded: without them the first load schedules an + // async settings-migration persist that rewrites config.json through + // the parsed view — which DROPS the null-path project — deregistering + // this raw-only id mid-test whenever the persist happens to land + // before the second list's raw reads (observed flake). await fsPromises.writeFile( path.join(config.rootDir, "config.json"), JSON.stringify({ projects: [ [null, { workspaces: [{ id: workspaceId, path: path.join(projectPath, "ws") }] }], ], + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, }) ); const extensionMetadata = new ExtensionMetadataService( @@ -5719,6 +5726,13 @@ describe("WorkspaceService activity list scoping", () => { }, ], ], + // Migration flags pre-seeded: without them the first load schedules + // an async settings-migration persist that rewrites config.json + // through the parsed view — attaching the resolved legacy id inline + // — which would make this entry raw-VISIBLE mid-test and skip the + // mid-list enumeration whenever the persist lands first. + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, }) ); const legacyStableId = "legacy-stable-mid-enum"; From 2d48374ee020af920b99529660d7ca7a2533cc78 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 08:52:20 +0000 Subject: [PATCH 57/72] fix: address round-51 Codex findings 1. Strict config loads reject empty/non-string project keys: lenient normalization drops the whole project, and an id-less legacy workspace inside it is raw-invisible too, so the startup prune would permanently delete its activity snapshot on a silently lossy id set. 2. consumeQuarantineSidecar propagates non-ENOENT token-probe failures instead of reporting success with the sidecar retained (which would let post-prune reads re-merge reclaimed entries permanently). 3. quarantineCorruptFile's existing-sidecar branch revalidates the bytes moved to .recreated: a concurrent backend's healthy save landing in the race window is restored to the main path and merged with the sidecar instead of being silently lost. Also rewrites the late-admitted-raw-id test vehicle (null project keys now fail strict loads) to model enumeration divergence directly. --- src/node/config.test.ts | 7 ++ src/node/config.ts | 13 +++ .../services/ExtensionMetadataService.test.ts | 107 ++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 61 ++++++++-- src/node/services/workspaceService.test.ts | 24 ++-- 5 files changed, 198 insertions(+), 14 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 59cd1c5ec6..aaf22c2c2c 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -918,6 +918,13 @@ describe("Config", () => { // incomplete — strict mode must not vouch "authoritatively empty" for // it (the prune would delete every snapshot of that project). ["missing workspaces key", { projects: [["/repo", {}]] }], + // The lenient path-filter silently drops the WHOLE project for empty + // or non-string keys, and an id-less legacy workspace inside it is + // raw-invisible too (its stable id lives only in session + // metadata.json) — strict mode must fail closed rather than hand the + // prune an id set missing that workspace. + ["null project key", { projects: [[null, { workspaces: [] }]] }], + ["empty-string project key", { projects: [["", { workspaces: [] }]] }], ]; for (const [label, shape] of invalidShapes) { diff --git a/src/node/config.ts b/src/node/config.ts index 9bc96e4932..866e4037aa 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1429,6 +1429,19 @@ export class Config { if (!Array.isArray(pair)) { throw new Error("Config projects entries must be [path, config] pairs"); } + const projectKey: unknown = pair[0]; + // The lenient normalization below silently drops the WHOLE + // project when its key is empty or non-string ("Filtering out + // project with invalid path"), and an id-less legacy workspace + // inside it is raw-invisible too (its stable id lives only in + // session metadata.json, which only the normalized enumeration + // resolves). Accepting the pair here would hand destructive + // strict callers an authoritative id set missing every one of + // that project's workspaces — the startup prune would then + // permanently delete their recency/goal/status snapshots. + if (typeof projectKey !== "string" || projectKey.length === 0) { + throw new Error("Config project entries must have a non-empty string path"); + } const projectConfig: unknown = pair[1]; // Arrays pass typeof "object": lenient normalization would turn // an array-valued project config into a project with no diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index ad2f2210c0..44472c4daa 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1032,6 +1032,113 @@ describe("ExtensionMetadataService", () => { expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{corrupt json"); }); + test("a healthy main raced into the quarantine move-aside is restored and merged, not lost", async () => { + // Multi-instance race: the in-queue corruption check sees a corrupt main + // and an existing healthy sidecar, then another backend's atomic save + // lands a NEWER healthy main before the move-aside. Nothing ever reads + // the .recreated leftover, so without revalidating the moved bytes the + // newer update would be silently lost while the OLDER sidecar restores. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-old": { recency: 700, streaming: false } }, + }) + ); + await writeFile(filePath, "{corrupt json"); + const internals = service as unknown as { + moveMainAsideAsRecreatedLeftover: () => Promise; + }; + const realMove = internals.moveMainAsideAsRecreatedLeftover.bind(service); + internals.moveMainAsideAsRecreatedLeftover = async () => { + // The concurrent backend's save landing inside the race window. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 900, streaming: false } }, + }) + ); + internals.moveMainAsideAsRecreatedLeftover = realMove; + return realMove(); + }; + try { + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-new")?.recency).toBe(900); + expect(snapshots.get("ws-old")?.recency).toBe(700); + } finally { + internals.moveMainAsideAsRecreatedLeftover = realMove; + } + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("a failing sidecar token probe during consumption propagates instead of reporting success", async () => { + // consumeQuarantineSidecar's post-merge stat() guards which sidecar + // generation gets unlinked. If a transient EACCES/EIO there were + // swallowed as success, the caller would proceed — e.g. the one-time + // prune deletes stale entries, the NEXT snapshot read re-merges them + // from the retained sidecar and consumes it, and with the prune latch + // already set the resurrected entries stay indefinitely. It must + // propagate (retryable) instead. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + const statics = ExtensionMetadataService as unknown as { + statQuarantineToken: (quarantinePath: string) => Promise; + }; + const realStat = statics.statQuarantineToken.bind(ExtensionMetadataService); + // Call #1 captures the reconcile's generation token (real); call #2 is + // the consumption-time identity probe this test degrades. + let statCalls = 0; + statics.statQuarantineToken = async (quarantinePath: string) => { + statCalls += 1; + if (statCalls === 2) { + const error = new Error("EACCES: permission denied, stat") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + } + return realStat(quarantinePath); + }; + try { + let strictRejected = false; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch { + strictRejected = true; + } + expect(strictRejected).toBe(true); + // Sidecar retained for the retry: never consumed on unverifiable identity. + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toContain("ws-stranded"); + } finally { + statics.statQuarantineToken = realStat; + } + // Retry with the probe healthy: the re-merge is idempotent and + // consumption completes. + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-stranded")?.recency).toBe(700); + expect(snapshots.get("ws-live")?.recency).toBe(900); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("a strict snapshot read propagates a failed sidecar reconcile instead of the partial main", async () => { // Live emissions read per-workspace snapshots after the subscription // bootstraps. With a sidecar stranded next to a recreated partial main diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 60c7cef961..33585a6a07 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -358,13 +358,13 @@ export class ExtensionMetadataService { // fail closed by leaving the file rather than unlinking blind. return; } - let current: QuarantineSidecarToken | null; - try { - current = await ExtensionMetadataService.statQuarantineToken(quarantinePath); - } catch { - // Unprobeable: leave the sidecar (retryable) rather than guessing. - return; - } + // Non-ENOENT probe failures propagate (retryable): reporting success + // with the sidecar retained would let the caller proceed — e.g. the + // one-time prune deletes stale entries from the main file, the next + // snapshot read re-merges them from the retained sidecar and consumes + // it, and with the prune latch already set the resurrected entries + // stay indefinitely. statQuarantineToken maps ENOENT to null below. + const current = await ExtensionMetadataService.statQuarantineToken(quarantinePath); if (current == null) { return; // Already consumed by a concurrent recovery. } @@ -1118,6 +1118,53 @@ export class ExtensionMetadataService { // propagates so the read stays retryable on unknowable evidence. if (await this.probeQuarantineSidecar()) { await this.moveMainAsideAsRecreatedLeftover(); + // Same cross-process race completeQuarantineRecovery closes for the + // rename-to-.corrupt branch: another backend's atomic save can land + // a NEWER healthy main between the in-queue corruption check above + // and the move — nothing ever reads the leftover, so the newer + // update would be silently lost while the OLDER sidecar restores. + // Re-validate what actually got moved; when it turns out healthy + // (or a preserved newer schema), restore it to the vacant main path + // and merge the existing sidecar into it via the recreated-main + // reconcile. Transient read failures propagate (retryable) — only + // deterministic corruption proceeds to the sidecar-restore path. + const leftoverPath = `${this.filePath}.recreated`; + let movedRaw: unknown; + let movedParses = true; + try { + movedRaw = JSON.parse(await readFile(leftoverPath, "utf-8")) as unknown; + } catch (readError) { + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + movedParses = false; + } + if ( + movedParses && + (ExtensionMetadataService.isValidMetadataFileShape(movedRaw) || + ExtensionMetadataService.isUnsupportedVersion(movedRaw)) + ) { + // Restore without overwriting a file yet another writer re-created + // at the main path (EEXIST): the reconcile below merges the + // sidecar into whichever file now owns the path. + try { + await link(leftoverPath, this.filePath); + } catch (linkError) { + if (!ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + try { + await copyFile(leftoverPath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + // Main path missing with the raced bytes only in the + // leftover: rethrow (retryable) rather than letting the + // sidecar restore over the vacant path and orphan them. + throw copyError; + } + } + } + } + return this.reconcileRecreatedMainWithSidecar(quarantinePath); + } return this.completeQuarantineRecovery(quarantinePath); } await rename(this.filePath, quarantinePath); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 352592cbc4..93fc9607d9 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5639,23 +5639,29 @@ describe("WorkspaceService activity list scoping", () => { try { const workspaceId = "raw-only-live"; const projectPath = path.join(config.rootDir, "project"); - // A null project path survives the raw id scan but is dropped by the - // normalized enumeration, keeping the id out of the per-id scope. // Migration flags pre-seeded: without them the first load schedules an // async settings-migration persist that rewrites config.json through - // the parsed view — which DROPS the null-path project — deregistering - // this raw-only id mid-test whenever the persist happens to land - // before the second list's raw reads (observed flake). + // the parsed view mid-test whenever it happens to land before the + // second list's raw reads (observed flake). await fsPromises.writeFile( path.join(config.rootDir, "config.json"), JSON.stringify({ projects: [ - [null, { workspaces: [{ id: workspaceId, path: path.join(projectPath, "ws") }] }], + [ + projectPath, + { workspaces: [{ id: workspaceId, path: path.join(projectPath, "ws") }] }, + ], ], taskSettings: { preserveSubagentsUntilArchive: true }, migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, }) ); + // Raw-visible but enumeration-invisible: the strict normalized + // enumeration resolves no ids while the raw persisted view carries the + // inline id, keeping it out of the per-id scope so it takes the + // late-candidate path. (Strict loads now reject the previously used + // malformed-project-key vehicle, so divergence is modeled directly.) + const enumerateSpy = spyOn(config, "getAllWorkspaceMetadata").mockResolvedValue([]); const extensionMetadata = new ExtensionMetadataService( path.join(config.rootDir, "extensionMetadata.json") ); @@ -5688,6 +5694,7 @@ describe("WorkspaceService activity list scoping", () => { expect(secondList?.[workspaceId]?.recency).toBe(42); } finally { snapshotsSpy.mockRestore(); + enumerateSpy.mockRestore(); } } finally { await cleanup(); @@ -7407,7 +7414,10 @@ describe("WorkspaceService activity list scoping", () => { // A parseable config entry that lenient normalization filters out (null // project path): the workspace vanishes from the normalized view — and // thus from the activity list, matching every other renderer surface — - // but its metadata entry must survive the prune (raw-superset guarantee). + // but its metadata entry must survive the prune. Two guards enforce it: + // strict loads reject the malformed project key outright (aborting the + // prune, fail closed), and the raw-superset union spares the inline id + // even if enumeration were to succeed. const { config, historyService, cleanup } = await createTestHistoryService(); try { const extensionMetadata = new ExtensionMetadataService( From 12a4610dfa8b047adfc9546e2e98cd26d3ecf39c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 09:17:27 +0000 Subject: [PATCH 58/72] fix: address round-52 Codex findings 1. consumeQuarantineSidecar claims the sidecar via atomic rename BEFORE verifying identity, closing the stat-to-unlink window in which a newer generation installed by another backend could be blindly destroyed. A claimed foreign generation is reconciled into the main file; crash- stranded claims are resumed by the leftover probe on authoritative reads. 2. Strict config loads reject non-object workspace entries and truthy non-string / empty-string workspace ids: a malformed id would ride the modern-entry branch as authoritative, omitting the real identity from the prune's known set. 3. getSnapshot routes deterministic corruption through the shared quarantine-and-reread recovery (loadWithCorruptionRecovery) so emit-path reads self-heal instead of dropping emissions forever. 4. pruneMissingWorkspaces loads its fresh snapshot strictly AFTER the re-registration recheck (the longest await), so concurrent writes landing during the recheck are not rolled back by the pruned rewrite; an unchanged-bytes guard keeps the inverse re-registration race fail-closed. --- src/node/config.test.ts | 13 ++ src/node/config.ts | 26 +++ .../services/ExtensionMetadataService.test.ts | 116 ++++++++-- src/node/services/ExtensionMetadataService.ts | 199 ++++++++++++++---- 4 files changed, 288 insertions(+), 66 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index aaf22c2c2c..c98dc3db1f 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -925,6 +925,19 @@ describe("Config", () => { // prune an id set missing that workspace. ["null project key", { projects: [[null, { workspaces: [] }]] }], ["empty-string project key", { projects: [["", { workspaces: [] }]] }], + // A truthy non-string workspace id would ride getAllWorkspaceMetadata's + // modern-entry branch as the authoritative id, omitting the workspace's + // REAL string identity from the prune's known set; non-object entries + // have no establishable identity at all. + ["non-object workspace entry", { projects: [["/repo", { workspaces: ["bogus"] }]] }], + [ + "numeric workspace id", + { projects: [["/repo", { workspaces: [{ id: 42, path: "/repo/ws" }] }]] }, + ], + [ + "empty-string workspace id", + { projects: [["/repo", { workspaces: [{ id: "", path: "/repo/ws" }] }]] }, + ], ]; for (const [label, shape] of invalidShapes) { diff --git a/src/node/config.ts b/src/node/config.ts index 866e4037aa..4d9c0b4390 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1463,6 +1463,32 @@ export class Config { if (!Array.isArray(workspaces)) { throw new Error("Config project workspaces must be an array"); } + for (const workspaceEntry of workspaces) { + // WorkspaceSchema persists workspace entries as objects; a + // non-object entry is mangled state whose identity cannot be + // established. + if ( + workspaceEntry === null || + typeof workspaceEntry !== "object" || + Array.isArray(workspaceEntry) + ) { + throw new Error("Config workspace entries must be objects"); + } + const workspaceId = (workspaceEntry as { id?: unknown }).id; + // A truthy non-string id (42, {}) would ride the modern-entry + // branch of getAllWorkspaceMetadata as the authoritative id, + // so the prune's known set would omit the workspace's REAL + // string identity and delete its activity snapshot. Nullish + // ids stay valid: legacy entries resolve their stable id + // through session metadata.json, and the strict guard there + // fails closed when that resolution yields no usable id. + if ( + workspaceId != null && + !(typeof workspaceId === "string" && workspaceId.length > 0) + ) { + throw new Error("Config workspace ids must be non-empty strings"); + } + } } } const rawPairs = Array.isArray(parsed.projects) ? parsed.projects : []; diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 44472c4daa..af9d4c4f12 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -539,6 +539,50 @@ describe("ExtensionMetadataService", () => { expect(persisted.workspaces["stale-workspace"]).toBeUndefined(); }); + test("pruneMissingWorkspaces preserves writes landing during the re-registration recheck", async () => { + // The recheck can perform a full legacy enumeration — the longest await + // in the prune. A concurrent backend's write landing during it must not + // be rolled back by the pruned rewrite: the fresh snapshot is loaded + // strictly AFTER the recheck resolves. + await service.updateRecency("known-workspace", 100); + await service.updateRecency("stale-workspace", 200); + const recheck = async (): Promise> => { + // Foreign process write landing while the recheck enumerates. + const onDisk = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + (onDisk.workspaces["known-workspace"] as { recency: number }).recency = 999; + await writeFile(filePath, JSON.stringify(onDisk)); + return new Set(["known-workspace"]); + }; + expect( + await service.pruneMissingWorkspaces( + () => Promise.resolve(new Set(["known-workspace"])), + recheck + ) + ).toBe(1); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect((persisted.workspaces["known-workspace"] as { recency?: number }).recency).toBe(999); + expect(persisted.workspaces["stale-workspace"]).toBeUndefined(); + }); + + test("pruneMissingWorkspaces spares a stale entry rewritten during the recheck", async () => { + // Inverse race of the post-recheck reload: an id re-registered and + // written by another backend after the recheck read its registration + // evidence is still classified deletable while holding fresh activity. + // The unchanged-bytes guard proves the writer and fails closed. + await service.updateRecency("stale-workspace", 200); + const recheck = async (): Promise> => { + const onDisk = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + (onDisk.workspaces["stale-workspace"] as { recency: number }).recency = 999; + await writeFile(filePath, JSON.stringify(onDisk)); + return new Set(); // Registration evidence predates the write. + }; + expect( + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set()), recheck) + ).toBe(0); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect((persisted.workspaces["stale-workspace"] as { recency?: number }).recency).toBe(999); + }); + test("late writers cannot resurrect entries reclaimed by pruneMissingWorkspaces", async () => { await service.updateRecency("stale-workspace", 100); await service.pruneMissingWorkspaces(() => Promise.resolve(new Set())); @@ -945,18 +989,24 @@ describe("ExtensionMetadataService", () => { test("consumption is scoped to the sidecar generation that was reconciled", async () => { // With multiple processes recovering the fixed .corrupt path, another // backend can consume the generation this process read and strand a NEW - // snapshot at the same path before the unlink runs. A path-only unlink - // would destroy that unreconciled generation permanently. + // snapshot at the same path before the removal runs. A path-only unlink + // would destroy that unreconciled generation permanently; the + // claim-then-verify consume must instead take exactly one generation off + // the shared path, detect the identity mismatch on the claimed file, and + // reconcile the foreign generation into the main file. const quarantinePath = `${filePath}.corrupt`; - const internals = ExtensionMetadataService as unknown as { + const statics = ExtensionMetadataService as unknown as { statQuarantineToken(path: string): Promise; + }; + const internals = service as unknown as { consumeQuarantineSidecar(path: string, token: unknown): Promise; }; + await writeFile(filePath, JSON.stringify({ version: 1, workspaces: {} })); await writeFile( quarantinePath, JSON.stringify({ version: 1, workspaces: { a: { recency: 1, streaming: false } } }) ); - const staleToken = await internals.statQuarantineToken(quarantinePath); + const staleToken = await statics.statQuarantineToken(quarantinePath); // Concurrent recovery consumes that generation and quarantines a new // snapshot at the same path (different content => different identity). await rm(quarantinePath); @@ -966,19 +1016,21 @@ describe("ExtensionMetadataService", () => { ); await internals.consumeQuarantineSidecar(quarantinePath, staleToken); - // The newer generation survives for its own recovery pass. - const survivor = JSON.parse(await readFile(quarantinePath, "utf-8")) as { - workspaces: Record; + // The newer generation was reconciled into the main file, not destroyed. + const main = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; }; - expect(survivor.workspaces.b).toBeDefined(); - - const currentToken = await internals.statQuarantineToken(quarantinePath); - await internals.consumeQuarantineSidecar(quarantinePath, currentToken); - const gone = await readFile(quarantinePath, "utf-8").then( + expect(main.workspaces.b?.recency).toBe(22222); + const sidecarGone = await readFile(quarantinePath, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + const claimGone = await readFile(`${quarantinePath}.claimed`, "utf-8").then( () => false, () => true ); - expect(gone).toBe(true); + expect(claimGone).toBe(true); }); test("quarantine preserves an existing healthy sidecar instead of clobbering it", async () => { @@ -1122,21 +1174,41 @@ describe("ExtensionMetadataService", () => { strictRejected = true; } expect(strictRejected).toBe(true); - // Sidecar retained for the retry: never consumed on unverifiable identity. - expect(await readFile(`${filePath}.corrupt`, "utf-8")).toContain("ws-stranded"); + // Bytes retained for the retry: the consume claimed the sidecar before + // the failing identity probe, so they now sit at the claim path and + // are never deleted on unverifiable identity. + expect(await readFile(`${filePath}.corrupt.claimed`, "utf-8")).toContain("ws-stranded"); } finally { statics.statQuarantineToken = realStat; } - // Retry with the probe healthy: the re-merge is idempotent and - // consumption completes. + // Retry with the probe healthy: the claim-leftover resume re-merges + // idempotently and consumption completes. const snapshots = await service.getAllSnapshots({ throwOnError: true }); expect(snapshots.get("ws-stranded")?.recency).toBe(700); expect(snapshots.get("ws-live")?.recency).toBe(900); - const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( - () => false, - () => true - ); - expect(sidecarGone).toBe(true); + for (const leftover of [`${filePath}.corrupt`, `${filePath}.corrupt.claimed`]) { + const gone = await readFile(leftover, "utf-8").then( + () => false, + () => true + ); + expect(gone).toBe(true); + } + }); + + test("a strict per-workspace read self-heals deterministic corruption", async () => { + // Live emissions read per-workspace snapshots after the subscription + // bootstraps. If the file becomes deterministically corrupt afterwards, + // the workflow-run/bash-monitor handlers drop their emissions on error — + // and nothing else is guaranteed to repair the file on an idle process, + // pinning stale activity in the renderer indefinitely. The strict read + // must route corruption through the same quarantine-and-reread recovery + // getAllSnapshots uses instead of failing every retry forever. + await writeFile(filePath, "{corrupt json"); + const snapshot = await service.getSnapshot("ws-any", { throwOnError: true }); + expect(snapshot).toBeNull(); + // The canonical file was quarantined and reset to a valid empty file. + const healed = JSON.parse(await readFile(filePath, "utf-8")) as { version?: number }; + expect(healed.version).toBe(1); }); test("a strict snapshot read propagates a failed sidecar reconcile instead of the partial main", async () => { diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 33585a6a07..df76e88c33 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -338,48 +338,95 @@ export class ExtensionMetadataService { * with the sidecar still present would let a later process reconcile the * stale sidecar again after the main file has moved on, repeatedly * re-merging entries that pruning or removal already reclaimed. + * + * Claim-then-verify: with multiple processes recovering the fixed sidecar + * path, another backend can consume the generation this caller read and + * quarantine a NEW snapshot at the same path — a stat-compare followed by + * a path unlink would leave a window in which that unreconciled newer + * generation is destroyed. The rename below atomically takes exactly one + * generation off the shared path first; identity is then verified on the + * claimed file, which no other recovery references by design. A claimed + * FOREIGN generation (identity mismatch) is reconciled into the main file + * rather than deleted. Invariant: a claim file whose identity matched its + * consumer's token holds already-reconciled bytes (callers merge/restore + * before consuming), so only a crash between a MISMATCH claim and its + * reconcile strands unconsumed data at the claim path — + * reconcileLeftoverSidecarIfPresent probes for exactly that and resumes + * the merge on the next authoritative read. */ - private static async consumeQuarantineSidecar( + private async consumeQuarantineSidecar( quarantinePath: string, token: QuarantineSidecarToken | null ): Promise { - // Generation check: with multiple processes recovering the fixed sidecar - // path, another backend can consume the generation this caller read and - // quarantine a NEW snapshot at the same path before this unlink runs — a - // path-only unlink would permanently destroy that unreconciled - // generation. Compare the identity captured when the caller READ the - // sidecar and leave a mismatched (newer) file for its own recovery pass. - // The residual stat→unlink window is the documented unlocked-shared-file - // boundary (see the PR's concurrency contract); closing it entirely - // requires an interprocess lock. if (token == null) { // The caller never observed a sidecar identity (it read the file only // after a successful probe, so this is unreachable in practice) — // fail closed by leaving the file rather than unlinking blind. return; } - // Non-ENOENT probe failures propagate (retryable): reporting success - // with the sidecar retained would let the caller proceed — e.g. the - // one-time prune deletes stale entries from the main file, the next - // snapshot read re-merges them from the retained sidecar and consumes - // it, and with the prune latch already set the resurrected entries - // stay indefinitely. statQuarantineToken maps ENOENT to null below. - const current = await ExtensionMetadataService.statQuarantineToken(quarantinePath); + const claimPath = `${quarantinePath}.claimed`; + // A leftover at the claim path is either already-reconciled bytes (crash + // between a matched claim and its unlink) or a crash-stranded foreign + // generation (see the invariant above): reconcile it first so nothing + // recoverable is lost, then clear the name — the rename below must not + // land on an occupied destination (Windows rename is not reliably a + // replace). Post-reconcile bytes are consumed or deterministically + // unrecoverable, so the unlink only ever discards dead bytes. + const claimLeftoverExists = await access(claimPath).then( + () => true, + (error: unknown) => { + if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + return false; + } + throw error; // Unknowable claim state: retryable, never unlink blind. + } + ); + if (claimLeftoverExists) { + await this.reconcileRecreatedMainWithSidecar(claimPath); + try { + await unlink(claimPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + } + try { + await rename(quarantinePath, claimPath); + } catch (renameError) { + if (ExtensionMetadataService.isErrnoCode(renameError, "ENOENT")) { + return; // Already consumed by a concurrent recovery. + } + throw renameError; + } + // Identity check on the CLAIMED file (immutable to other recoveries): + // non-ENOENT probe failures propagate (retryable) with the claim left in + // place for the leftover resume above — reporting success would let the + // caller proceed, e.g. the one-time prune deletes stale entries, a later + // read re-merges them from a retained sidecar, and with the prune latch + // already set the resurrected entries stay indefinitely. + const current = await ExtensionMetadataService.statQuarantineToken(claimPath); if (current == null) { - return; // Already consumed by a concurrent recovery. + return; // Claim vanished (external cleanup); nothing left to consume. } if ( current.ino !== token.ino || current.mtimeNs !== token.mtimeNs || current.size !== token.size ) { - log.debug("Leaving replaced quarantine sidecar for its own recovery pass", { + // The claim took a NEWER generation another process installed after + // consuming ours: merge it into the main file instead of destroying + // it. The reconcile consumes the claim itself on success; a corrupt + // foreign generation stays at the bounded claim name until the next + // consume's leftover pass discards it. + log.debug("Claimed a replaced quarantine sidecar generation; reconciling it", { quarantinePath, }); + await this.reconcileRecreatedMainWithSidecar(claimPath); return; } try { - await unlink(quarantinePath); + await unlink(claimPath); } catch (unlinkError) { if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { throw unlinkError; @@ -416,10 +463,38 @@ export class ExtensionMetadataService { * emissions). */ private async reconcileLeftoverSidecarIfPresent(): Promise { - if (!(await this.probeQuarantineSidecar())) { + if (await this.probeQuarantineSidecar()) { + await this.resumeQuarantineRecovery(); + return true; + } + // A crash between consumeQuarantineSidecar's mismatch claim and its + // reconcile strands an unreconciled foreign generation at the fixed + // claim path, which the sidecar probe above cannot see. Same access() + // cost profile: the reconcile only runs when a claim file actually + // exists (re-checked inside the queue — a concurrent consume's leftover + // pass may have taken it while this caller waited). + const claimPath = `${this.filePath}.corrupt.claimed`; + const claimExists = await access(claimPath).then( + () => true, + (error: unknown) => { + if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + return false; + } + throw error; // Unknowable claim state stays a retryable failure. + } + ); + if (!claimExists) { return false; } - await this.resumeQuarantineRecovery(); + await this.withSerializedMutation(async () => { + const stillExists = await access(claimPath).then( + () => true, + () => false + ); + if (stillExists) { + await this.reconcileRecreatedMainWithSidecar(claimPath); + } + }); return true; } @@ -784,7 +859,7 @@ export class ExtensionMetadataService { } log.debug("Leftover sidecar reconcile failed during snapshot read", { error }); } - const data = await this.load(options); + const data = await this.loadWithCorruptionRecovery(options); return this.toSnapshot(data.workspaces[workspaceId]); } @@ -966,17 +1041,25 @@ export class ExtensionMetadataService { if (staleWorkspaceIds.length === 0) { return 0; } - // Deletion-only merge against a fresh snapshot (see doc comment above). - const fresh = await this.load(); - // Re-fetch the known ids AFTER the fresh load: with multiple instances - // a downgraded backend can re-register a deterministic legacy id (and - // write new activity for it) between the enumeration above and here. - // Deleting on the stale classification would destroy that new entry's + // Re-registration recheck: with multiple instances a downgraded + // backend can re-register a deterministic legacy id (and write new + // activity for it) between the enumeration above and here. Deleting + // on the stale classification would destroy that new entry's // recency/goal/status — clearing the tombstone later cannot restore // data. A re-registered id is dropped from the deletion set and its // write tombstone lifted. If the recheck fails, abort the prune (throw // to the caller's catch) rather than deleting on stale knowledge. const recheckedKnownIds = await (recheckKnownWorkspaceIds ?? getKnownWorkspaceIds)(); + // Deletion-only merge against a fresh snapshot loaded strictly AFTER + // the recheck — the LAST await before the save below. The recheck can + // perform a full legacy enumeration, and any recency/goal/status + // another backend writes during that await would be absent from a + // pre-recheck snapshot: save() would silently roll it back while + // deleting the stale keys. The inverse race (an id re-registered + // after the recheck read but before this load) is covered by the + // unchanged-bytes guard below, which is strictly narrower than the + // enumeration-wide window this ordering closes. + const fresh = await this.load(); let prunedCount = 0; for (const workspaceId of staleWorkspaceIds) { if (recheckedKnownIds.has(workspaceId)) { @@ -993,10 +1076,25 @@ export class ExtensionMetadataService { } continue; } - if (workspaceId in fresh.workspaces) { - delete fresh.workspaces[workspaceId]; - prunedCount++; + if (!(workspaceId in fresh.workspaces)) { + continue; + } + // Fail-closed bytes guard: the registration evidence above predates + // this load, so an id re-registered in that gap says "unregistered" + // while a concurrent backend may already have written fresh + // activity for it. A stale-classified entry whose bytes CHANGED + // between the two loads proves such a writer — spare it (the entry + // is re-evaluated on the next process start; the write tombstone + // stays until fresh registration evidence clears it through the + // normal revival paths). + if ( + JSON.stringify(fresh.workspaces[workspaceId]) !== + JSON.stringify(data.workspaces[workspaceId]) + ) { + continue; } + delete fresh.workspaces[workspaceId]; + prunedCount++; } if (prunedCount > 0) { await this.save(fresh); @@ -1268,7 +1366,7 @@ export class ExtensionMetadataService { throw copyError; } } - await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath, sidecarToken); + await this.consumeQuarantineSidecar(quarantinePath, sidecarToken); return false; } // Replace the quarantined main file with a valid EMPTY file instead of @@ -1420,7 +1518,7 @@ export class ExtensionMetadataService { throw copyError; } } - await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath, sidecarToken); + await this.consumeQuarantineSidecar(quarantinePath, sidecarToken); return false; } if (!ExtensionMetadataService.isValidMetadataFileShape(sidecarParsed)) { @@ -1550,21 +1648,28 @@ export class ExtensionMetadataService { } // Consumed either way: every surviving sidecar entry is now represented // at the main path. - await ExtensionMetadataService.consumeQuarantineSidecar(quarantinePath, sidecarToken); + await this.consumeQuarantineSidecar(quarantinePath, sidecarToken); return false; } - async getAllSnapshots(options?: { + /** + * load() with self-healing for deterministic corruption. Strict reads must + * propagate transient failures (renderer keeps last-known state and + * retries), but deterministic corruption would fail every retry forever on + * an idle process — no subsequent list read or metadata writer is + * guaranteed to repair the file. Quarantine the corrupt bytes and re-read: + * the post-quarantine state is authoritative. Shared by getAllSnapshots + * and getSnapshot so per-workspace emit reads (workflow-run/bash-monitor + * handlers, which drop their emissions on error) self-heal the same way + * instead of leaving a workflow-only workspace's stale activity pinned in + * the renderer indefinitely. + */ + private async loadWithCorruptionRecovery(options?: { throwOnError?: boolean; - }): Promise> { - let data: ExtensionMetadataFile; + }): Promise { try { - data = await this.load(options); + return await this.load(options); } catch (error) { - // Strict reads must propagate transient failures (renderer keeps - // last-known state and retries), but deterministic corruption would - // fail every retry forever on an idle process. Quarantine the corrupt - // bytes and re-read: the post-quarantine empty state is authoritative. if (ExtensionMetadataService.isDeterministicCorruption(error)) { try { await this.quarantineCorruptFile(); @@ -1589,8 +1694,14 @@ export class ExtensionMetadataService { } else { throw error; } - data = await this.load(options); + return this.load(options); } + } + + async getAllSnapshots(options?: { + throwOnError?: boolean; + }): Promise> { + let data: ExtensionMetadataFile = await this.loadWithCorruptionRecovery(options); // A crash between quarantineCorruptFile's rename and its completion can // strand the full snapshot in the sidecar while another backend // recreates a VALID (typically partial, single-entry) main file from From c4d8c8f0bc678d6a20f038a10dfa4f12aa4e7862 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 09:36:10 +0000 Subject: [PATCH 59/72] fix: address round-53 Codex findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Claim files use unique per-consume names (pid + uuid) discovered by prefix: a fixed claim name let a concurrent recovery's rename land on — or its leftover pass clear — another recovery's live claim, re-opening the destroyed-generation window the claim exists to close. 2. The stranded-claim discovery propagates non-ENOENT probe failures (and readdir failures) instead of reporting success, so strict reads never vouch for a possibly partial main while recoverable fields sit in an unprobeable claim. 3. The registration probe tries a targeted lenient findWorkspace positive before the strict enumeration: a re-registered workspace with healthy metadata no longer stays write-suppressed forever because an UNRELATED legacy entry's compatibility file is malformed. Negatives still require the complete strict view (fail closed). --- .../services/ExtensionMetadataService.test.ts | 33 +++-- src/node/services/ExtensionMetadataService.ts | 122 +++++++++--------- src/node/services/coreServices.ts | 15 ++- 3 files changed, 90 insertions(+), 80 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index af9d4c4f12..13187d7730 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "fs/promises"; import { tmpdir } from "os"; import * as path from "path"; import { ExtensionMetadataService } from "./ExtensionMetadataService"; @@ -1026,11 +1026,9 @@ describe("ExtensionMetadataService", () => { () => true ); expect(sidecarGone).toBe(true); - const claimGone = await readFile(`${quarantinePath}.claimed`, "utf-8").then( - () => false, - () => true - ); - expect(claimGone).toBe(true); + // No claim file left behind (unique names; discovered by prefix). + const leftovers = (await readdir(tempDir)).filter((name) => name.includes(".corrupt-claim-")); + expect(leftovers).toEqual([]); }); test("quarantine preserves an existing healthy sidecar instead of clobbering it", async () => { @@ -1175,24 +1173,25 @@ describe("ExtensionMetadataService", () => { } expect(strictRejected).toBe(true); // Bytes retained for the retry: the consume claimed the sidecar before - // the failing identity probe, so they now sit at the claim path and - // are never deleted on unverifiable identity. - expect(await readFile(`${filePath}.corrupt.claimed`, "utf-8")).toContain("ws-stranded"); + // the failing identity probe, so they now sit at a unique claim name + // and are never deleted on unverifiable identity. + const claims = (await readdir(tempDir)).filter((name) => name.includes(".corrupt-claim-")); + expect(claims.length).toBe(1); + expect(await readFile(path.join(tempDir, claims[0]), "utf-8")).toContain("ws-stranded"); } finally { statics.statQuarantineToken = realStat; } - // Retry with the probe healthy: the claim-leftover resume re-merges + // Retry with the probe healthy: the stranded-claim discovery re-merges // idempotently and consumption completes. const snapshots = await service.getAllSnapshots({ throwOnError: true }); expect(snapshots.get("ws-stranded")?.recency).toBe(700); expect(snapshots.get("ws-live")?.recency).toBe(900); - for (const leftover of [`${filePath}.corrupt`, `${filePath}.corrupt.claimed`]) { - const gone = await readFile(leftover, "utf-8").then( - () => false, - () => true - ); - expect(gone).toBe(true); - } + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + expect((await readdir(tempDir)).filter((name) => name.includes(".corrupt-claim-"))).toEqual([]); }); test("a strict per-workspace read self-heals deterministic corruption", async () => { diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index df76e88c33..93da7a272f 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1,7 +1,9 @@ -import { dirname } from "path"; +import { basename, dirname, join } from "path"; +import { randomUUID } from "crypto"; import { mkdir, readFile, + readdir, access, rename, link, @@ -76,6 +78,13 @@ export interface ExtensionMetadataStreamingUpdate { export class ExtensionMetadataService { private readonly filePath: string; + + /** + * Name infix for per-consume claim files (`
-`). + * Unique names keep concurrent recoveries from replacing or clearing each + * other's live claims; stranded claims are discovered by this prefix. + */ + private static readonly CLAIM_INFIX = ".corrupt-claim-"; private mutationQueue: Promise = Promise.resolve(); /** * Per-process write tombstones for removed workspaces. Workspace removal @@ -345,14 +354,19 @@ export class ExtensionMetadataService { * a path unlink would leave a window in which that unreconciled newer * generation is destroyed. The rename below atomically takes exactly one * generation off the shared path first; identity is then verified on the - * claimed file, which no other recovery references by design. A claimed + * claimed file. The claim name is UNIQUE per consume (pid + uuid): a fixed + * name would let a concurrent consumer's claim land on (POSIX rename + * replaces) or clear (its own leftover pass) another recovery's live + * claim, re-opening the destroyed-generation window this claim exists to + * close — and a fresh unique destination also never collides on Windows, + * where rename onto an existing file is not reliably a replace. A claimed * FOREIGN generation (identity mismatch) is reconciled into the main file * rather than deleted. Invariant: a claim file whose identity matched its * consumer's token holds already-reconciled bytes (callers merge/restore * before consuming), so only a crash between a MISMATCH claim and its - * reconcile strands unconsumed data at the claim path — - * reconcileLeftoverSidecarIfPresent probes for exactly that and resumes - * the merge on the next authoritative read. + * reconcile strands unconsumed data at a claim name — + * reconcileLeftoverSidecarIfPresent discovers stranded claims by prefix + * and resumes the merge on the next authoritative read. */ private async consumeQuarantineSidecar( quarantinePath: string, @@ -364,33 +378,7 @@ export class ExtensionMetadataService { // fail closed by leaving the file rather than unlinking blind. return; } - const claimPath = `${quarantinePath}.claimed`; - // A leftover at the claim path is either already-reconciled bytes (crash - // between a matched claim and its unlink) or a crash-stranded foreign - // generation (see the invariant above): reconcile it first so nothing - // recoverable is lost, then clear the name — the rename below must not - // land on an occupied destination (Windows rename is not reliably a - // replace). Post-reconcile bytes are consumed or deterministically - // unrecoverable, so the unlink only ever discards dead bytes. - const claimLeftoverExists = await access(claimPath).then( - () => true, - (error: unknown) => { - if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { - return false; - } - throw error; // Unknowable claim state: retryable, never unlink blind. - } - ); - if (claimLeftoverExists) { - await this.reconcileRecreatedMainWithSidecar(claimPath); - try { - await unlink(claimPath); - } catch (unlinkError) { - if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { - throw unlinkError; - } - } - } + const claimPath = `${this.filePath}${ExtensionMetadataService.CLAIM_INFIX}${process.pid}-${randomUUID()}`; try { await rename(quarantinePath, claimPath); } catch (renameError) { @@ -399,15 +387,17 @@ export class ExtensionMetadataService { } throw renameError; } - // Identity check on the CLAIMED file (immutable to other recoveries): - // non-ENOENT probe failures propagate (retryable) with the claim left in - // place for the leftover resume above — reporting success would let the - // caller proceed, e.g. the one-time prune deletes stale entries, a later - // read re-merges them from a retained sidecar, and with the prune latch - // already set the resurrected entries stay indefinitely. + // Identity check on the CLAIMED file (nothing else references the + // unique name except the stranded-claim discovery, which reconciles + // rather than destroys): non-ENOENT probe failures propagate + // (retryable) with the claim left for that discovery — reporting + // success would let the caller proceed, e.g. the one-time prune deletes + // stale entries, a later read re-merges them from a retained sidecar, + // and with the prune latch already set the resurrected entries stay + // indefinitely. const current = await ExtensionMetadataService.statQuarantineToken(claimPath); if (current == null) { - return; // Claim vanished (external cleanup); nothing left to consume. + return; // Claim vanished (concurrent discovery consumed it). } if ( current.ino !== token.ino || @@ -417,8 +407,9 @@ export class ExtensionMetadataService { // The claim took a NEWER generation another process installed after // consuming ours: merge it into the main file instead of destroying // it. The reconcile consumes the claim itself on success; a corrupt - // foreign generation stays at the bounded claim name until the next - // consume's leftover pass discards it. + // foreign generation stays at the claim name until the discovery + // pass's reconcile classifies it (bounded: one file per crashed or + // failed recovery, consumed on the next successful pass). log.debug("Claimed a replaced quarantine sidecar generation; reconciling it", { quarantinePath, }); @@ -468,31 +459,38 @@ export class ExtensionMetadataService { return true; } // A crash between consumeQuarantineSidecar's mismatch claim and its - // reconcile strands an unreconciled foreign generation at the fixed - // claim path, which the sidecar probe above cannot see. Same access() - // cost profile: the reconcile only runs when a claim file actually - // exists (re-checked inside the queue — a concurrent consume's leftover - // pass may have taken it while this caller waited). - const claimPath = `${this.filePath}.corrupt.claimed`; - const claimExists = await access(claimPath).then( - () => true, - (error: unknown) => { - if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { - return false; - } - throw error; // Unknowable claim state stays a retryable failure. - } + // reconcile strands an unreconciled foreign generation at a unique + // claim name, which the sidecar probe above cannot see. Discover + // stranded claims by prefix (one readdir next to the full-file read the + // caller already does); an unreadable directory propagates so the read + // stays retryable rather than vouching for a possibly partial main. + const claimNames = (await readdir(dirname(this.filePath))).filter((name) => + name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.CLAIM_INFIX}`) ); - if (!claimExists) { + if (claimNames.length === 0) { return false; } await this.withSerializedMutation(async () => { - const stillExists = await access(claimPath).then( - () => true, - () => false - ); - if (stillExists) { - await this.reconcileRecreatedMainWithSidecar(claimPath); + for (const claimName of claimNames) { + const claimPath = join(dirname(this.filePath), claimName); + // Re-check inside the queue: a concurrent recovery's discovery (or + // the claim's own consume) may have taken the file while this + // caller waited. Only a verified ENOENT skips — an unprobeable + // claim propagates, same contract as the sidecar probe: reporting + // success would let a strict read accept and emit a possibly + // partial main while recoverable fields sit in the claim. + const stillExists = await access(claimPath).then( + () => true, + (error: unknown) => { + if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + return false; + } + throw error; + } + ); + if (stillExists) { + await this.reconcileRecreatedMainWithSidecar(claimPath); + } } }); return true; diff --git a/src/node/services/coreServices.ts b/src/node/services/coreServices.ts index 63045b49a7..270d57e0cc 100644 --- a/src/node/services/coreServices.ts +++ b/src/node/services/coreServices.ts @@ -115,7 +115,20 @@ export function createCoreServices(opts: CoreServicesOptions): CoreServices { if (!evidence.hasWorkspaceEntriesWithoutIds) { return false; } - // Alias ids: a second resolvable compatibility file's identity stays + // Targeted lenient positive first: a POSITIVE identity match needs no + // completeness, so a re-registered workspace whose own compatibility + // metadata is healthy must not stay write-suppressed because an + // UNRELATED legacy entry's metadata is malformed (the strict + // enumeration below throws on the first such entry, and the tombstone + // would then pin every one of the target's writes as transient + // indefinitely). A lenient scan only skips unreadable entries — it + // never fabricates a match. + if (config.findWorkspace(workspaceId) != null) { + return true; + } + // Negatives keep requiring the complete strict view: a lenient miss is + // indistinguishable from an identity hidden by a read failure. Alias + // ids: a second resolvable compatibility file's identity stays // registered for findWorkspace even though it is not any entry's // primary id — refusing its writes/deletions requires knowing it here. const legacyAliasIds = new Set(); From e9e261d596262b5845e5add11428f2f3d2036aa1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 10:03:34 +0000 Subject: [PATCH 60/72] fix: address round-54 Codex findings 1. getActivityList's final phase falls back to the authoritative enumeration when the post-probe raw config read fails transiently: an inline-id workspace deregistered during the late-candidate probes no longer rides the response just because the raw view was unreadable. The fallback removes only ids the scope enumeration vouched for; raw-only ids stay retained on transient failures. 2. Matched claims are renamed to a .corrupt-consumed- marker before the unlink, and discovery deletes those markers instead of replaying them: a replayed merge would re-fill fields another backend explicitly cleared to null (the null-fill merge is not idempotent across clears). 3. getAllWorkspaceMetadata keeps a healthy canonical legacy record when the basename-backed second candidate is unreadable in lenient loads (previously the outer catch discarded it for skeletal path-id fallback metadata, surfacing the workspace under the wrong identity); strict enumeration still fails closed. --- src/node/config.test.ts | 52 +++++++++++ src/node/config.ts | 11 +++ .../services/ExtensionMetadataService.test.ts | 27 ++++++ src/node/services/ExtensionMetadataService.ts | 48 +++++++++- src/node/services/workspaceService.test.ts | 90 ++++++++++++++++++- src/node/services/workspaceService.ts | 50 +++++++++-- 6 files changed, 266 insertions(+), 12 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index c98dc3db1f..cf9503f959 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -968,6 +968,58 @@ describe("Config", () => { const loaded = new Config(tempDir).loadConfigOrDefault({ throwOnError: true }); expect(loaded.projects.size).toBe(0); }); + + it("keeps the canonical legacy identity when a secondary alias file is unreadable in lenient loads", async () => { + // An id-less legacy entry with a HEALTHY canonical (generated-legacy) + // metadata file and an unreadable basename-backed second candidate: + // lenient loads must keep the canonical identity instead of + // discarding it for skeletal path-id fallback metadata (which would + // surface the workspace under the WRONG id with its session history + // apparently missing). Strict enumeration still fails closed — the + // unreadable alias may hide a registered identity. + const projectPath = "/repo"; + const workspacePath = "/repo/legacy-ws"; + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + // Migration flags pre-seeded so the first load never schedules the + // async settings-migration persist mid-test. + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }) + ); + const config = new Config(tempDir); + const canonicalId = ( + config as unknown as { + generateLegacyId(projectPath: string, workspacePath: string): string; + } + ).generateLegacyId(projectPath, workspacePath); + const canonicalDir = config.getSessionDir(canonicalId); + fs.mkdirSync(canonicalDir, { recursive: true }); + fs.writeFileSync( + path.join(canonicalDir, "metadata.json"), + JSON.stringify({ id: "stable-canonical-id", name: "legacy-ws" }) + ); + // Basename-backed second candidate is unreadable: a directory at the + // metadata.json path fails reads with EISDIR (non-ENOENT). + fs.mkdirSync(path.join(config.getSessionDir("legacy-ws"), "metadata.json"), { + recursive: true, + }); + + let strictRejected = false; + try { + await config.getAllWorkspaceMetadata({ throwOnError: true }); + } catch { + strictRejected = true; + } + expect(strictRejected).toBe(true); + + const lenient = await config.getAllWorkspaceMetadata(); + const lenientIds = lenient.map((metadata) => metadata.id); + expect(lenientIds).toContain("stable-canonical-id"); + expect(lenientIds).not.toContain(canonicalId); + }); }); describe("readPersistedWorkspaceIdSuperset", () => { diff --git a/src/node/config.ts b/src/node/config.ts index 4d9c0b4390..59fcefdf42 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -2749,6 +2749,17 @@ export class Config { // to the catch below instead of silently substituting the // generated legacy path id via the !metadataFound branch. if (!isEnoentError(readError)) { + // Secondary-candidate failure with a usable canonical + // record already in hand: lenient loads keep the canonical + // identity instead of discarding it for skeletal path-id + // fallback metadata (which would surface the workspace + // under the WRONG id with its session history apparently + // missing). Destructive strict enumeration still fails + // closed — the unreadable alias file may hide a registered + // identity. + if (legacyMetadataRaw !== undefined && !options?.throwOnError) { + continue; + } throw readError; } continue; diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 13187d7730..47dfadc736 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1194,6 +1194,33 @@ describe("ExtensionMetadataService", () => { expect((await readdir(tempDir)).filter((name) => name.includes(".corrupt-claim-"))).toEqual([]); }); + test("stranded consumed claims are deleted without replay", async () => { + // A crash between the consumed-rename and the unlink strands a marker + // whose bytes are PROVEN represented at the main path. Replaying the + // merge would re-fill fields another backend explicitly cleared to + // null since (the null-fill merge is not idempotent across clears) — + // and resurrect entries pruning already reclaimed. Discovery must + // delete the marker, never merge it. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt-consumed-123-stranded`, + JSON.stringify({ + version: 1, + workspaces: { "ws-reclaimed": { recency: 700, streaming: false } }, + }) + ); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.has("ws-reclaimed")).toBe(false); + expect(snapshots.get("ws-live")?.recency).toBe(900); + expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-consumed-"))).toEqual([]); + }); + test("a strict per-workspace read self-heals deterministic corruption", async () => { // Live emissions read per-workspace snapshots after the subscription // bootstraps. If the file becomes deterministically corrupt afterwards, diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 93da7a272f..9c84beacdf 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -85,6 +85,16 @@ export class ExtensionMetadataService { * other's live claims; stranded claims are discovered by this prefix. */ private static readonly CLAIM_INFIX = ".corrupt-claim-"; + + /** + * Name infix a claim is renamed to once its bytes are verified to match + * the generation the caller reconciled — i.e. the bytes are proven + * represented at the main path. Discovery DELETES these instead of + * replaying them: re-merging already-reconciled bytes would re-fill + * fields another backend explicitly cleared to null in the meantime + * (the merge's null-fill rule is not idempotent across clears). + */ + private static readonly CONSUMED_INFIX = ".corrupt-consumed-"; private mutationQueue: Promise = Promise.resolve(); /** * Per-process write tombstones for removed workspaces. Workspace removal @@ -416,8 +426,24 @@ export class ExtensionMetadataService { await this.reconcileRecreatedMainWithSidecar(claimPath); return; } + // Matched: the claim's bytes are proven represented at the main path. + // Mark them consumed BEFORE deleting so a crash between the two steps + // strands a file discovery deletes rather than replays (a replayed + // merge would re-fill fields another backend explicitly cleared to + // null after passing its own claim scan). The residual replay window + // shrinks to the claim-rename → consumed-rename gap above, part of the + // documented unlocked-shared-file boundary. + const consumedPath = `${this.filePath}${ExtensionMetadataService.CONSUMED_INFIX}${process.pid}-${randomUUID()}`; try { - await unlink(claimPath); + await rename(claimPath, consumedPath); + } catch (renameError) { + if (ExtensionMetadataService.isErrnoCode(renameError, "ENOENT")) { + return; // A concurrent discovery already took the claim. + } + throw renameError; + } + try { + await unlink(consumedPath); } catch (unlinkError) { if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { throw unlinkError; @@ -464,9 +490,27 @@ export class ExtensionMetadataService { // stranded claims by prefix (one readdir next to the full-file read the // caller already does); an unreadable directory propagates so the read // stays retryable rather than vouching for a possibly partial main. - const claimNames = (await readdir(dirname(this.filePath))).filter((name) => + const dirNames = await readdir(dirname(this.filePath)); + const claimNames = dirNames.filter((name) => name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.CLAIM_INFIX}`) ); + // Consumed markers hold bytes PROVEN represented at the main path (a + // crash landed between the consumed-rename and the unlink): delete, + // never replay — a re-merge would re-fill fields another backend + // explicitly cleared to null since. Cleanup failures are logged, not + // propagated: dead bytes cannot compromise a read, and the next pass + // retries the unlink. + for (const consumedName of dirNames.filter((name) => + name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.CONSUMED_INFIX}`) + )) { + try { + await unlink(join(dirname(this.filePath), consumedName)); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + log.debug("Failed to clean up consumed quarantine claim", { consumedName, unlinkError }); + } + } + } if (claimNames.length === 0) { return false; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 93fc9607d9..ec3cacae6d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9,7 +9,7 @@ import { createDisplayUsage } from "@/common/utils/tokens/displayUsage"; import { askUserQuestionManager } from "./askUserQuestionManager"; import { WorkspaceLifecycleHooks } from "./workspaceLifecycleHooks"; import { EventEmitter } from "events"; -import { existsSync } from "fs"; +import { existsSync, writeFileSync } from "fs"; import * as fsPromises from "fs/promises"; import { tmpdir } from "os"; import path from "path"; @@ -5810,6 +5810,94 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("final enumeration proves removal when the post-probe raw read fails", async () => { + // An inline-id workspace is deregistered by another backend while the + // late-candidate workflow probes await, and the post-probe raw reads + // fail transiently. Without the enumeration fallback every raw + // deregistration guard is disabled (finalConfigIds null) and the stale + // retained entry rides the authoritative response with no cross-process + // event to repair it. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const removedId = "inline-removed-final"; + const survivorId = "inline-survivor-final"; + const lateId = "late-registered-final"; + const projectPath = path.join(config.rootDir, "project"); + const configPath = path.join(config.rootDir, "config.json"); + const configFor = (ids: string[]): string => + JSON.stringify({ + projects: [ + [ + projectPath, + { workspaces: ids.map((id) => ({ id, path: path.join(projectPath, id) })) }, + ], + ], + // Migration flags pre-seeded so the first load never schedules the + // async settings-migration persist mid-test. + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }); + await fsPromises.writeFile(configPath, configFor([removedId, survivorId])); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(removedId, 77); + await extensionMetadata.updateRecency(survivorId, 55); + // Fresh snapshot re-read (call 2) doubles as the moment "another + // backend" registers a new workspace: its id enters the fresh raw + // view outside the initial scope, forcing the late-candidate probes + // and with them the final post-probe views this test exercises. + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile(configPath, configFor([removedId, survivorId, lateId])); + } + return realGetAllSnapshots(options); + } + ); + // Raw superset reads: #1 initial baseline, #2 prune enumeration + // union, #3 post-prune scope refresh. Call #4 is the post-probe read + // — the concurrent deregistration lands just before it and every + // later raw read fails transiently, so only the fallback enumeration + // can prove the removal. + const realSuperset = config.readPersistedWorkspaceIdSuperset.bind(config); + let supersetCalls = 0; + const supersetSpy = spyOn(config, "readPersistedWorkspaceIdSuperset").mockImplementation( + () => { + supersetCalls += 1; + if (supersetCalls >= 4) { + if (supersetCalls === 4) { + writeFileSync(configPath, configFor([survivorId, lateId])); + } + throw new Error("transient raw config read failure"); + } + return realSuperset(); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[survivorId]?.recency).toBe(55); + // The enumeration-proven removal is dropped despite the raw view + // being unreadable. + expect(activityList?.[removedId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + supersetSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { // getAllWorkspaceMetadata walks every workspace with per-workspace disk // probes; the latency-sensitive first bootstrap must not pay it twice. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2d5617df3e..19bf3fcc3a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13904,13 +13904,27 @@ export class WorkspaceService extends EventEmitter { const isRawInvisible = (workspaceId: string): boolean => !(initialConfigIds?.has(workspaceId) ?? false) && !(freshConfigIds?.has(workspaceId) ?? false); + let finalConfigIds: ReadonlySet | null = null; + try { + finalConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + } catch { + finalConfigIds = null; + } if ( Array.from(probedWorkflowRunIds.keys()).some(isRawInvisible) || // Retained entries are re-filtered with the final views below // (they were admitted before the probes awaited), and a // raw-invisible retained id's removal is provable only through // the same fresh enumeration. - Object.keys(activityById).some(isRawInvisible) + Object.keys(activityById).some(isRawInvisible) || + // A transiently unreadable post-probe raw view would otherwise + // disable the raw deregistration guards below entirely: an + // inline-id workspace deregistered during the probes (its + // metadata key not yet cleaned up) would ride the response with + // no cross-process event to repair it. The enumeration + // substitutes as removal evidence for ids the scope enumeration + // vouched for. + finalConfigIds == null ) { try { finalAuthoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); @@ -13920,14 +13934,15 @@ export class WorkspaceService extends EventEmitter { }); finalAuthoritativeIds = null; } - } - // Read last (synchronous), after every await above, so the raw - // deregistration guard sees the freshest possible view. - let finalConfigIds: ReadonlySet | null = null; - try { - finalConfigIds = this.config.readPersistedWorkspaceIdSuperset(); - } catch { - finalConfigIds = null; + // Re-read the raw view after the enumeration await so the raw + // deregistration guards see the freshest possible view; on a + // repeat failure keep the earlier successful read (still a + // valid post-probe view) rather than degrading to null. + try { + finalConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + } catch { + // Keep the pre-enumeration read (possibly null). + } } // The retained-entry filter above ran BEFORE the workflow probes, // so a removal landing during those awaits is invisible to every @@ -13959,6 +13974,15 @@ export class WorkspaceService extends EventEmitter { (isRawInvisible(workspaceId) && !(finalConfigIds?.has(workspaceId) ?? false) && finalAuthoritativeIds != null && + !finalAuthoritativeIds.has(workspaceId)) || + // Raw view unreadable post-probe: the fallback enumeration + // substitutes as removal evidence, but only for ids the scope + // enumeration itself vouched for — raw-only ids are + // enumeration-invisible by design and stay retained on + // transient read failures. + (finalConfigIds == null && + finalAuthoritativeIds != null && + (scopeEnumerationIds?.has(workspaceId) ?? false) && !finalAuthoritativeIds.has(workspaceId)) ) { delete activityById[workspaceId]; @@ -14013,6 +14037,14 @@ export class WorkspaceService extends EventEmitter { !(freshConfigIds?.has(workspaceId) ?? false) && !(finalConfigIds?.has(workspaceId) ?? false) && finalAuthoritativeIds != null && + !finalAuthoritativeIds.has(workspaceId)) || + // Raw view unreadable post-probe: same enumeration fallback + // as the retained-entry filter, scoped to ids the scope + // enumeration vouched for (raw-only ids stay admitted on + // transient read failures — see lateSnapshot above). + (finalConfigIds == null && + finalAuthoritativeIds != null && + (scopeEnumerationIds?.has(workspaceId) ?? false) && !finalAuthoritativeIds.has(workspaceId)) ) { continue; From 407d9e200907c0ccc7af17cb036531fa3477c2b7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 10:21:17 +0000 Subject: [PATCH 61/72] fix: address round-55 Codex findings 1. Claim names embed the verified generation token (ino-mtimeNs-size), so every crash point is replay-safe: discovery deletes a stranded claim whose file identity matches its embedded token (bytes proven merged before the consume began) and replays mismatched/token-less claims (foreign generations nobody merged). This subsumes the separate consumed-marker rename, closing the claim-to-consumed crash gap it left. 2. getActivityList re-reads snapshot evidence after the final fallback enumeration (and the raw view after that), so a raw-invisible legacy workspace admitted by the enumeration but removed before it finished is caught by the vanish guard instead of riding the response. --- .../services/ExtensionMetadataService.test.ts | 55 ++++++-- src/node/services/ExtensionMetadataService.ts | 118 +++++++++--------- src/node/services/workspaceService.ts | 16 ++- 3 files changed, 116 insertions(+), 73 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 47dfadc736..cc8395eaa8 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test"; -import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rename, rm, writeFile } from "fs/promises"; import { tmpdir } from "os"; import * as path from "path"; import { ExtensionMetadataService } from "./ExtensionMetadataService"; @@ -1194,13 +1194,14 @@ describe("ExtensionMetadataService", () => { expect((await readdir(tempDir)).filter((name) => name.includes(".corrupt-claim-"))).toEqual([]); }); - test("stranded consumed claims are deleted without replay", async () => { - // A crash between the consumed-rename and the unlink strands a marker - // whose bytes are PROVEN represented at the main path. Replaying the - // merge would re-fill fields another backend explicitly cleared to - // null since (the null-fill merge is not idempotent across clears) — - // and resurrect entries pruning already reclaimed. Discovery must - // delete the marker, never merge it. + test("stranded claims whose bytes match their embedded token are deleted without replay", async () => { + // A crash between a matched claim's rename and its unlink strands a + // claim whose file identity equals the token embedded in its name — + // proof its bytes were already merged into the main file before the + // consume began. Replaying the merge would re-fill fields another + // backend explicitly cleared to null since (the null-fill merge is not + // idempotent across clears) and resurrect entries pruning reclaimed. + // Discovery must delete it, never merge it. await writeFile( filePath, JSON.stringify({ @@ -1208,17 +1209,51 @@ describe("ExtensionMetadataService", () => { workspaces: { "ws-live": { recency: 900, streaming: false } }, }) ); + const strandedTmp = `${filePath}.stranded-tmp`; await writeFile( - `${filePath}.corrupt-consumed-123-stranded`, + strandedTmp, JSON.stringify({ version: 1, workspaces: { "ws-reclaimed": { recency: 700, streaming: false } }, }) ); + const token = await ( + ExtensionMetadataService as unknown as { + statQuarantineToken(p: string): Promise<{ ino: bigint; mtimeNs: bigint; size: bigint }>; + } + ).statQuarantineToken(strandedTmp); + await rename( + strandedTmp, + `${filePath}.corrupt-claim-${token.ino}-${token.mtimeNs}-${token.size}-123-stranded` + ); const snapshots = await service.getAllSnapshots({ throwOnError: true }); expect(snapshots.has("ws-reclaimed")).toBe(false); expect(snapshots.get("ws-live")?.recency).toBe(900); - expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-consumed-"))).toEqual([]); + expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-claim-"))).toEqual([]); + }); + + test("stranded claims that do not match their embedded token are replayed", async () => { + // Crash between a MISMATCH claim (foreign generation another backend + // installed at the sidecar path) and its reconcile: nobody merged + // those bytes, so discovery must replay them into the main file. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt-claim-99-99-99-123-foreign`, + JSON.stringify({ + version: 1, + workspaces: { "ws-foreign": { recency: 700, streaming: false } }, + }) + ); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-foreign")?.recency).toBe(700); + expect(snapshots.get("ws-live")?.recency).toBe(900); + expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-claim-"))).toEqual([]); }); test("a strict per-workspace read self-heals deterministic corruption", async () => { diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 9c84beacdf..bb518c65d4 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -80,21 +80,34 @@ export class ExtensionMetadataService { private readonly filePath: string; /** - * Name infix for per-consume claim files (`
-`). - * Unique names keep concurrent recoveries from replacing or clearing each - * other's live claims; stranded claims are discovered by this prefix. + * Name infix for per-consume claim files + * (`
----`). Unique names keep + * concurrent recoveries from replacing or clearing each other's live + * claims, and the embedded identity token makes every crash point + * replay-safe: the consumer only calls consume AFTER merging the + * generation identified by the token, so a stranded claim whose file + * identity MATCHES its embedded token is proven represented at the main + * path (rename preserves ino/mtime/size) and is deleted by discovery — + * replaying it would re-fill fields another backend explicitly cleared + * to null (the null-fill merge is not idempotent across clears). A + * mismatched or unparseable claim holds a foreign generation nobody + * merged and is replayed. */ private static readonly CLAIM_INFIX = ".corrupt-claim-"; - /** - * Name infix a claim is renamed to once its bytes are verified to match - * the generation the caller reconciled — i.e. the bytes are proven - * represented at the main path. Discovery DELETES these instead of - * replaying them: re-merging already-reconciled bytes would re-fill - * fields another backend explicitly cleared to null in the meantime - * (the merge's null-fill rule is not idempotent across clears). - */ - private static readonly CONSUMED_INFIX = ".corrupt-consumed-"; + /** Parse the identity token embedded in a claim file name; null when the + * name does not carry one (fail toward replay — the pre-token behavior). */ + private static parseClaimToken(claimName: string): QuarantineSidecarToken | null { + const match = /\.corrupt-claim-(\d+)-(\d+)-(\d+)-/.exec(claimName); + if (match == null) { + return null; + } + return { ino: BigInt(match[1]), mtimeNs: BigInt(match[2]), size: BigInt(match[3]) }; + } + + private static tokensMatch(a: QuarantineSidecarToken, b: QuarantineSidecarToken): boolean { + return a.ino === b.ino && a.mtimeNs === b.mtimeNs && a.size === b.size; + } private mutationQueue: Promise = Promise.resolve(); /** * Per-process write tombstones for removed workspaces. Workspace removal @@ -388,7 +401,11 @@ export class ExtensionMetadataService { // fail closed by leaving the file rather than unlinking blind. return; } - const claimPath = `${this.filePath}${ExtensionMetadataService.CLAIM_INFIX}${process.pid}-${randomUUID()}`; + // The claim name embeds the token of the generation the caller just + // merged (see CLAIM_INFIX): a crash at ANY later point leaves a claim + // discovery can classify by identity — matching bytes were merged + // (delete), anything else is a foreign generation (replay). + const claimPath = `${this.filePath}${ExtensionMetadataService.CLAIM_INFIX}${token.ino}-${token.mtimeNs}-${token.size}-${process.pid}-${randomUUID()}`; try { await rename(quarantinePath, claimPath); } catch (renameError) { @@ -409,11 +426,7 @@ export class ExtensionMetadataService { if (current == null) { return; // Claim vanished (concurrent discovery consumed it). } - if ( - current.ino !== token.ino || - current.mtimeNs !== token.mtimeNs || - current.size !== token.size - ) { + if (!ExtensionMetadataService.tokensMatch(current, token)) { // The claim took a NEWER generation another process installed after // consuming ours: merge it into the main file instead of destroying // it. The reconcile consumes the claim itself on success; a corrupt @@ -426,24 +439,12 @@ export class ExtensionMetadataService { await this.reconcileRecreatedMainWithSidecar(claimPath); return; } - // Matched: the claim's bytes are proven represented at the main path. - // Mark them consumed BEFORE deleting so a crash between the two steps - // strands a file discovery deletes rather than replays (a replayed - // merge would re-fill fields another backend explicitly cleared to - // null after passing its own claim scan). The residual replay window - // shrinks to the claim-rename → consumed-rename gap above, part of the - // documented unlocked-shared-file boundary. - const consumedPath = `${this.filePath}${ExtensionMetadataService.CONSUMED_INFIX}${process.pid}-${randomUUID()}`; + // Matched: the claim's bytes are proven represented at the main path, + // and its name says so (the embedded token matches the file), so a + // crash before this unlink strands a file discovery deletes rather + // than replays. try { - await rename(claimPath, consumedPath); - } catch (renameError) { - if (ExtensionMetadataService.isErrnoCode(renameError, "ENOENT")) { - return; // A concurrent discovery already took the claim. - } - throw renameError; - } - try { - await unlink(consumedPath); + await unlink(claimPath); } catch (unlinkError) { if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { throw unlinkError; @@ -494,47 +495,40 @@ export class ExtensionMetadataService { const claimNames = dirNames.filter((name) => name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.CLAIM_INFIX}`) ); - // Consumed markers hold bytes PROVEN represented at the main path (a - // crash landed between the consumed-rename and the unlink): delete, - // never replay — a re-merge would re-fill fields another backend - // explicitly cleared to null since. Cleanup failures are logged, not - // propagated: dead bytes cannot compromise a read, and the next pass - // retries the unlink. - for (const consumedName of dirNames.filter((name) => - name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.CONSUMED_INFIX}`) - )) { - try { - await unlink(join(dirname(this.filePath), consumedName)); - } catch (unlinkError) { - if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { - log.debug("Failed to clean up consumed quarantine claim", { consumedName, unlinkError }); - } - } - } if (claimNames.length === 0) { return false; } await this.withSerializedMutation(async () => { for (const claimName of claimNames) { const claimPath = join(dirname(this.filePath), claimName); - // Re-check inside the queue: a concurrent recovery's discovery (or + // Re-probe inside the queue: a concurrent recovery's discovery (or // the claim's own consume) may have taken the file while this // caller waited. Only a verified ENOENT skips — an unprobeable // claim propagates, same contract as the sidecar probe: reporting // success would let a strict read accept and emit a possibly // partial main while recoverable fields sit in the claim. - const stillExists = await access(claimPath).then( - () => true, - (error: unknown) => { - if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { - return false; + const current = await ExtensionMetadataService.statQuarantineToken(claimPath); + if (current == null) { + continue; + } + // Identity classification (see CLAIM_INFIX): a claim whose file + // matches its embedded token holds bytes its consumer had already + // merged before claiming — delete, never replay (a re-merge would + // re-fill fields another backend explicitly cleared to null + // since). Mismatched or token-less claims hold a foreign + // generation nobody merged: replay it. + const expected = ExtensionMetadataService.parseClaimToken(claimName); + if (expected != null && ExtensionMetadataService.tokensMatch(current, expected)) { + try { + await unlink(claimPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; } - throw error; } - ); - if (stillExists) { - await this.reconcileRecreatedMainWithSidecar(claimPath); + continue; } + await this.reconcileRecreatedMainWithSidecar(claimPath); } }); return true; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 19bf3fcc3a..2c973da0ed 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13934,7 +13934,21 @@ export class WorkspaceService extends EventEmitter { }); finalAuthoritativeIds = null; } - // Re-read the raw view after the enumeration await so the raw + // The enumeration is itself an await: a raw-invisible legacy + // workspace can be admitted by it and then removed (metadata + // key deleted) before it finishes — invisible to the snapshot + // view captured before it and to every raw view. Re-read the + // snapshot evidence after the enumeration so the vanish guard + // sees the removal; on failure keep the pre-enumeration view + // (still a post-probe view). + try { + finalSnapshots = await this.extensionMetadata.getAllSnapshots({ + throwOnError: true, + }); + } catch { + // Keep the pre-enumeration re-read (possibly null). + } + // Re-read the raw view after the awaits above so the raw // deregistration guards see the freshest possible view; on a // repeat failure keep the earlier successful read (still a // valid post-probe view) rather than degrading to null. From d5b016a62bd7806991e80df009e9710bdc211b6e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 10:38:48 +0000 Subject: [PATCH 62/72] fix: persist writes when a tombstone is lifted mid-probe (round 56) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recheckTombstonedRegistration evaluated the lifted-tombstone case only after a POSITIVE probe: a negative or failing probe returned early, so a write racing a concurrent activity bootstrap (which lifted the tombstone on its own registration evidence) produced an unpersisted transient snapshot that emitWorkspaceActivity still broadcast — renderer recency/goal/status advanced past disk until restart. The lifted-tombstone check now runs after every probe outcome; negative probes keep suppressing only while the tombstone still stands (emits stay suppressed by the same tombstone). --- .../services/ExtensionMetadataService.test.ts | 24 ++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 28 +++++++++++++------ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index cc8395eaa8..b5e7f1d715 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1540,6 +1540,30 @@ describe("ExtensionMetadataService", () => { expect(service.isWorkspaceDeleted("ws-legacy")).toBe(true); }); + test("a write persists when the tombstone is lifted mid-probe despite a negative probe", async () => { + // The registration probe and a concurrent activity bootstrap race: the + // bootstrap proves re-registration and lifts the tombstone while this + // write's own probe resolves negative (stale evidence) or fails. Once + // the tombstone is gone, broadcasts are un-suppressed — suppressing + // only this in-flight write would broadcast an unpersisted transient + // snapshot, leaving renderer state ahead of disk until restart. The + // write must persist. + await service.updateRecency("ws-revived", 100); + service.setRegistrationProbe(() => Promise.resolve(false)); + await service.deleteWorkspace("ws-revived"); + expect(service.isWorkspaceDeleted("ws-revived")).toBe(true); + service.setRegistrationProbe(() => { + // Concurrent bootstrap lifts the tombstone while the probe is in + // flight, on registration evidence of its own. + service.clearTombstonesForRegisteredIds(new Set(["ws-revived"]), service.getTombstonedIds()); + return Promise.resolve(false); // This probe's own (stale) negative outcome. + }); + const snapshot = await service.updateRecency("ws-revived", 500); + expect(snapshot.recency).toBe(500); + expect((await service.getAllSnapshots()).get("ws-revived")?.recency).toBe(500); + expect(service.isWorkspaceDeleted("ws-revived")).toBe(false); + }); + test("live snapshot reads reconcile a stranded sidecar too", async () => { // After the activity subscription bootstraps, live metadata/workflow // emissions read via getSnapshot — a healthy subscription never issues diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index bb518c65d4..8e09084450 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -265,23 +265,33 @@ export class ExtensionMetadataService { // queued deletion may run before the caller's queued write, which would // then recreate the removed entry). const generationBefore = this.deletedWorkspaceIds.get(workspaceId); + let probeRegistered = false; try { - if (!(await this.registrationProbe(workspaceId))) { - return false; - } + probeRegistered = await this.registrationProbe(workspaceId); } catch { - return false; + probeRegistered = false; } const generationAfter = this.deletedWorkspaceIds.get(workspaceId); if (generationAfter === undefined) { // Another path (activity bootstrap, reconcile) lifted the tombstone - // while the probe was in flight — the id is verifiably registered and - // no newer removal exists, so the write must PERSIST. Returning false - // here would hand the caller an unpersisted transient snapshot that - // WorkspaceService still broadcasts (the tombstone is gone), leaving - // renderer state ahead of disk until restart. + // while the probe was in flight, on registration evidence that + // postdates the tombstone. The write must PERSIST regardless of THIS + // probe's outcome (negative and failing probes included): once the + // tombstone is gone, broadcasts are un-suppressed and every later + // write persists normally — suppressing only this in-flight write + // would hand the caller an unpersisted transient snapshot that + // WorkspaceService still broadcasts, leaving renderer recency/goal/ + // status ahead of disk until restart. A genuinely newer same-process + // removal republishes a tombstone, which the caller's in-queue + // re-check still honors. return true; } + if (!probeRegistered) { + // Negative or failing probe with the tombstone still standing: keep + // suppressing. Emits stay suppressed by the same tombstone, so no + // transient state can reach the renderer. + return false; + } if (generationAfter !== generationBefore) { // Republished mid-probe: a newer same-process removal wins. return false; From c9b5d9478921c7cc2ce82c7e7be08412d558a481 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 10:59:32 +0000 Subject: [PATCH 63/72] fix: address round-57 Codex findings 1. readPersistedWorkspaceIdEvidence traverses only projects[*][1].workspaces direct entries: nested workspaces-keyed fields inside workspace entries (or newer-build extension objects) can no longer inject phantom registered ids that lift removed ids' tombstones. 2. Stranded-claim discovery runs even when the fixed sidecar was processed: an intentionally retained corrupt sidecar no longer starves recovery of an unreconciled foreign-generation claim. 3. reconcileRecreatedMainWithSidecar re-stats after reading the sidecar and aborts on token mismatch, so bytes swapped in between the stat and the read are never merged under the stale token (which would double-apply them through the consume-side mismatch replay). 4. getActivityList's cross-process removal guards evict process-local workflow/monitor caches (mid-list filter, final retained filter, late candidates): foreign removals publish no local tombstone, so the listener-driven eviction never fires and a re-registered deterministic legacy id would otherwise be served the removed incarnation's ghost runs. --- src/node/config.test.ts | 31 +++++++ src/node/config.ts | 66 ++++++-------- .../services/ExtensionMetadataService.test.ts | 83 ++++++++++++++++- src/node/services/ExtensionMetadataService.ts | 25 +++++- src/node/services/workspaceService.test.ts | 63 +++++++++++++ src/node/services/workspaceService.ts | 90 ++++++++++++------- 6 files changed, 284 insertions(+), 74 deletions(-) diff --git a/src/node/config.test.ts b/src/node/config.test.ts index cf9503f959..7753c6012c 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -1023,6 +1023,37 @@ describe("Config", () => { }); describe("readPersistedWorkspaceIdSuperset", () => { + it("ignores nested workspaces arrays inside workspace entries", () => { + // Workspace entries (and unknown newer-build extension objects) can + // carry nested `workspaces`-keyed fields whose ids reference OTHER — + // including removed — workspaces. Treating them as registered would + // lift removed ids' tombstones and recreate stale metadata + // indefinitely; only `projects[*][1].workspaces` direct entries are + // registration evidence. + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ + projects: [ + [ + "/repo", + { + workspaces: [ + { + id: "real-ws", + path: "/repo/ws", + workspaces: [{ id: "phantom-ws", path: "/x" }], + }, + ], + }, + ], + ], + }) + ); + const evidence = new Config(tempDir).readPersistedWorkspaceIdEvidence(); + expect([...evidence.ids]).toEqual(["real-ws"]); + expect(evidence.hasWorkspaceEntriesWithoutIds).toBe(false); + }); + it("collects ids from entries that lenient normalization discards", () => { // `[null, ...]` fails the project-path filter and vanishes from the // normalized view; the raw superset must still surface its workspace id diff --git a/src/node/config.ts b/src/node/config.ts index 59fcefdf42..5d2a408ea0 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -1176,49 +1176,24 @@ export class Config { const id = (value as { id?: unknown }).id; return typeof id === "string" && id.length > 0; }; - // Collect ids ONLY from direct entries of `workspaces` arrays — never - // from arbitrary nested objects. Workspace entries carry nested id-bearing - // objects (e.g. taskPendingGuidance items) whose ids can reference OTHER - // (including removed) workspaces; a whole-subtree scan would report those - // as registered, corrupting registration evidence (aborted deletions, - // lifted tombstones, ghost activity probes) and unbounding the activity + // Collect ids ONLY from the direct entries of `projects[*][1].workspaces` + // — never from arbitrary nested objects. Workspace entries carry nested + // id-bearing objects (e.g. taskPendingGuidance items) and can even carry + // nested `workspaces`-keyed fields written by other builds; ids found + // there can reference OTHER (including removed) workspaces, and treating + // them as registered corrupts registration evidence (aborted deletions, + // lifted tombstones, ghost activity probes) and unbounds the activity // scope. Under-collection stays safe: any workspaces container whose // entries cannot be verified (id-less entry, non-array container) flips // the incompleteness flag, routing callers to the strict enumeration. - const collect = (value: unknown): void => { - if (Array.isArray(value)) { - for (const entry of value) { - collect(entry); - } - return; - } - if (value !== null && typeof value === "object") { - const workspaces = (value as { workspaces?: unknown }).workspaces; - if (Array.isArray(workspaces)) { - for (const entry of workspaces) { - if (hasInlineStringId(entry)) { - ids.add((entry as { id: string }).id); - } else { - hasWorkspaceEntriesWithoutIds = true; - } - } - } else if (workspaces !== undefined) { - // A PRESENT workspaces container in any non-array shape (object, - // null, primitive) is uninterpretable evidence — the original - // entries may have been mangled, so the raw id set cannot be - // proven complete. Only an absent key means "no entries here". - hasWorkspaceEntriesWithoutIds = true; - } - for (const nested of Object.values(value)) { - collect(nested); - } - } - }; + // // The OUTER structure must be interpretable too, or the id set cannot be // proven complete: a present non-array `projects`, a non-pair element, a // non-object project config, or a project config with no workspaces key // at all may be a mangled remnant of real registrations. Only an absent // projects key is healthy emptiness (the strict loader accepts it). + // A malformed sibling project only flips the flag — ids from the + // remaining well-formed projects are still collected. const projects = (parsedValue as { projects?: unknown }).projects; if (projects !== undefined) { if (!Array.isArray(projects)) { @@ -1229,16 +1204,29 @@ export class Config { if ( projectConfig === null || typeof projectConfig !== "object" || - Array.isArray(projectConfig) || - (projectConfig as { workspaces?: unknown }).workspaces === undefined + Array.isArray(projectConfig) ) { hasWorkspaceEntriesWithoutIds = true; - break; + continue; + } + const workspaces = (projectConfig as { workspaces?: unknown }).workspaces; + if (!Array.isArray(workspaces)) { + // A missing key or a PRESENT container in any non-array shape is + // uninterpretable evidence — the original entries may have been + // mangled, so the raw id set cannot be proven complete. + hasWorkspaceEntriesWithoutIds = true; + continue; + } + for (const entry of workspaces) { + if (hasInlineStringId(entry)) { + ids.add((entry as { id: string }).id); + } else { + hasWorkspaceEntriesWithoutIds = true; + } } } } } - collect(projects); return { ids, hasWorkspaceEntriesWithoutIds }; } diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index b5e7f1d715..957d98f29f 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1152,12 +1152,13 @@ describe("ExtensionMetadataService", () => { statQuarantineToken: (quarantinePath: string) => Promise; }; const realStat = statics.statQuarantineToken.bind(ExtensionMetadataService); - // Call #1 captures the reconcile's generation token (real); call #2 is - // the consumption-time identity probe this test degrades. + // Call #1 captures the reconcile's generation token, call #2 binds the + // token to the read bytes (both real); call #3 is the consumption-time + // identity probe this test degrades. let statCalls = 0; statics.statQuarantineToken = async (quarantinePath: string) => { statCalls += 1; - if (statCalls === 2) { + if (statCalls === 3) { const error = new Error("EACCES: permission denied, stat") as NodeJS.ErrnoException; error.code = "EACCES"; throw error; @@ -1232,6 +1233,82 @@ describe("ExtensionMetadataService", () => { expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-claim-"))).toEqual([]); }); + test("a retained corrupt sidecar does not starve stranded claim recovery", async () => { + // A deterministically corrupt sidecar is intentionally retained by its + // reconcile. If the leftover pass returned early on it, a stranded + // mismatched claim (unreconciled foreign generation) would never be + // discovered — its recency/goal/status hidden indefinitely. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + await writeFile(`${filePath}.corrupt`, "{corrupt sidecar"); + await writeFile( + `${filePath}.corrupt-claim-99-99-99-123-foreign`, + JSON.stringify({ + version: 1, + workspaces: { "ws-claim": { recency: 700, streaming: false } }, + }) + ); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-claim")?.recency).toBe(700); + expect(snapshots.get("ws-live")?.recency).toBe(900); + // Corrupt sidecar intentionally retained (for inspection); claim consumed. + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toBe("{corrupt sidecar"); + expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-claim-"))).toEqual([]); + }); + + test("a sidecar swapped between stat and read is not merged under the stale token", async () => { + // Another recovery consumes the captured generation and installs a NEW + // one at the fixed path between this reconcile's stat and read. Merging + // the new bytes under the old token would double-apply them (the + // consume claims the new generation, sees the mismatch, and replays), + // re-filling fields another backend explicitly cleared to null in + // between. The post-read binding must abort instead. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + const statics = ExtensionMetadataService as unknown as { + statQuarantineToken(p: string): Promise; + }; + const realStat = statics.statQuarantineToken.bind(ExtensionMetadataService); + let statCalls = 0; + statics.statQuarantineToken = async (p: string) => { + statCalls += 1; + if (statCalls === 1) { + // The capture observed a generation that was then consumed by + // another recovery, which installed the CURRENT file before our + // read. + return { ino: 1n, mtimeNs: 2n, size: 3n }; + } + return realStat(p); + }; + try { + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + // Not merged under the stale token; left for its own recovery pass. + expect(snapshots.has("ws-stranded")).toBe(false); + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toContain("ws-stranded"); + } finally { + statics.statQuarantineToken = realStat; + } + const healed = await service.getAllSnapshots({ throwOnError: true }); + expect(healed.get("ws-stranded")?.recency).toBe(700); + }); + test("stranded claims that do not match their embedded token are replayed", async () => { // Crash between a MISMATCH claim (foreign generation another backend // installed at the sidecar path) and its reconcile: nobody merged diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 8e09084450..201f2b7f97 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -491,9 +491,10 @@ export class ExtensionMetadataService { * emissions). */ private async reconcileLeftoverSidecarIfPresent(): Promise { + let reconciledSidecar = false; if (await this.probeQuarantineSidecar()) { await this.resumeQuarantineRecovery(); - return true; + reconciledSidecar = true; } // A crash between consumeQuarantineSidecar's mismatch claim and its // reconcile strands an unreconciled foreign generation at a unique @@ -501,12 +502,16 @@ export class ExtensionMetadataService { // stranded claims by prefix (one readdir next to the full-file read the // caller already does); an unreadable directory propagates so the read // stays retryable rather than vouching for a possibly partial main. + // Runs even when the fixed sidecar was just processed: a + // deterministically corrupt sidecar is intentionally RETAINED by its + // reconcile, and returning early on it would starve stranded-claim + // recovery indefinitely (recency/goal/status hidden in the claim). const dirNames = await readdir(dirname(this.filePath)); const claimNames = dirNames.filter((name) => name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.CLAIM_INFIX}`) ); if (claimNames.length === 0) { - return false; + return reconciledSidecar; } await this.withSerializedMutation(async () => { for (const claimName of claimNames) { @@ -1513,6 +1518,22 @@ export class ExtensionMetadataService { let sidecarParsed: unknown; try { sidecarParsed = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; + // Bind the token to the bytes just read: another recovery can consume + // the captured generation and install a NEWER one at the fixed path + // between the stat and the read. Proceeding would merge the new bytes + // under the OLD token — the consume then claims the new generation, + // sees the token mismatch, and replays the same bytes a second time, + // re-filling fields another backend explicitly cleared to null in + // between (the null-fill merge is not idempotent across clears). + // On mismatch leave the file for its own recovery pass (next read). + const postReadToken = await ExtensionMetadataService.statQuarantineToken(quarantinePath); + if ( + sidecarToken == null || + postReadToken == null || + !ExtensionMetadataService.tokensMatch(sidecarToken, postReadToken) + ) { + return false; + } } catch (readError) { // Sidecar gone: a concurrent recovery consumed it and the recreated // main is all there is — nothing left to reconcile. diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ec3cacae6d..c96c9d5b8d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5898,6 +5898,69 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("foreign removals observed mid-list evict process-local activity caches", async () => { + // A cross-process removal publishes no local tombstone, so the + // tombstone-cleared eviction listener never fires. Without eviction at + // the removal guards, the removed incarnation's workflow caches survive + // — and a downgraded backend re-registering the same deterministic + // legacy id would then be served ghost runs instead of a fresh probe. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "foreign-removed-evict"; + const projectPath = path.join(config.rootDir, "project"); + const configPath = path.join(config.rootDir, "config.json"); + const configFor = (ids: string[]): string => + JSON.stringify({ + projects: [ + [ + projectPath, + { workspaces: ids.map((id) => ({ id, path: path.join(projectPath, id) })) }, + ], + ], + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }); + await fsPromises.writeFile(configPath, configFor([workspaceId])); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 42); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const first = await workspaceService.getActivityList(); + expect(first?.[workspaceId]?.recency).toBe(42); + const internals = workspaceService as unknown as { + activeWorkflowRunIdsByWorkspace: Map>; + }; + expect(internals.activeWorkflowRunIdsByWorkspace.has(workspaceId)).toBe(true); + // Another backend removes the workspace between the second list's + // initial and fresh raw reads (its metadata cleanup may lag). + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile(configPath, configFor([])); + } + return realGetAllSnapshots(options); + } + ); + try { + const second = await workspaceService.getActivityList(); + expect(second?.[workspaceId]).toBeUndefined(); + expect(internals.activeWorkflowRunIdsByWorkspace.has(workspaceId)).toBe(false); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { // getAllWorkspaceMetadata walks every workspace with per-workspace disk // probes; the latency-sensitive first bootstrap must not pay it twice. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2c973da0ed..b53f3dd0ad 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13778,30 +13778,45 @@ export class WorkspaceService extends EventEmitter { ); } const activityById = Object.fromEntries( - entries.filter( - (entry): entry is readonly [string, WorkspaceActivitySnapshot] => - entry != null && - // Revalidate after the per-workspace awaits above: a workspace - // removed while this list was computing would otherwise ride the - // delayed response past emitWorkspaceActivity's tombstone - // suppression — a renderer that already processed the removal - // event would re-insert the deleted id until the next reconnect. - !this.extensionMetadata.isWorkspaceDeleted(entry[0]) && + entries.filter((entry): entry is readonly [string, WorkspaceActivitySnapshot] => { + if (entry == null) { + return false; + } + const workspaceId = entry[0]; + // Revalidate after the per-workspace awaits above: a workspace + // removed while this list was computing would otherwise ride the + // delayed response past emitWorkspaceActivity's tombstone + // suppression — a renderer that already processed the removal + // event would re-insert the deleted id until the next reconnect. + if (this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { + return false; + } + const foreignRemoved = // Persisted snapshot vanished from the shared file mid-list // (metadata keys only disappear through removal). Entries that // never had a persisted snapshot are covered by the config check. - !( - freshPersistedIds != null && - snapshots.has(entry[0]) && - !freshPersistedIds.has(entry[0]) - ) && + (freshPersistedIds != null && + snapshots.has(workspaceId) && + !freshPersistedIds.has(workspaceId)) || // Deregistered from the shared config mid-list — also covers // workflow/bash-monitor-only entries with no persisted snapshot. - !isRemovedFromConfig(entry[0]) && + isRemovedFromConfig(workspaceId) || // Raw-invisible (legacy stable) ids: authoritative-lookup // counterpart of the raw-superset comparison above. - !isRemovedPerAuthoritativeIdentity(entry[0]) - ) + isRemovedPerAuthoritativeIdentity(workspaceId); + if (foreignRemoved) { + // A cross-process removal publishes no local tombstone, so the + // tombstone-cleared eviction listener never fires for it. Stale + // workflow-run/monitor caches would then survive the removal — + // and if a downgraded backend re-registers the same + // deterministic legacy id later, getActiveWorkflowRunIds would + // serve the REMOVED incarnation's cached runs as ghost activity + // instead of probing the recreated session. + this.evictWorkspaceActivityCaches(workspaceId); + return false; + } + return true; + }) ); // Addition-side counterpart of the fresh re-read: another backend can // register a workspace AND persist its first activity after this @@ -13968,8 +13983,7 @@ export class WorkspaceService extends EventEmitter { // have no snapshot and no live counts, so the probed candidates' // emptiness check must not run here. for (const workspaceId of Object.keys(activityById)) { - if ( - this.extensionMetadata.isWorkspaceDeleted(workspaceId) || + const foreignRemoved = // Persisted snapshot vanished during the probes (metadata // keys only disappear through removal) — also covers legacy // ids the raw views cannot see. @@ -13997,8 +14011,14 @@ export class WorkspaceService extends EventEmitter { (finalConfigIds == null && finalAuthoritativeIds != null && (scopeEnumerationIds?.has(workspaceId) ?? false) && - !finalAuthoritativeIds.has(workspaceId)) - ) { + !finalAuthoritativeIds.has(workspaceId)); + if (foreignRemoved) { + // Cross-process removals publish no local tombstone, so the + // tombstone-cleared eviction listener never fires — see the + // mid-list filter above. + this.evictWorkspaceActivityCaches(workspaceId); + delete activityById[workspaceId]; + } else if (this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { delete activityById[workspaceId]; } } @@ -14015,14 +14035,7 @@ export class WorkspaceService extends EventEmitter { : freshSnapshots != null ? (freshSnapshots.get(workspaceId) ?? null) : (snapshots.get(workspaceId) ?? null); - if ( - // Nothing to contribute: no persisted snapshot and no live - // counts (a workflow-only candidate legitimately has no - // snapshot, so its absence alone is not removal evidence). - (lateSnapshot == null && - activeWorkflowRunIds.size === 0 && - this.getActiveBashMonitorCount(workspaceId) === 0) || - this.extensionMetadata.isWorkspaceDeleted(workspaceId) || + const lateForeignRemoved = isRemovedFromConfig(workspaceId) || isRemovedPerAuthoritativeIdentity(workspaceId) || // Persisted snapshot vanished during the probes (metadata keys @@ -14059,7 +14072,24 @@ export class WorkspaceService extends EventEmitter { (finalConfigIds == null && finalAuthoritativeIds != null && (scopeEnumerationIds?.has(workspaceId) ?? false) && - !finalAuthoritativeIds.has(workspaceId)) + !finalAuthoritativeIds.has(workspaceId)); + if (lateForeignRemoved) { + // Same cross-process eviction rationale as the mid-list + // filter: no local tombstone means no listener-driven + // eviction, and the probes above may have installed caches + // for the removed incarnation. + this.evictWorkspaceActivityCaches(workspaceId); + continue; + } + if ( + // Nothing to contribute: no persisted snapshot and no live + // counts (a workflow-only candidate legitimately has no + // snapshot, so its absence alone is not removal evidence) — + // never removal proof, so no cache eviction. + (lateSnapshot == null && + activeWorkflowRunIds.size === 0 && + this.getActiveBashMonitorCount(workspaceId) === 0) || + this.extensionMetadata.isWorkspaceDeleted(workspaceId) ) { continue; } From 825b498cfcc239cb6bc0f055a857df50384320df Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 11:15:05 +0000 Subject: [PATCH 64/72] fix: address round-58 Codex findings 1. completeQuarantineRecovery binds the captured token to the bytes it read (same post-read re-stat as the recreated-main reconcile): a sidecar generation swapped between the stat and the read is never restored under the stale token, which would double-apply it through the consume-side mismatch replay. 2. deleteWorkspace reconciles a crash-stranded sidecar before its queued deletion, like the emitting write entry points: deleting from a recreated partial main would leave the removed workspace's complete entry in the sidecar for a concurrent backend (without this process's tombstone) to resurrect. --- .../services/ExtensionMetadataService.test.ts | 83 +++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 29 +++++++ 2 files changed, 112 insertions(+) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 957d98f29f..cfa5977a33 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1309,6 +1309,89 @@ describe("ExtensionMetadataService", () => { expect(healed.get("ws-stranded")?.recency).toBe(700); }); + test("deleteWorkspace reconciles a stranded sidecar so the removed entry cannot be resurrected", async () => { + // Full sidecar beside a recreated partial main: deleting from the + // partial main alone would leave the removed workspace's complete entry + // in the sidecar, where a concurrent backend without this + // process-local tombstone could reconcile it back onto disk (visible + // immediately on unscoped builds; inherited as stale goal/status by a + // deterministic legacy-id re-registration). + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 300, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-other": { recency: 300, streaming: false }, + "ws-removed": { recency: 700, streaming: false }, + }, + }) + ); + service.setRegistrationProbe(() => Promise.resolve(false)); + await service.deleteWorkspace("ws-removed"); + // Sidecar consumed (tombstoned entry skipped by the merge); the removed + // entry survives on NEITHER file. + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + expect("ws-removed" in persisted.workspaces).toBe(false); + expect("ws-other" in persisted.workspaces).toBe(true); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("a sidecar swapped during the missing-main restore is not restored under the stale token", async () => { + // Same stat-to-read race as the recreated-main reconcile, on the + // missing-main restore path (completeQuarantineRecovery): another + // recovery consumes the captured generation and installs a newer one + // before the read. Restoring under the stale token would double-apply + // the bytes through the consume-side mismatch replay. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + const statics = ExtensionMetadataService as unknown as { + statQuarantineToken(p: string): Promise; + }; + const realStat = statics.statQuarantineToken.bind(ExtensionMetadataService); + let statCalls = 0; + statics.statQuarantineToken = async (p: string) => { + statCalls += 1; + if (statCalls === 1) { + return { ino: 1n, mtimeNs: 2n, size: 3n }; + } + return realStat(p); + }; + try { + let strictRejected = false; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch { + strictRejected = true; + } + // Not restored under the stale token: the read stays retryable with + // the sidecar retained for the next pass. + expect(strictRejected).toBe(true); + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toContain("ws-stranded"); + } finally { + statics.statQuarantineToken = realStat; + } + const healed = await service.getAllSnapshots({ throwOnError: true }); + expect(healed.get("ws-stranded")?.recency).toBe(700); + }); + test("stranded claims that do not match their embedded token are replayed", async () => { // Crash between a MISMATCH claim (foreign generation another backend // installed at the sidecar path) and its reconcile: nobody merged diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 201f2b7f97..bc281b39fd 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -980,6 +980,18 @@ export class ExtensionMetadataService { // Synchronously, before the queued mutation: any writer enqueued from now // on must see the tombstone (see deletedWorkspaceIds). const publishedGeneration = this.publishTombstone(workspaceId); + // Reconcile a crash-stranded sidecar BEFORE the queued deletion (outside + // the queue — the resume path serializes itself), same as the emitting + // write entry points: deleting from a recreated PARTIAL main would skip + // the removed workspace's complete entry sitting in the sidecar, and a + // concurrent backend without this process-local tombstone could later + // reconcile that sidecar and restore the removed entry (visible + // immediately on unscoped builds; inherited as stale goal/status by a + // deterministic legacy-id re-registration). A failing reconcile + // propagates: the tombstone stays (fail closed, writes suppressed) and + // the disk entry remains recoverable for a retried removal or the + // process-start prune. + await this.reconcileLeftoverSidecarIfPresent(); await this.withSerializedMutation(async () => { const data = await this.load(); // In-queue registration revalidation, strictly AFTER the load (same @@ -1371,6 +1383,23 @@ export class ExtensionMetadataService { let movedParses = true; try { moved = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; + // Bind the token to the bytes just read (same contract as + // reconcileRecreatedMainWithSidecar): another recovery can consume + // the captured generation and install a newer one between the stat + // and the read. Restoring the new bytes under the OLD token would + // double-apply them — the consume claims the new generation, sees + // the mismatch, and replays it, re-filling fields another backend + // explicitly cleared to null in between. On mismatch leave the file + // for its own recovery pass (the retried read resumes with a fresh + // token). + const postReadToken = await ExtensionMetadataService.statQuarantineToken(quarantinePath); + if ( + sidecarToken == null || + postReadToken == null || + !ExtensionMetadataService.tokensMatch(sidecarToken, postReadToken) + ) { + return false; + } } catch (readError) { if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { throw readError; From 5989d56f0df64c80366d44aac94d9f3333fda613 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 11:37:36 +0000 Subject: [PATCH 65/72] fix: address round-59 Codex findings --- .../services/ExtensionMetadataService.test.ts | 41 ++++++- src/node/services/ExtensionMetadataService.ts | 100 ++++++++++++++---- src/node/services/workspaceService.test.ts | 80 ++++++++++++++ src/node/services/workspaceService.ts | 44 +++++--- 4 files changed, 229 insertions(+), 36 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index cfa5977a33..e8d135734d 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1097,7 +1097,7 @@ describe("ExtensionMetadataService", () => { ); await writeFile(filePath, "{corrupt json"); const internals = service as unknown as { - moveMainAsideAsRecreatedLeftover: () => Promise; + moveMainAsideAsRecreatedLeftover: () => Promise; }; const realMove = internals.moveMainAsideAsRecreatedLeftover.bind(service); internals.moveMainAsideAsRecreatedLeftover = async () => { @@ -1124,6 +1124,45 @@ describe("ExtensionMetadataService", () => { () => true ); expect(sidecarGone).toBe(true); + // The restored bytes were never superseded: the in-flight moved-aside + // file is dropped after the restore rather than finalized (or stranded) + // as a leftover. + const strayLeftovers = (await readdir(tempDir)).filter((name) => name.includes(".recreated")); + expect(strayLeftovers).toEqual([]); + }); + + test("recovery never touches another process's in-flight moved-aside main", async () => { + // Another backend is mid-recovery for the same sidecar: it moved a + // raced HEALTHY main aside and has not yet re-validated it. Under the + // old fixed-name protocol those bytes lived at `.recreated`, where THIS + // process's recovery would unlink them before the owner could restore + // them — both recoveries would then restore the OLDER sidecar, + // permanently losing the newer update. In-flight bytes now live under + // a unique per-invocation name that no other recovery may touch; only + // proven-superseded bytes are finalized to the shared fixed name. + const foreignInflight = `${filePath}.recreated-99999-f0e1d2c3`; + await writeFile( + foreignInflight, + JSON.stringify({ + version: 1, + workspaces: { "ws-newer": { recency: 900, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-old": { recency: 100, streaming: false } }, + }) + ); + await writeFile(filePath, "{corrupt json"); + const healed = await service.getAllSnapshots({ throwOnError: true }); + // This process's recovery completed: sidecar restored, corrupt main + // finalized as the bounded fixed-name leftover. + expect(healed.get("ws-old")?.recency).toBe(100); + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{corrupt json"); + // The foreign backend's in-flight bytes stay restorable by their owner. + expect(await readFile(foreignInflight, "utf-8")).toContain("ws-newer"); }); test("a failing sidecar token probe during consumption propagates instead of reporting success", async () => { diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index bc281b39fd..c002178256 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1278,7 +1278,7 @@ export class ExtensionMetadataService { // sidecar bytes reset it to empty exactly as before. A failing probe // propagates so the read stays retryable on unknowable evidence. if (await this.probeQuarantineSidecar()) { - await this.moveMainAsideAsRecreatedLeftover(); + const inflightLeftoverPath = await this.moveMainAsideAsRecreatedLeftover(); // Same cross-process race completeQuarantineRecovery closes for the // rename-to-.corrupt branch: another backend's atomic save can land // a NEWER healthy main between the in-queue corruption check above @@ -1287,13 +1287,14 @@ export class ExtensionMetadataService { // Re-validate what actually got moved; when it turns out healthy // (or a preserved newer schema), restore it to the vacant main path // and merge the existing sidecar into it via the recreated-main - // reconcile. Transient read failures propagate (retryable) — only + // reconcile. Transient read failures propagate (retryable) with the + // in-flight file left in place — finalizing UNVERIFIED bytes could + // bury a raced healthy save as the superseded leftover. Only // deterministic corruption proceeds to the sidecar-restore path. - const leftoverPath = `${this.filePath}.recreated`; let movedRaw: unknown; let movedParses = true; try { - movedRaw = JSON.parse(await readFile(leftoverPath, "utf-8")) as unknown; + movedRaw = JSON.parse(await readFile(inflightLeftoverPath, "utf-8")) as unknown; } catch (readError) { if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { throw readError; @@ -1308,24 +1309,49 @@ export class ExtensionMetadataService { // Restore without overwriting a file yet another writer re-created // at the main path (EEXIST): the reconcile below merges the // sidecar into whichever file now owns the path. + let restored = true; try { - await link(leftoverPath, this.filePath); + await link(inflightLeftoverPath, this.filePath); } catch (linkError) { - if (!ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + restored = false; + } else { try { - await copyFile(leftoverPath, this.filePath, constants.COPYFILE_EXCL); + await copyFile(inflightLeftoverPath, this.filePath, constants.COPYFILE_EXCL); } catch (copyError) { if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { // Main path missing with the raced bytes only in the - // leftover: rethrow (retryable) rather than letting the - // sidecar restore over the vacant path and orphan them. + // in-flight file: rethrow (retryable) rather than letting + // the sidecar restore over the vacant path and orphan them. throw copyError; } + restored = false; } } } + if (restored) { + // Restored to the main path: the in-flight file is a duplicate + // hard link of the live main, not a superseded leftover — drop + // it. Non-ENOENT failures propagate (retryable); the retried + // read finds the main healthy and at worst strands the + // harmless duplicate. + try { + await unlink(inflightLeftoverPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + } else { + // Yet another writer re-created the main path first: the moved + // bytes are superseded — keep them as the bounded leftover. + await this.finalizeRecreatedLeftover(inflightLeftoverPath); + } return this.reconcileRecreatedMainWithSidecar(quarantinePath); } + // Deterministically corrupt moved bytes: proven superseded — keep + // them as the bounded fixed-name leftover. + await this.finalizeRecreatedLeftover(inflightLeftoverPath); return this.completeQuarantineRecovery(quarantinePath); } await rename(this.filePath, quarantinePath); @@ -1334,18 +1360,42 @@ export class ExtensionMetadataService { } /** - * Move the current main file aside as the bounded fixed-name `.recreated` - * leftover ("keeps the latest superseded file"). A stale leftover from an - * earlier recovery is unlinked first: POSIX rename() replaces the - * destination silently, but Windows rename onto an existing file is not - * reliably a replace (it can fail with EPERM/EEXIST), which would fail - * every strict activity read's recovery until the user removed the + * Move the current main file aside to an in-flight uniquely named + * `.recreated--` path and return that path. In-flight + * moved-aside bytes must never live at the shared fixed-name `.recreated` + * leftover: the mutation queue is process-local, so two backends can both + * pass the corrupt-main validation for the same sidecar, and if process A + * had moved a concurrently saved HEALTHY main to the fixed name, process + * B's finalize would unlink A's only copy before A could re-validate and + * restore it — both recoveries would then restore the OLDER sidecar, + * permanently losing the newer update. Callers finalize via + * finalizeRecreatedLeftover once the moved bytes are proven superseded, + * or unlink the in-flight file once they are restored. A crash + * mid-recovery strands at most one uniquely named file per crash; no + * automated cleanup — liveness cannot be probed, and folding a live + * process's in-flight file into the fixed name would reintroduce the + * destroyed-restore race this unique naming exists to close. + */ + private async moveMainAsideAsRecreatedLeftover(): Promise { + const inflightPath = `${this.filePath}.recreated-${process.pid}-${randomUUID()}`; + await rename(this.filePath, inflightPath); + return inflightPath; + } + + /** + * Install proven-superseded moved-aside bytes as the bounded fixed-name + * `.recreated` leftover ("keeps the latest superseded file"). Replacing + * the fixed name is safe: with every in-flight move under a unique name + * (see moveMainAsideAsRecreatedLeftover), the fixed name only ever holds + * FINALIZED superseded bytes that no recovery will re-read. The prior + * leftover is unlinked first because Windows rename onto an existing file + * is not reliably a replace (it can fail with EPERM/EEXIST), which would + * fail every strict activity read's recovery until the user removed the * leftover by hand. A crash between the unlink and the rename only loses - * the STALE leftover; the main file stays in place and the recovery - * remains resumable. Non-ENOENT unlink failures propagate (retryable) — - * the rename would fail on the occupied destination anyway. + * the OLDER finalized leftover. Non-ENOENT unlink failures propagate + * (retryable) — the rename would fail on the occupied destination anyway. */ - private async moveMainAsideAsRecreatedLeftover(): Promise { + private async finalizeRecreatedLeftover(inflightPath: string): Promise { const leftoverPath = `${this.filePath}.recreated`; try { await unlink(leftoverPath); @@ -1354,7 +1404,7 @@ export class ExtensionMetadataService { throw unlinkError; } } - await rename(this.filePath, leftoverPath); + await rename(inflightPath, leftoverPath); } /** @@ -1591,10 +1641,14 @@ export class ExtensionMetadataService { // (upgrade↔downgrade preservation; this build's readers then see the // same non-destructive retryable unsupported-version signal as any // downgrade overlap) and preserve the recreated file as a bounded - // fixed-name leftover. A crash between the two steps leaves the + // fixed-name leftover. A crash between any of the steps leaves the // resumable missing-main + sidecar state, which restores the - // unsupported sidecar via completeQuarantineRecovery. - await this.moveMainAsideAsRecreatedLeftover(); + // unsupported sidecar via completeQuarantineRecovery. The moved bytes + // are superseded BY DECISION (the newer schema wins), so they finalize + // to the fixed leftover name immediately — no re-validation pass ever + // re-reads them. + const inflightLeftoverPath = await this.moveMainAsideAsRecreatedLeftover(); + await this.finalizeRecreatedLeftover(inflightLeftoverPath); try { await link(quarantinePath, this.filePath); } catch (linkError) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c96c9d5b8d..3a4d76c7bc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6004,6 +6004,86 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("first bootstrap scope keeps raw-registered ids the normalized view cannot see", async () => { + // A duplicate project path key (e.g. a trailing-slash variant) shadows + // the earlier pair in the normalized view — its workspace is registered + // and raw-visible (spared by the prune) yet absent from every strict + // enumeration. The first-bootstrap scope must come from the prune's + // FULL raw-plus-normalized union, not the enumeration alone: when the + // later raw refreshes fail transiently, an enumeration-only scope would + // serve an authoritative response omitting the live workspace, clearing + // its renderer activity state with no event to correct it. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const shadowedId = "raw-only-shadowed-ws"; + const winnerId = "normalized-winner-ws"; + const projectPath = path.join(config.rootDir, "project"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [ + // Map construction keeps the LAST duplicate key: the first pair + // (trailing-slash variant of the same path) is dropped from the + // normalized view with its workspace, while the raw id scan + // still collects it. + [ + `${projectPath}/`, + { workspaces: [{ id: shadowedId, path: path.join(projectPath, "shadowed") }] }, + ], + [ + projectPath, + { workspaces: [{ id: winnerId, path: path.join(projectPath, "winner") }] }, + ], + ], + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(shadowedId, 42); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Every raw config read AFTER the prune's successful one fails + // transiently (the finding's window: the id was already loaded, and + // only the discarded return kept it out of scope). + const internals = workspaceService as unknown as { + pruneStaleExtensionMetadataOnce(): Promise; + }; + const realPrune = internals.pruneStaleExtensionMetadataOnce.bind(workspaceService); + let failRawReads = false; + internals.pruneStaleExtensionMetadataOnce = async () => { + const prefetched = await realPrune(); + failRawReads = true; + return prefetched; + }; + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + if (failRawReads) { + throw new Error("transient config read failure"); + } + return realEvidence(); + } + ); + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[shadowedId]?.recency).toBe(42); + // Its snapshot was spared by the prune too, not merely re-admitted. + expect((await extensionMetadata.getAllSnapshots()).has(shadowedId)).toBe(true); + } finally { + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap admits snapshotless ids registered after the prune enumerated config", async () => { // A workspace another backend registers after the prune captured its id // set may have workflow- or bash-monitor-only activity and therefore no diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index b53f3dd0ad..c11e5910b5 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13369,13 +13369,26 @@ export class WorkspaceService extends EventEmitter { */ private prunedStaleExtensionMetadata = false; /** - * Returns the config-known (normalized-view) workspace ids captured during - * the prune so the first activity bootstrap can reuse them for scoping — + * Returns the config-known workspace ids captured during the prune so the + * first activity bootstrap can reuse them for scoping — * getAllWorkspaceMetadata walks every workspace with per-workspace disk * probes, which large deployments should not pay twice in the - * latency-sensitive bootstrap. Null when the prune was skipped or failed. + * latency-sensitive bootstrap. `knownIds` is the FULL raw-plus-normalized + * union the prune spared from deletion: scoping to anything narrower (the + * normalized view alone) would drop raw-registered ids the normalized + * view cannot see (e.g. a project pair shadowed by a duplicate path key), + * and if the bootstrap's later raw refreshes then failed transiently, the + * authoritative response would omit a live workspace whose id this READ + * already loaded successfully — clearing its renderer state with no event + * to correct it. `enumeratedIds` is what the strict enumeration itself + * vouched for (see scopeEnumerationIds: only those ids may treat + * enumeration absence as removal evidence). Null when the prune was + * skipped or failed. */ - private async pruneStaleExtensionMetadataOnce(): Promise | null> { + private async pruneStaleExtensionMetadataOnce(): Promise<{ + knownIds: Set; + enumeratedIds: ReadonlySet; + } | null> { if (this.prunedStaleExtensionMetadata) { return null; } @@ -13384,7 +13397,7 @@ export class WorkspaceService extends EventEmitter { // than on every read. this.prunedStaleExtensionMetadata = true; try { - let normalizedIds: Set | null = null; + let prunedScope: { knownIds: Set; enumeratedIds: ReadonlySet } | null = null; const prunedCount = await this.extensionMetadata.pruneMissingWorkspaces( async () => { // Invoked inside the file's serialized mutation AFTER the file load, @@ -13402,10 +13415,11 @@ export class WorkspaceService extends EventEmitter { // config migrations that are not yet persisted verbatim. // A missing config file resolves as a healthy empty set in both. const knownIds = this.config.readPersistedWorkspaceIdSuperset(); - normalizedIds = await this.enumerateAuthoritativeWorkspaceIds(); - for (const workspaceId of normalizedIds) { + const enumeratedIds = await this.enumerateAuthoritativeWorkspaceIds(); + for (const workspaceId of enumeratedIds) { knownIds.add(workspaceId); } + prunedScope = { knownIds, enumeratedIds }; return knownIds; }, async () => { @@ -13433,7 +13447,7 @@ export class WorkspaceService extends EventEmitter { if (prunedCount > 0) { log.info(`Pruned ${prunedCount} stale extension metadata entries`); } - return normalizedIds; + return prunedScope; } catch (error) { log.debug("Failed to prune stale extension metadata entries", { error }); return null; @@ -13456,7 +13470,7 @@ export class WorkspaceService extends EventEmitter { } catch { initialConfigIds = null; } - const prefetchedKnownIds = await this.pruneStaleExtensionMetadataOnce(); + const prefetchedScope = await this.pruneStaleExtensionMetadataOnce(); // throwOnError: the default load self-heals an unreadable/malformed // metadata file into an empty one, which this list would then present // as an authoritative "no activity anywhere" answer — the renderer @@ -13485,9 +13499,15 @@ export class WorkspaceService extends EventEmitter { // (invalid project path) are legitimately absent from every // enumeration and must not read that absence as removal. let scopeEnumerationIds: ReadonlySet | null = null; - if (prefetchedKnownIds != null) { - scopeEnumerationIds = new Set(prefetchedKnownIds); - workspaceIds = prefetchedKnownIds; + if (prefetchedScope != null) { + scopeEnumerationIds = prefetchedScope.enumeratedIds; + // Scope to the FULL known-id union the prune spared (raw + + // normalized), not the enumeration alone: a raw-registered id the + // normalized view cannot see was already loaded successfully by the + // prune's raw read, and relying solely on the refresh below to + // re-admit it would let a transient refresh failure turn into an + // authoritative response that omits the live workspace. + workspaceIds = prefetchedScope.knownIds; // The prune enumerated config BEFORE the snapshot read above, so a // workspace registered in between would be missing here — and an // authoritative list omitting it would clear its live-arrived From 08537d20cae6397a37a84d8a1d84427cd35eb9aa Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 13:17:13 +0000 Subject: [PATCH 66/72] fix: address round-60 Codex findings --- .../services/ExtensionMetadataService.test.ts | 67 ++++-- src/node/services/ExtensionMetadataService.ts | 192 +++++++++++++++++- src/node/services/workspaceService.test.ts | 72 ++++++- src/node/services/workspaceService.ts | 56 +++-- 4 files changed, 351 insertions(+), 36 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index e8d135734d..46bb755542 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1131,15 +1131,13 @@ describe("ExtensionMetadataService", () => { expect(strayLeftovers).toEqual([]); }); - test("recovery never touches another process's in-flight moved-aside main", async () => { - // Another backend is mid-recovery for the same sidecar: it moved a - // raced HEALTHY main aside and has not yet re-validated it. Under the - // old fixed-name protocol those bytes lived at `.recreated`, where THIS - // process's recovery would unlink them before the owner could restore - // them — both recoveries would then restore the OLDER sidecar, - // permanently losing the newer update. In-flight bytes now live under - // a unique per-invocation name that no other recovery may touch; only - // proven-superseded bytes are finalized to the shared fixed name. + test("a crash-stranded in-flight moved-aside main is recovered, not orphaned", async () => { + // Another backend crashed mid-recovery after moving a raced HEALTHY + // main aside under its unique in-flight name (the rename is the commit + // point; revalidation happens after it). Unique names are invisible to + // every fixed-path probe, so without the stranded-leftover scan the + // newer snapshot would be orphaned forever while the older sidecar + // restores. The scan must merge it back instead. const foreignInflight = `${filePath}.recreated-99999-f0e1d2c3`; await writeFile( foreignInflight, @@ -1157,12 +1155,55 @@ describe("ExtensionMetadataService", () => { ); await writeFile(filePath, "{corrupt json"); const healed = await service.getAllSnapshots({ throwOnError: true }); - // This process's recovery completed: sidecar restored, corrupt main - // finalized as the bounded fixed-name leftover. + // Sidecar restored AND the stranded newer snapshot merged on top. expect(healed.get("ws-old")?.recency).toBe(100); + expect(healed.get("ws-newer")?.recency).toBe(900); + // The corrupt main finalized as the bounded fixed-name leftover; the + // stranded unique file was consumed, not left to accumulate. expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{corrupt json"); - // The foreign backend's in-flight bytes stay restorable by their owner. - expect(await readFile(foreignInflight, "utf-8")).toContain("ws-newer"); + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("stranded leftover entries merge by strictly newer recency with streaming cleared", async () => { + // The stranded bytes are a complete main snapshot of unknowable age: + // per-entry recency orders the copies, a stale duplicate must never + // overwrite newer main data (this also makes crash replay a no-op), + // and a truthy streaming flag from a crashed process must not pin the + // workspace "streaming" forever. + await service.updateRecency("ws-current", 500); + await writeFile( + `${filePath}.recreated-4242-cafebabe`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-current": { recency: 300, streaming: false }, + "ws-imported": { recency: 900, streaming: true }, + }, + }) + ); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-current")?.recency).toBe(500); + expect(snapshots.get("ws-imported")?.recency).toBe(900); + expect(snapshots.get("ws-imported")?.streaming).toBe(false); + // Consumed cleanly: no stranded unique file and no garbage finalized to + // the fixed leftover name. + const leftovers = (await readdir(tempDir)).filter((name) => name.includes(".recreated")); + expect(leftovers).toEqual([]); + }); + + test("a corrupt stranded in-flight leftover is finalized, not merged", async () => { + // Corrupt stranded bytes are exactly the validated-corrupt main their + // owner moved aside: the scan keeps them as the bounded fixed-name + // leftover (the owner's own terminal state) instead of merging garbage + // or leaving unique files to accumulate. + await service.updateRecency("ws-live", 100); + await writeFile(`${filePath}.recreated-777-feedface`, "{not json"); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-live")?.recency).toBe(100); + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{not json"); + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); }); test("a failing sidecar token probe during consumption propagates instead of reporting success", async () => { diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index c002178256..7314f611e4 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -94,6 +94,13 @@ export class ExtensionMetadataService { * merged and is replayed. */ private static readonly CLAIM_INFIX = ".corrupt-claim-"; + /** + * Prefix (after the main file name) for uniquely named in-flight + * moved-aside mains AND the scan claims that consume them when stranded. + * The trailing dash keeps the bounded fixed-name `.recreated` leftover + * (finalized, proven-superseded bytes) out of the stranded-file scan. + */ + private static readonly RECREATED_INFIX = ".recreated-"; /** Parse the identity token embedded in a claim file name; null when the * name does not carry one (fail toward replay — the pre-token behavior). */ @@ -510,10 +517,24 @@ export class ExtensionMetadataService { const claimNames = dirNames.filter((name) => name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.CLAIM_INFIX}`) ); - if (claimNames.length === 0) { + // Crash-stranded in-flight moved-aside mains (see + // moveMainAsideAsRecreatedLeftover): their unique names make them + // invisible to every fixed-path probe, so without this scan an owner + // crash before revalidation would orphan a raced healthy main's only + // copy forever while the next recovery restores the OLDER sidecar. A + // live owner's file listed here is gone again by the time the queue + // slot below runs its claim rename (ENOENT — see + // recoverStrandedRecreatedLeftover for the steal semantics). + const strandedRecreatedNames = dirNames.filter((name) => + name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.RECREATED_INFIX}`) + ); + if (claimNames.length === 0 && strandedRecreatedNames.length === 0) { return reconciledSidecar; } await this.withSerializedMutation(async () => { + for (const strandedName of strandedRecreatedNames) { + await this.recoverStrandedRecreatedLeftover(join(dirname(this.filePath), strandedName)); + } for (const claimName of claimNames) { const claimPath = join(dirname(this.filePath), claimName); // Re-probe inside the queue: a concurrent recovery's discovery (or @@ -1370,14 +1391,15 @@ export class ExtensionMetadataService { * restore it — both recoveries would then restore the OLDER sidecar, * permanently losing the newer update. Callers finalize via * finalizeRecreatedLeftover once the moved bytes are proven superseded, - * or unlink the in-flight file once they are restored. A crash - * mid-recovery strands at most one uniquely named file per crash; no - * automated cleanup — liveness cannot be probed, and folding a live - * process's in-flight file into the fixed name would reintroduce the - * destroyed-restore race this unique naming exists to close. + * or unlink the in-flight file once they are restored. A crash (or a + * transient revalidation failure) can strand the file mid-recovery — + * possibly holding a raced healthy save's ONLY copy — so the leftover + * scan discovers stranded files by prefix and recovers them (see + * recoverStrandedRecreatedLeftover); the fixed name still never holds + * unverified bytes another recovery could destroy. */ private async moveMainAsideAsRecreatedLeftover(): Promise { - const inflightPath = `${this.filePath}.recreated-${process.pid}-${randomUUID()}`; + const inflightPath = `${this.filePath}${ExtensionMetadataService.RECREATED_INFIX}${process.pid}-${randomUUID()}`; await rename(this.filePath, inflightPath); return inflightPath; } @@ -1407,6 +1429,162 @@ export class ExtensionMetadataService { await rename(inflightPath, leftoverPath); } + /** + * Recover one stranded in-flight moved-aside main (unique RECREATED_INFIX + * name). The owner recovery's rename is its commit point — revalidation + * happens AFTER the move — so an owner crash (or transient revalidation + * failure) can strand a raced healthy save's ONLY copy at the unique + * name, where no fixed-path probe ever finds it; the next recovery would + * then restore the OLDER sidecar and orphan the newer + * recency/goal/status forever. Claiming may steal a LIVE owner's + * in-flight file: that is deliberate and data-preserving — the owner's + * revalidation read fails with a retryable ENOENT (never reported as + * success) while the bytes are merged or finalized here, so no copy is + * destroyed unmerged and no process-liveness probing is needed. Runs + * inside withSerializedMutation. + */ + private async recoverStrandedRecreatedLeftover(strandedPath: string): Promise { + // Claim to a fresh unique name under the same scanned prefix first: + // concurrent scanners race on the rename (ENOENT = already + // claimed/consumed) and a crash after the claim leaves the claim itself + // rediscoverable by the same prefix scan. No identity token in the name + // (unlike CLAIM_INFIX, whose claims hold VALID bytes whose + // already-merged vs foreign-generation identity is undecidable by + // parsing): here parsing decides everything — corrupt bytes are proven + // superseded, and valid bytes merge idempotently under the + // strictly-newer recency gate below, so replaying a claim after a crash + // between merge and unlink is a no-op rather than a resurrection. + const claimPath = `${this.filePath}${ExtensionMetadataService.RECREATED_INFIX}claim-${process.pid}-${randomUUID()}`; + try { + await rename(strandedPath, claimPath); + } catch (renameError) { + if (ExtensionMetadataService.isErrnoCode(renameError, "ENOENT")) { + return; + } + throw renameError; + } + let parsed: unknown; + try { + parsed = JSON.parse(await readFile(claimPath, "utf-8")) as unknown; + } catch (readError) { + // The claim path is process-unique, so any read failure other than + // deterministic parse corruption is transient: propagate (retryable; + // the claim stays discoverable). Corrupt bytes are exactly the + // validated-corrupt main the owner moved aside — keep them as the + // bounded fixed-name leftover, the same terminal state the owner + // itself would have chosen. + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + await this.finalizeRecreatedLeftover(claimPath); + return; + } + if (ExtensionMetadataService.isUnsupportedVersion(parsed)) { + // A newer build's main stranded mid-recovery: same upgrade- + // preservation swap as an unsupported sidecar (the reconcile treats + // the claim path as its sidecar argument and consumes it). + await this.reconcileRecreatedMainWithSidecar(claimPath); + return; + } + if (!ExtensionMetadataService.isValidMetadataFileShape(parsed)) { + // Parseable but not a metadata file: proven superseded garbage. + await this.finalizeRecreatedLeftover(claimPath); + return; + } + // Same tombstone revalidation contract as the sidecar reconcile: + // resolve probes BEFORE reading the main file (probes await config, and + // holding a pre-probe main snapshot across those awaits would let the + // save below clobber a concurrent backend's newer write). Without a + // probe the local removal knowledge stands; a FAILING probe propagates + // (retryable — the claim stays discoverable) rather than consuming the + // stranded bytes on unknowable evidence. + for (const workspaceId of Object.keys(parsed.workspaces)) { + if (!this.deletedWorkspaceIds.has(workspaceId) || this.registrationProbe == null) { + continue; + } + const generationBefore = this.deletedWorkspaceIds.get(workspaceId); + const registered = await this.registrationProbe(workspaceId); + if (registered && this.deletedWorkspaceIds.get(workspaceId) === generationBefore) { + this.liftTombstone(workspaceId); + } + } + let main: ExtensionMetadataFile; + try { + const mainParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; + if (!ExtensionMetadataService.isValidMetadataFileShape(mainParsed)) { + // Corrupt or newer-schema main: leave the claim for a later pass + // (it stays under the scanned prefix) rather than merging across + // schemas. + return; + } + main = mainParsed; + } catch (readError) { + if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { + // Missing-main window: the resumable sidecar recovery owns the + // path right now; the claim stays discoverable for the next pass. + return; + } + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + return; + } + let modified = false; + for (const [workspaceId, entry] of Object.entries(parsed.workspaces)) { + if (this.deletedWorkspaceIds.has(workspaceId)) { + // Still tombstoned after the revalidation pass above: the local + // removal knowledge stands. + continue; + } + if (entry === null || typeof entry !== "object" || Array.isArray(entry)) { + continue; + } + const candidateRecency = (entry as { recency?: unknown }).recency; + if (typeof candidateRecency !== "number") { + // Unorderable candidate: keep the main entry (fail closed). + continue; + } + const target: unknown = main.workspaces[workspaceId]; + const targetRecency = + target !== null && typeof target === "object" && !Array.isArray(target) + ? (target as { recency?: unknown }).recency + : undefined; + // Entry-level newest-wins, NOT the sidecar reconcile's main-wins + // field merge: the stranded bytes are a complete healthy main + // snapshot whose age relative to the CURRENT main is unknowable per + // field (the current main may be an older sidecar restore plus newer + // writes). Every mutation flows through mutateWorkspaceSnapshot with + // a fresh recency, so per-entry recency totally orders the two + // copies; requiring STRICTLY newer makes crash replay a no-op and + // can never re-fill a field a same-or-newer write cleared to null. + if (typeof targetRecency === "number" && targetRecency >= candidateRecency) { + continue; + } + // Cross-process crash leftover rule (same as sidecar-only entries): a + // truthy streaming flag from a stranded file must not pin the + // workspace "streaming" forever; a genuinely streaming workspace + // re-asserts the flag with its next write. + main.workspaces[workspaceId] = + (entry as { streaming?: unknown }).streaming === true + ? { ...entry, streaming: false } + : entry; + modified = true; + } + if (modified) { + await this.save(main); + } + // The claim path is process-unique: nothing else consumes it, so a + // plain unlink suffices (no token verification needed — see the claim + // comment above for why replay is idempotent anyway). + try { + await unlink(claimPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + } + /** * Finish a quarantine whose main file was already moved to the sidecar: * restore the sidecar to the main path when its bytes turn out healthy, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 3a4d76c7bc..3783ab1681 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6084,6 +6084,60 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("a mid-list corruption reset does not read as workspace removal", async () => { + // getAllSnapshots self-heals a deterministically corrupt metadata file + // into a valid (possibly EMPTY) one, so a quarantine landing between the + // initial and fresh reads makes every earlier snapshot key vanish from a + // SUCCESSFUL re-read while the config still registers the workspaces. + // Treating that disappearance as foreign-removal evidence would evict + // the workflow caches and omit live workspaces from an authoritative + // response — with no cross-process event to repair the renderer. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "corruption-reset-survivor"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 123); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const realGetAll = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotReads = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + (options) => { + snapshotReads += 1; + if (snapshotReads === 1) { + return realGetAll(options); + } + // Every re-read after the initial one models the post-quarantine + // self-healed EMPTY file: a successful, authoritative-looking + // read with every previous key gone. + return Promise.resolve(new Map()); + } + ); + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.recency).toBe(123); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap admits snapshotless ids registered after the prune enumerated config", async () => { // A workspace another backend registers after the prune captured its id // set may have workflow- or bash-monitor-only activity and therefore no @@ -7159,8 +7213,10 @@ describe("WorkspaceService activity list scoping", () => { probeCalls += 1; if (probeCalls === 2) { // The late candidate's probe is awaited: another backend removes - // the RETAINED workspace, deleting its persisted metadata entry - // unseen by this process's tombstones. + // the RETAINED workspace — config deregistration first (the real + // removeUnlocked order), then the metadata entry deletion unseen + // by this process's tombstones. + await config.removeWorkspace(retainedId); await new ExtensionMetadataService(metadataPath).deleteWorkspace(retainedId); } return []; @@ -7350,7 +7406,10 @@ describe("WorkspaceService activity list scoping", () => { spyOn(extensionMetadata, "getAllSnapshots").mockImplementationOnce(async () => { const snapshots = await readSnapshots(); // Simulates a concurrent removal completing after this request read - // its snapshot view but before the response was assembled. + // its snapshot view but before the response was assembled — in the + // real removeUnlocked order: config deregistration first, then the + // metadata deletion. + await config.removeWorkspace(workspaceId); await extensionMetadata.deleteWorkspace(workspaceId); return snapshots; }); @@ -7391,7 +7450,12 @@ describe("WorkspaceService activity list scoping", () => { const snapshots = await readSnapshots(options); // Simulates another backend's removal landing after this request read // its snapshot view: rewrite the shared file without the entry, with - // no in-process deleteWorkspace tombstone. + // no in-process deleteWorkspace tombstone. Faithful to the removal + // protocol's write order (removeUnlocked deregisters config BEFORE + // deleting metadata): a vanished snapshot with config still + // registering the id is a corruption-reset lookalike and must be + // retained, so removal simulations must deregister first. + await config.removeWorkspace(workspaceId); await fsPromises.writeFile( metadataPath, JSON.stringify({ version: 1, workspaces: {} }), diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c11e5910b5..6b10b0cf31 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13812,12 +13812,28 @@ export class WorkspaceService extends EventEmitter { return false; } const foreignRemoved = - // Persisted snapshot vanished from the shared file mid-list - // (metadata keys only disappear through removal). Entries that - // never had a persisted snapshot are covered by the config check. + // Persisted snapshot vanished from the shared file mid-list. + // Metadata keys normally only disappear through removal, but + // NOT always: a deterministically corrupt file self-heals into + // a valid (possibly EMPTY) one on the strict re-read, so a + // mid-list quarantine makes every earlier key vanish while the + // workspaces stay registered. Disappearance therefore counts as + // removal evidence only when a fresh config-derived view + // CAPABLE of seeing the id corroborates the deregistration: + // raw-visible ids need the fresh raw view to deny them, + // raw-invisible (legacy) ids the authoritative enumeration. + // With neither view available the entry is retained — keeping a + // stale entry briefly is recoverable, wiping live renderer + // state on a corruption reset is not. Entries that never had a + // persisted snapshot are covered by the config check. (freshPersistedIds != null && snapshots.has(workspaceId) && - !freshPersistedIds.has(workspaceId)) || + !freshPersistedIds.has(workspaceId) && + !(authoritativeIds?.has(workspaceId) ?? false) && + ((initialConfigIds?.has(workspaceId) ?? false) || + (freshConfigIds?.has(workspaceId) ?? false) + ? freshConfigIds != null && !freshConfigIds.has(workspaceId) + : authoritativeIds != null)) || // Deregistered from the shared config mid-list — also covers // workflow/bash-monitor-only entries with no persisted snapshot. isRemovedFromConfig(workspaceId) || @@ -14004,12 +14020,21 @@ export class WorkspaceService extends EventEmitter { // emptiness check must not run here. for (const workspaceId of Object.keys(activityById)) { const foreignRemoved = - // Persisted snapshot vanished during the probes (metadata - // keys only disappear through removal) — also covers legacy - // ids the raw views cannot see. + // Persisted snapshot vanished during the probes — also covers + // legacy ids the raw views cannot see. Corroborated like the + // mid-list filter's vanish arm: a strict re-read self-heals + // deterministic corruption into a valid (possibly EMPTY) + // file, so disappearance alone is not removal evidence — a + // post-probe config-derived view capable of seeing the id + // must also deny it. (finalSnapshots != null && (snapshots.has(workspaceId) || (freshSnapshots?.has(workspaceId) ?? false)) && - !finalSnapshots.has(workspaceId)) || + !finalSnapshots.has(workspaceId) && + !(finalConfigIds?.has(workspaceId) ?? false) && + !(finalAuthoritativeIds?.has(workspaceId) ?? false) && + (isRawInvisible(workspaceId) + ? finalAuthoritativeIds != null + : finalConfigIds != null)) || // Verifiably deregistered from the raw config during the // probes: visible in an earlier raw view, gone from the // post-probe one. @@ -14058,12 +14083,19 @@ export class WorkspaceService extends EventEmitter { const lateForeignRemoved = isRemovedFromConfig(workspaceId) || isRemovedPerAuthoritativeIdentity(workspaceId) || - // Persisted snapshot vanished during the probes (metadata keys - // only disappear through removal) — also covers legacy ids the - // raw views cannot see. + // Persisted snapshot vanished during the probes — also covers + // legacy ids the raw views cannot see. Same corruption-reset + // corroboration as the retained-entry vanish arms: a + // self-healed (possibly empty) re-read is not removal + // evidence on its own. (finalSnapshots != null && (freshSnapshots?.has(workspaceId) ?? false) && - !finalSnapshots.has(workspaceId)) || + !finalSnapshots.has(workspaceId) && + !(finalConfigIds?.has(workspaceId) ?? false) && + !(finalAuthoritativeIds?.has(workspaceId) ?? false) && + (isRawInvisible(workspaceId) + ? finalAuthoritativeIds != null + : finalConfigIds != null)) || // Verifiably deregistered from the raw config during the // probes: the id was visible in an EARLIER raw view — the // initial baseline or the fresh re-read that admitted it (a From c06fa7a1471bf76fc2443299f18987463cc24380 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 13:44:44 +0000 Subject: [PATCH 67/72] fix: address round-61 Codex findings --- .../services/ExtensionMetadataService.test.ts | 82 ++++++++++ src/node/services/ExtensionMetadataService.ts | 147 +++++++++++++++--- src/node/services/workspaceService.test.ts | 18 ++- src/node/services/workspaceService.ts | 20 ++- src/node/utils/extensionMetadata.ts | 12 ++ 5 files changed, 249 insertions(+), 30 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 46bb755542..a1553761b6 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1192,6 +1192,88 @@ describe("ExtensionMetadataService", () => { expect(leftovers).toEqual([]); }); + test("stranded leftover merge orders by write generation, not recency alone", async () => { + // recency is a user-interaction timestamp that status/goal writers + // deliberately preserve: a newer status write captured in the stranded + // copy can share its recency with the older restored main and must + // still win via the per-entry write generation — and the reverse copy + // (older generation, equal recency) must lose. + const entry = (writeGeneration: number, message: string) => ({ + recency: 500, + streaming: false, + writeGeneration, + lastModel: null, + lastThinkingLevel: null, + agentStatus: null, + lastStatusUrl: null, + todoStatus: { emoji: "s", message }, + }); + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "ws-status": entry(3, "old status"), + "ws-keep": entry(5, "current status"), + }, + }) + ); + await writeFile( + `${filePath}.recreated-31337-0ddba11`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-status": entry(7, "newer status"), + "ws-keep": entry(2, "stale status"), + }, + }) + ); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-status")?.todoStatus?.message).toBe("newer status"); + expect(snapshots.get("ws-keep")?.todoStatus?.message).toBe("current status"); + }); + + test("persisted mutations advance the per-entry write generation", async () => { + // The durable ordering contract behind the merge above: metadata + // mutations advance the generation even when they preserve recency, so + // cross-copy ordering never regresses to the recency tiebreak for + // entries written by this build. + await service.updateRecency("ws-gen", 100); + await service.setTodoStatus("ws-gen", { emoji: "s", message: "working" }, true); + const raw = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + expect(raw.workspaces["ws-gen"].writeGeneration).toBe(2); + expect(raw.workspaces["ws-gen"].recency).toBe(100); + }); + + test("stranded entries for foreign-removed workspaces are not resurrected", async () => { + // The stranded snapshot predates the current main: an entry missing + // from the main may have been REMOVED by another backend after the + // file was stranded (local tombstones cannot know). Adoption requires + // registration evidence — a probed-unregistered id is dropped instead + // of resurrected, while registered ids still recover. + service.setRegistrationProbe((workspaceId) => Promise.resolve(workspaceId !== "ws-removed")); + await service.updateRecency("ws-live", 100); + await writeFile( + `${filePath}.recreated-1234-abcd12`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-removed": { recency: 900, streaming: false }, + "ws-created": { recency: 800, streaming: false }, + }, + }) + ); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.has("ws-removed")).toBe(false); + expect(snapshots.get("ws-created")?.recency).toBe(800); + expect(snapshots.get("ws-live")?.recency).toBe(100); + // Consumed on evidence, not left for an endless retry loop. + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + test("a corrupt stranded in-flight leftover is finalized, not merged", async () => { // Corrupt stranded bytes are exactly the validated-corrupt main their // owner moved aside: the scan keeps them as the bounded fixed-name diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 7314f611e4..abdc924529 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -143,6 +143,23 @@ export class ExtensionMetadataService { return generation; } + /** + * In-memory suppression for a CROSS-PROCESS removal proven by the + * activity list's config-corroborated guards. The foreign backend's + * removal cannot publish a tombstone in this process, so without one a + * late local producer (workflow-run or bash-monitor completion) would + * repopulate the just-evicted caches and emitWorkspaceActivity — whose + * isWorkspaceDeleted check only knows local removals — would broadcast + * the removed incarnation's activity right after an authoritative list + * dropped it. Reuses the standard tombstone lifecycle, so suppression + * ends exactly on fresh registration evidence: writes re-probe via + * recheckTombstonedRegistration and later lists clear it via + * clearTombstonesForRegisteredIds when the id is genuinely re-registered. + */ + suppressForeignRemoval(workspaceId: string): void { + this.publishTombstone(workspaceId); + } + /** * Single exit point for clearing a tombstone (re-registration revival). * Notifies the owner so process-local activity caches bootstrapped for the @@ -341,11 +358,22 @@ export class ExtensionMetadataService { const data = await this.load(); const workspace = this.getOrCreateWorkspaceEntry(data, workspaceId, recency); mutate(workspace); + // Every persisted mutation advances the write generation: `recency` + // is a user-interaction timestamp (status/goal/streaming writers + // deliberately preserve it), so recovery merges cannot order metadata + // copies by recency alone — see recoverStrandedRecreatedLeftover. + workspace.writeGeneration = ExtensionMetadataService.nextWriteGeneration(workspace); await this.save(data); return toWorkspaceActivitySnapshot(workspace); }); } + private static nextWriteGeneration(entry: ExtensionMetadata): number { + return typeof entry.writeGeneration === "number" && Number.isFinite(entry.writeGeneration) + ? entry.writeGeneration + 1 + : 1; + } + constructor(filePath?: string) { this.filePath = filePath ?? getXumExtensionMetadataPath(); } @@ -838,6 +866,9 @@ export class ExtensionMetadataService { delete workspace.todoStatus; delete workspace.sidebarStatusInputHash; } + // Same write-generation contract as mutateWorkspaceSnapshot: this + // writer persists a mutation while deliberately preserving recency. + workspace.writeGeneration = ExtensionMetadataService.nextWriteGeneration(workspace); data.workspaces[workspaceId] = workspace; await this.save(data); return toWorkspaceActivitySnapshot(workspace); @@ -1491,37 +1522,79 @@ export class ExtensionMetadataService { await this.finalizeRecreatedLeftover(claimPath); return; } - // Same tombstone revalidation contract as the sidecar reconcile: - // resolve probes BEFORE reading the main file (probes await config, and - // holding a pre-probe main snapshot across those awaits would let the - // save below clobber a concurrent backend's newer write). Without a - // probe the local removal knowledge stands; a FAILING probe propagates - // (retryable — the claim stays discoverable) rather than consuming the + // Preliminary main read, only to bound the probe set below: which + // candidate ids are MISSING from the current main. Never saved. + let preliminary: ExtensionMetadataFile; + try { + const preliminaryParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; + if (!ExtensionMetadataService.isValidMetadataFileShape(preliminaryParsed)) { + // Corrupt or newer-schema main: leave the claim for a later pass + // (it stays under the scanned prefix) rather than merging across + // schemas. + return; + } + preliminary = preliminaryParsed; + } catch (readError) { + if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { + // Missing-main window: the resumable sidecar recovery owns the + // path right now; the claim stays discoverable for the next pass. + return; + } + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + return; + } + // Registration evidence, resolved BEFORE the mergeable main read below + // (probes await config, and holding a pre-probe main snapshot across + // those awaits would let the save clobber a concurrent backend's newer + // write). Two decisions share the probe: + // - tombstoned ids follow the sidecar reconcile's lift-or-suppress + // generation contract; + // - ids MISSING from the main are adopted only with registration + // evidence: the stranded snapshot predates the current main, so a + // missing entry may mean another backend REMOVED the workspace after + // the file was stranded — the local tombstones cannot know — and + // restoring it would resurrect deleted metadata indefinitely (an + // unscoped older build exposes it, and a re-registered deterministic + // legacy id would inherit the stale goal/status). + // Without a wired probe, missing-target adoption stays fail-open (the + // sidecar reconcile's sidecar-only adoption contract; dropping is + // destructive and production always wires the probe). A FAILING probe + // propagates (claim retained, retryable) rather than consuming the // stranded bytes on unknowable evidence. + const registrationEvidence = new Map(); for (const workspaceId of Object.keys(parsed.workspaces)) { - if (!this.deletedWorkspaceIds.has(workspaceId) || this.registrationProbe == null) { + const tombstoned = this.deletedWorkspaceIds.has(workspaceId); + if (this.registrationProbe == null) { + registrationEvidence.set(workspaceId, !tombstoned); + continue; + } + if (!tombstoned && workspaceId in preliminary.workspaces) { + // No decision needs evidence: the target exists and no local + // tombstone stands. continue; } const generationBefore = this.deletedWorkspaceIds.get(workspaceId); const registered = await this.registrationProbe(workspaceId); - if (registered && this.deletedWorkspaceIds.get(workspaceId) === generationBefore) { + if ( + tombstoned && + registered && + this.deletedWorkspaceIds.get(workspaceId) === generationBefore + ) { this.liftTombstone(workspaceId); } + registrationEvidence.set(workspaceId, registered); } let main: ExtensionMetadataFile; try { const mainParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; if (!ExtensionMetadataService.isValidMetadataFileShape(mainParsed)) { - // Corrupt or newer-schema main: leave the claim for a later pass - // (it stays under the scanned prefix) rather than merging across - // schemas. return; } main = mainParsed; } catch (readError) { if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { - // Missing-main window: the resumable sidecar recovery owns the - // path right now; the claim stays discoverable for the next pass. return; } if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { @@ -1544,20 +1617,50 @@ export class ExtensionMetadataService { // Unorderable candidate: keep the main entry (fail closed). continue; } + if (!(workspaceId in main.workspaces)) { + const registered = registrationEvidence.get(workspaceId); + if (registered === undefined) { + // The entry vanished between the preliminary read and this one + // (deleted during the probes), so no evidence was gathered for + // it. Abort the pass without saving or consuming: the claim + // stays discoverable and the next scan probes the id. Earlier + // candidates' merges replay idempotently. + return; + } + if (!registered) { + // Proven removed: do not resurrect the deleted entry. + continue; + } + } const target: unknown = main.workspaces[workspaceId]; - const targetRecency = - target !== null && typeof target === "object" && !Array.isArray(target) - ? (target as { recency?: unknown }).recency - : undefined; + const targetIsObject = + target !== null && typeof target === "object" && !Array.isArray(target); + const targetRecency = targetIsObject ? (target as { recency?: unknown }).recency : undefined; + const targetGeneration = targetIsObject + ? (target as { writeGeneration?: unknown }).writeGeneration + : undefined; + const candidateGeneration = (entry as { writeGeneration?: unknown }).writeGeneration; // Entry-level newest-wins, NOT the sidecar reconcile's main-wins // field merge: the stranded bytes are a complete healthy main // snapshot whose age relative to the CURRENT main is unknowable per // field (the current main may be an older sidecar restore plus newer - // writes). Every mutation flows through mutateWorkspaceSnapshot with - // a fresh recency, so per-entry recency totally orders the two - // copies; requiring STRICTLY newer makes crash replay a no-op and - // can never re-fill a field a same-or-newer write cleared to null. - if (typeof targetRecency === "number" && targetRecency >= candidateRecency) { + // writes). Ordering uses the per-entry writeGeneration (advanced by + // EVERY persisted mutation) when both copies carry distinct ones — + // recency alone cannot order metadata changes because status/goal/ + // streaming writers deliberately preserve it, so an equal-recency + // newer status must not be dropped. When generations are missing + // (bytes written by a build without them) or equal (concurrent + // bumps from the same base), the recency comparison is the best + // remaining order. Both gates require STRICTLY newer, so crash + // replay is a no-op and can never re-fill a field a same-or-newer + // write cleared to null. + const candidateNewer = + typeof candidateGeneration === "number" && + typeof targetGeneration === "number" && + candidateGeneration !== targetGeneration + ? candidateGeneration > targetGeneration + : !(typeof targetRecency === "number" && targetRecency >= candidateRecency); + if (!candidateNewer) { continue; } // Cross-process crash leftover rule (same as sidecar-only entries): a diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 3783ab1681..90cb4eaa1f 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -7472,8 +7472,17 @@ describe("WorkspaceService activity list scoping", () => { const activityList = await workspaceService.getActivityList(); expect(activityList).not.toBeNull(); expect(activityList?.[workspaceId]).toBeUndefined(); - // The in-process tombstone was NOT the mechanism here. - expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + // The removal was detected from foreign evidence — and retained as a + // local tombstone: cache eviction alone cannot stop a LATE local + // producer (workflow-run/bash-monitor completion) from re-emitting + // the removed incarnation's activity right after this authoritative + // response dropped it, because emitWorkspaceActivity's + // isWorkspaceDeleted check only knows local removals. + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(true); + // A late producer's write stays unpersisted (transient) instead of + // recreating the removed entry on disk. + await extensionMetadata.updateRecency(workspaceId, 200); + expect((await extensionMetadata.getAllSnapshots()).has(workspaceId)).toBe(false); } finally { await cleanup(); } @@ -7516,7 +7525,10 @@ describe("WorkspaceService activity list scoping", () => { const activityList = await workspaceService.getActivityList(); expect(activityList).not.toBeNull(); expect(activityList?.[workspaceId]).toBeUndefined(); - expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + // Foreign removals proven by the list guards publish a local + // tombstone (late-producer suppression — see the metadata-removal + // test above). + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(true); } finally { await cleanup(); } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6b10b0cf31..e93bb1683f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13849,6 +13849,13 @@ export class WorkspaceService extends EventEmitter { // serve the REMOVED incarnation's cached runs as ghost activity // instead of probing the recreated session. this.evictWorkspaceActivityCaches(workspaceId); + // Eviction alone cannot stop a LATE local producer (workflow-run + // or bash-monitor completion) from repopulating the caches and + // re-emitting the removed incarnation's activity right after + // this authoritative response dropped it — publish a local + // tombstone so emits and writes stay suppressed until fresh + // config evidence proves a revival. + this.extensionMetadata.suppressForeignRemoval(workspaceId); return false; } return true; @@ -14060,8 +14067,10 @@ export class WorkspaceService extends EventEmitter { if (foreignRemoved) { // Cross-process removals publish no local tombstone, so the // tombstone-cleared eviction listener never fires — see the - // mid-list filter above. + // mid-list filter above (including the late-producer + // suppression rationale). this.evictWorkspaceActivityCaches(workspaceId); + this.extensionMetadata.suppressForeignRemoval(workspaceId); delete activityById[workspaceId]; } else if (this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { delete activityById[workspaceId]; @@ -14126,11 +14135,12 @@ export class WorkspaceService extends EventEmitter { (scopeEnumerationIds?.has(workspaceId) ?? false) && !finalAuthoritativeIds.has(workspaceId)); if (lateForeignRemoved) { - // Same cross-process eviction rationale as the mid-list - // filter: no local tombstone means no listener-driven - // eviction, and the probes above may have installed caches - // for the removed incarnation. + // Same cross-process eviction + late-producer suppression + // rationale as the mid-list filter: no local tombstone means + // no listener-driven eviction, and the probes above may have + // installed caches for the removed incarnation. this.evictWorkspaceActivityCaches(workspaceId); + this.extensionMetadata.suppressForeignRemoval(workspaceId); continue; } if ( diff --git a/src/node/utils/extensionMetadata.ts b/src/node/utils/extensionMetadata.ts index 50f77655c3..0249c8faeb 100644 --- a/src/node/utils/extensionMetadata.ts +++ b/src/node/utils/extensionMetadata.ts @@ -35,6 +35,15 @@ export interface ExtensionMetadata { // a restart so unchanged chats are not regenerated. Never exposed on // WorkspaceActivitySnapshot (IPC shape). sidebarStatusInputHash?: string | null; + // Backend-only monotonic write counter, advanced by every persisted + // mutation of this entry. Recovery merges cannot order metadata copies by + // `recency` — that is a USER-INTERACTION timestamp which status/goal/ + // streaming writers deliberately preserve — so cross-copy ordering uses + // this generation instead (see recoverStrandedRecreatedLeftover). Never + // exposed on WorkspaceActivitySnapshot (IPC shape). Builds without this + // field drop it on their writes; ordering then degrades to the recency + // tiebreak, never resurrecting against a generation-less newer write. + writeGeneration?: number; goal?: GoalSnapshot | null; } @@ -129,6 +138,9 @@ export function coerceExtensionMetadata(value: unknown): ExtensionMetadata | nul ...(typeof record.sidebarStatusInputHash === "string" ? { sidebarStatusInputHash: record.sidebarStatusInputHash } : {}), + ...(typeof record.writeGeneration === "number" && Number.isFinite(record.writeGeneration) + ? { writeGeneration: record.writeGeneration } + : {}), ...(goal !== undefined ? { goal } : {}), }; } From 9c4e943c6e24ff73747ef3c31d0600101bd2a257 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 14:08:00 +0000 Subject: [PATCH 68/72] fix: address round-62 Codex findings --- .../services/ExtensionMetadataService.test.ts | 81 +++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 53 ++++++++++-- src/node/services/workspaceService.test.ts | 65 +++++++++++++++ src/node/services/workspaceService.ts | 19 ++++- 4 files changed, 206 insertions(+), 12 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index a1553761b6..ee83dca3b9 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1192,6 +1192,87 @@ describe("ExtensionMetadataService", () => { expect(leftovers).toEqual([]); }); + test("a raced healthy main losing the restore collision is merged, not parked", async () => { + // EEXIST during the restore does not prove the main-path owner is + // newer: a competing recovery can restore the OLDER sidecar to the + // vacant main path first and consume the sidecar. The valid moved + // bytes must stay at their unique in-flight name so the + // stranded-leftover scan merges them — finalizing them to the fixed + // (unscanned) leftover would silently drop the newer update. + const sidecarPath = `${filePath}.corrupt`; + const olderRestored = JSON.stringify({ + version: 1, + workspaces: { "ws-old": { recency: 100, streaming: false } }, + }); + await writeFile(sidecarPath, olderRestored); + await writeFile(filePath, "{corrupt json"); + const internals = service as unknown as { + moveMainAsideAsRecreatedLeftover: () => Promise; + }; + const realMove = internals.moveMainAsideAsRecreatedLeftover.bind(service); + internals.moveMainAsideAsRecreatedLeftover = async () => { + // The raced newer save lands before the move-aside… + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 900, streaming: false } }, + }) + ); + internals.moveMainAsideAsRecreatedLeftover = realMove; + const inflightPath = await realMove(); + // …and the competing recovery restores the older sidecar to the + // vacant main path and consumes it before this recovery's restore. + await writeFile(filePath, olderRestored); + await rm(sidecarPath); + return inflightPath; + }; + try { + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-old")?.recency).toBe(100); + expect(snapshots.get("ws-new")?.recency).toBe(900); + } finally { + internals.moveMainAsideAsRecreatedLeftover = realMove; + } + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("an unsupported sidecar never displaces a same-or-newer unsupported canonical file", async () => { + // Multi-version overlap: a v3 writer re-created the canonical file + // while an older v2 sidecar remained. A v1 build cannot order or merge + // foreign schemas, so it must keep the canonical copy in place and + // retain the sidecar for a build that understands both — not park v3 + // at the unscanned fixed leftover and restore older v2 data over it. + const v3Main = JSON.stringify({ version: 3, workspaces: {}, futureField: true }); + const v2Sidecar = JSON.stringify({ version: 2, workspaces: {} }); + await writeFile(filePath, v3Main); + await writeFile(`${filePath}.corrupt`, v2Sidecar); + await service.getSnapshot("any", { throwOnError: true }).catch(() => null); + expect(await readFile(filePath, "utf-8")).toBe(v3Main); + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toBe(v2Sidecar); + const leftovers = (await readdir(tempDir)).filter((name) => name.includes(".recreated")); + expect(leftovers).toEqual([]); + }); + + test("a strictly newer unsupported sidecar still replaces an older unsupported canonical file", async () => { + // Guard for the inverse direction: the version comparison must not + // block the legitimate upgrade-preservation swap when the sidecar + // schema is strictly newer than the canonical one. + const v2Main = JSON.stringify({ version: 2, workspaces: {} }); + const v3Sidecar = JSON.stringify({ version: 3, workspaces: {} }); + await writeFile(filePath, v2Main); + await writeFile(`${filePath}.corrupt`, v3Sidecar); + await service.getSnapshot("any", { throwOnError: true }).catch(() => null); + expect(await readFile(filePath, "utf-8")).toBe(v3Sidecar); + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe(v2Main); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + test("stranded leftover merge orders by write generation, not recency alone", async () => { // recency is a user-interaction timestamp that status/goal writers // deliberately preserve: a newer status write captured in the stranded diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index abdc924529..46fc9eff3d 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1394,11 +1394,16 @@ export class ExtensionMetadataService { throw unlinkError; } } - } else { - // Yet another writer re-created the main path first: the moved - // bytes are superseded — keep them as the bounded leftover. - await this.finalizeRecreatedLeftover(inflightLeftoverPath); } + // EEXIST (restored === false) does NOT prove the main-path owner + // is newer than the moved bytes: a competing recovery can restore + // the OLDER sidecar to the vacant main path first (and consume + // the sidecar, making the reconcile below a no-op). Finalizing + // the valid moved bytes to the fixed leftover would exclude them + // from stranded-file discovery and silently drop the newer + // recency/goal/status — leave them at the unique in-flight name + // instead: the stranded-leftover scan merges them by write + // generation/recency against whatever now owns the main path. return this.reconcileRecreatedMainWithSidecar(quarantinePath); } // Deterministically corrupt moved bytes: proven superseded — keep @@ -1924,10 +1929,42 @@ export class ExtensionMetadataService { // downgrade overlap) and preserve the recreated file as a bounded // fixed-name leftover. A crash between any of the steps leaves the // resumable missing-main + sidecar state, which restores the - // unsupported sidecar via completeQuarantineRecovery. The moved bytes - // are superseded BY DECISION (the newer schema wins), so they finalize - // to the fixed leftover name immediately — no re-validation pass ever - // re-reads them. + // unsupported sidecar via completeQuarantineRecovery. + // + // Inspect the canonical bytes FIRST: during multi-version overlap the + // canonical path may itself hold an unsupported file whose version is + // same-or-newer than the sidecar's (e.g. a v3 writer recreated it + // while a v2 sidecar remained). This build cannot order or merge + // foreign schemas beyond their version numbers, so swapping blindly + // would park the possibly-newer canonical copy at the unscanned + // fixed leftover and restore older data over it. Keep the canonical + // file in place and RETAIN the sidecar for a build that understands + // both schemas (same retention precedent as a deterministically + // corrupt sidecar: one no-op reconcile per read while the overlap + // lasts). A transiently unreadable canonical propagates (retryable); + // a missing or corrupt canonical proceeds with the swap exactly as + // before. + let canonicalParsed: unknown = null; + try { + canonicalParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; + } catch (canonicalReadError) { + if ( + !ExtensionMetadataService.isErrnoCode(canonicalReadError, "ENOENT") && + !ExtensionMetadataService.isDeterministicCorruption(canonicalReadError) + ) { + throw canonicalReadError; + } + } + if ( + ExtensionMetadataService.isUnsupportedVersion(canonicalParsed) && + (canonicalParsed as { version: number }).version >= + (sidecarParsed as { version: number }).version + ) { + return false; + } + // The moved bytes are superseded BY DECISION (the newer schema wins), + // so they finalize to the fixed leftover name immediately — no + // re-validation pass ever re-reads them. const inflightLeftoverPath = await this.moveMainAsideAsRecreatedLeftover(); await this.finalizeRecreatedLeftover(inflightLeftoverPath); try { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 90cb4eaa1f..a96590363c 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -7309,6 +7309,71 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("drops retained legacy entries removed during the mid-list identity scan", async () => { + // An id-less legacy workspace is retained on the strength of the + // mid-list authoritative enumeration — which can observe the stable id + // right before another backend deregisters it and deletes its metadata + // later in the same await. Raw config scans can never see the stable + // id and the fresh snapshot re-read predates the removal, so with zero + // late candidates nothing else re-reads: the final revalidation must + // run for retained raw-invisible ids too, or the deleted workspace + // rides every authoritative response until reconnect. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "legacy-retained-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + // Persisted snapshot: the entry is RETAINED by the per-id loop, so + // the late-candidate merge has nothing to probe. + await extensionMetadata.updateRecency(stableId, 321); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const internals = workspaceService as unknown as { + enumerateAuthoritativeWorkspaceIds(): Promise>; + }; + const realEnumerate = internals.enumerateAuthoritativeWorkspaceIds.bind(workspaceService); + let enumerateCalls = 0; + internals.enumerateAuthoritativeWorkspaceIds = async () => { + enumerateCalls += 1; + const ids = await realEnumerate(); + if (enumerateCalls === 2) { + // The removal lands INSIDE the mid-list enumeration await, after + // the enumeration observed the id: config deregistration first + // (the real removal write order), then the metadata deletion by + // another backend (no local tombstone). + await fsPromises.writeFile(configPath, JSON.stringify({ projects: [] })); + await new ExtensionMetadataService(metadataPath).deleteWorkspace(stableId); + } + return ids; + }; + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[stableId]).toBeUndefined(); + } finally { + await cleanup(); + } + }); + test("getActivityList quarantines a deterministically corrupt metadata file", async () => { // Parse/structure corruption fails identically on every retry, so a // strict read that only rethrows would leave activity hydration broken diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e93bb1683f..05b18a215e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13934,7 +13934,21 @@ export class WorkspaceService extends EventEmitter { } probedWorkflowRunIds.set(workspaceId, await this.getActiveWorkflowRunIds(workspaceId)); } - if (probedWorkflowRunIds.size > 0) { + const isRawInvisible = (workspaceId: string): boolean => + !(initialConfigIds?.has(workspaceId) ?? false) && + !(freshConfigIds?.has(workspaceId) ?? false); + // The final revalidation must ALSO run when a retained entry is + // raw-invisible even with zero late candidates: the mid-list + // authoritative enumeration can observe a legacy stable id right + // before another backend deregisters it and deletes its metadata + // later in the same await. Every view the retained filter used + // (pre-enumeration freshSnapshots, raw scans blind to stable ids, + // the stale enumeration itself) then still shows the workspace, and + // with no candidates to probe nothing else would re-read — the + // deleted workspace would ride every authoritative response + // indefinitely because cross-process removals emit no local event. + // Modern deployments (all ids raw-visible) never pay this path. + if (probedWorkflowRunIds.size > 0 || Object.keys(activityById).some(isRawInvisible)) { // Final post-probe views: the probes awaited disk, so a removal // landing during them is invisible to every view captured above — // inserting on those alone would ride the deleted id back into the @@ -13959,9 +13973,6 @@ export class WorkspaceService extends EventEmitter { // set is the view that ADMITTED it, so only a fresh enumeration // can prove the removal. Modern deployments never pay this walk. let finalAuthoritativeIds: ReadonlySet | null = null; - const isRawInvisible = (workspaceId: string): boolean => - !(initialConfigIds?.has(workspaceId) ?? false) && - !(freshConfigIds?.has(workspaceId) ?? false); let finalConfigIds: ReadonlySet | null = null; try { finalConfigIds = this.config.readPersistedWorkspaceIdSuperset(); From a9d70816fdded29d4704ac7bb460b8089e54699c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 14:27:14 +0000 Subject: [PATCH 69/72] fix: address round-63 Codex finding --- .../services/ExtensionMetadataService.test.ts | 43 +++++++++++++++++++ src/node/services/ExtensionMetadataService.ts | 38 +++++++++++----- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index ee83dca3b9..71eca5357c 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1314,6 +1314,49 @@ describe("ExtensionMetadataService", () => { expect(snapshots.get("ws-keep")?.todoStatus?.message).toBe("current status"); }); + test("equal-recency stranded entries from generation-less builds are preserved", async () => { + // A downgraded build's writers drop writeGeneration when mutating an + // entry, and its later goal/status write can share recency with an + // older generation-carrying copy restored by recovery. The + // generation-less side must win the equal-recency tie in BOTH + // positions — as the stranded candidate (or the downgrade's update is + // permanently lost when the claim is consumed) and as the target (a + // stale generation-carrying stranded copy must not displace it). + const entry = (message: string, writeGeneration?: number) => ({ + recency: 500, + streaming: false, + ...(writeGeneration !== undefined ? { writeGeneration } : {}), + lastModel: null, + lastThinkingLevel: null, + agentStatus: null, + lastStatusUrl: null, + todoStatus: { emoji: "s", message }, + }); + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "ws-downgrade": entry("old status", 4), + "ws-target": entry("downgrade status"), + }, + }) + ); + await writeFile( + `${filePath}.recreated-2468-beefcafe`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-downgrade": entry("downgrade newer status"), + "ws-target": entry("stale status", 9), + }, + }) + ); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-downgrade")?.todoStatus?.message).toBe("downgrade newer status"); + expect(snapshots.get("ws-target")?.todoStatus?.message).toBe("downgrade status"); + }); + test("persisted mutations advance the per-entry write generation", async () => { // The durable ordering contract behind the merge above: metadata // mutations advance the generation even when they preserve recency, so diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 46fc9eff3d..4e16cc0d73 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1649,22 +1649,38 @@ export class ExtensionMetadataService { // field merge: the stranded bytes are a complete healthy main // snapshot whose age relative to the CURRENT main is unknowable per // field (the current main may be an older sidecar restore plus newer - // writes). Ordering uses the per-entry writeGeneration (advanced by - // EVERY persisted mutation) when both copies carry distinct ones — - // recency alone cannot order metadata changes because status/goal/ - // streaming writers deliberately preserve it, so an equal-recency - // newer status must not be dropped. When generations are missing - // (bytes written by a build without them) or equal (concurrent - // bumps from the same base), the recency comparison is the best - // remaining order. Both gates require STRICTLY newer, so crash - // replay is a no-op and can never re-fill a field a same-or-newer - // write cleared to null. + // writes). Ordering, in precedence order: + // - per-entry writeGeneration when both copies carry distinct ones + // (advanced by EVERY persisted mutation — recency alone cannot + // order metadata changes because status/goal/streaming writers + // deliberately preserve it); + // - strict recency otherwise; + // - at EQUAL recency, the generation-LESS side wins. A downgraded + // build's writers drop writeGeneration from the entry they mutate, + // so a generation-less copy facing a generation-carrying one may be + // that build's LATER goal/status write whose only surviving copy is + // here — dropping it (and unlinking the claim) would lose the + // downgrade's update permanently (upgrade↔downgrade preservation), + // while wrongly preferring an ancient pre-generation copy only + // resurrects stale metadata that the next status write or + // regeneration self-heals. The same preference keeps a + // generation-less TARGET against a generation-carrying candidate. + // Ties with no generation information keep the target, so crash + // replay of an already-merged claim stays a no-op (after adopting a + // generation-less candidate the main entry is generation-less too); + // the replay window can re-adopt over an interleaved same-recency + // local write, but it is bounded by the claim's lifetime (crash + // between merge and unlink) and self-heals like any stale status. const candidateNewer = typeof candidateGeneration === "number" && typeof targetGeneration === "number" && candidateGeneration !== targetGeneration ? candidateGeneration > targetGeneration - : !(typeof targetRecency === "number" && targetRecency >= candidateRecency); + : typeof targetRecency !== "number" + ? true + : targetRecency !== candidateRecency + ? candidateRecency > targetRecency + : typeof targetGeneration === "number" && typeof candidateGeneration !== "number"; if (!candidateNewer) { continue; } From 5e097ede94348fab70658d70c95ecb51f30f57b1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 14:52:38 +0000 Subject: [PATCH 70/72] fix: order equal-recency stranded merges via epoch-ms write stamps A generation-less stranded .recreated-* file could revert a newer local goal/status write: this build's mutation preserves recency and stamps writeGeneration, but the equal-recency tiebreak blindly preferred the generation-less side, resurrecting the stale copy indefinitely. Make writeGeneration a wall-clock epoch-ms stamp (monotonic per entry) so the stranded-leftover merge can compare a generation-carrying target against the stranded file's mtime (rename preserves it): a stamp that strictly postdates the stranded snapshot keeps the target; otherwise the order is unknowable and the generation-less side still wins to preserve downgraded-build writes. --- .../services/ExtensionMetadataService.test.ts | 67 ++++++++++++++--- src/node/services/ExtensionMetadataService.ts | 75 ++++++++++++++----- 2 files changed, 113 insertions(+), 29 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 71eca5357c..66ec1c3d31 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test"; -import { mkdir, mkdtemp, readdir, readFile, rename, rm, writeFile } from "fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rename, rm, utimes, writeFile } from "fs/promises"; import { tmpdir } from "os"; import * as path from "path"; import { ExtensionMetadataService } from "./ExtensionMetadataService"; @@ -1317,8 +1317,10 @@ describe("ExtensionMetadataService", () => { test("equal-recency stranded entries from generation-less builds are preserved", async () => { // A downgraded build's writers drop writeGeneration when mutating an // entry, and its later goal/status write can share recency with an - // older generation-carrying copy restored by recovery. The - // generation-less side must win the equal-recency tie in BOTH + // older generation-carrying copy restored by recovery. When the + // generation-carrying stamp does NOT postdate the stranded file's + // mtime (here: legacy counter-sized stamps far below any mtime), the + // order is unknowable and the generation-less side must win in BOTH // positions — as the stranded candidate (or the downgrade's update is // permanently lost when the claim is consumed) and as the target (a // stale generation-carrying stranded copy must not displace it). @@ -1357,18 +1359,63 @@ describe("ExtensionMetadataService", () => { expect(snapshots.get("ws-target")?.todoStatus?.message).toBe("downgrade status"); }); + test("an equal-recency generation-less stranded copy does not revert a later local write", async () => { + // The decidable side of the equal-recency tie: a pre-generation main + // (no writeGeneration anywhere — e.g. written before the upgrade to + // this build) is stranded, and THIS build then mutates the entry's + // goal/status, preserving recency but stamping an epoch-ms + // writeGeneration that postdates the stranded file's mtime. The merge + // must keep the newer local write — no later mutation is guaranteed to + // repair a wrong overwrite — instead of blindly preferring the + // generation-less candidate. + await service.updateRecency("ws-fresh", 500); + await service.setTodoStatus("ws-fresh", { emoji: "s", message: "current status" }, true); + const strandedPath = `${filePath}.recreated-8642-deadbea7`; + await writeFile( + strandedPath, + JSON.stringify({ + version: 1, + workspaces: { + "ws-fresh": { + recency: 500, + streaming: false, + todoStatus: { emoji: "s", message: "stale pre-upgrade status" }, + }, + }, + }) + ); + // Backdate the stranded file's mtime below the mutation stamps above + // (rename preserves mtime in production; writeFile here stamps "now", + // which could tie with the mutation's stamp at ms granularity). + const past = new Date(Date.now() - 60_000); + await utimes(strandedPath, past, past); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-fresh")?.todoStatus?.message).toBe("current status"); + // Consumed, not retried forever. + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + test("persisted mutations advance the per-entry write generation", async () => { // The durable ordering contract behind the merge above: metadata - // mutations advance the generation even when they preserve recency, so + // mutations advance the generation even when they preserve recency (so // cross-copy ordering never regresses to the recency tiebreak for - // entries written by this build. + // entries written by this build), and the stamp is wall-clock epoch-ms + // so recovery can order it against a stranded file's mtime. + const readGeneration = async () => { + const raw = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + return raw.workspaces["ws-gen"]; + }; + const before = Date.now(); await service.updateRecency("ws-gen", 100); + const first = await readGeneration(); await service.setTodoStatus("ws-gen", { emoji: "s", message: "working" }, true); - const raw = JSON.parse(await readFile(filePath, "utf-8")) as { - workspaces: Record; - }; - expect(raw.workspaces["ws-gen"].writeGeneration).toBe(2); - expect(raw.workspaces["ws-gen"].recency).toBe(100); + const second = await readGeneration(); + expect(first.writeGeneration).toBeGreaterThanOrEqual(before); + expect(second.writeGeneration).toBeGreaterThan(first.writeGeneration ?? 0); + expect(second.recency).toBe(100); }); test("stranded entries for foreign-removed workspaces are not resurrected", async () => { diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 4e16cc0d73..c18b4c44d5 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -363,15 +363,32 @@ export class ExtensionMetadataService { // deliberately preserve it), so recovery merges cannot order metadata // copies by recency alone — see recoverStrandedRecreatedLeftover. workspace.writeGeneration = ExtensionMetadataService.nextWriteGeneration(workspace); + // (The stamp is epoch-ms so the stranded-leftover merge can order a + // generation-carrying entry against a generation-LESS copy via the + // stranded file's mtime — see nextWriteGeneration.) await this.save(data); return toWorkspaceActivitySnapshot(workspace); }); } + /** + * Next per-entry write stamp: wall-clock epoch milliseconds, floored to + * strictly exceed the previous stamp (same-millisecond mutations and + * backward clock steps stay monotonic per entry). A plain counter would + * order two generation-carrying copies just as well, but the stranded- + * leftover merge must also order a generation-carrying entry against a + * generation-LESS copy (written by a pre-generation or downgraded build): + * with a wall-clock stamp it can compare the entry's last write time + * against the stranded FILE's mtime — same host, same clock — and prove + * the generation-carrying write postdates every byte of the stranded + * snapshot. See recoverStrandedRecreatedLeftover's ordering comment. + */ private static nextWriteGeneration(entry: ExtensionMetadata): number { - return typeof entry.writeGeneration === "number" && Number.isFinite(entry.writeGeneration) - ? entry.writeGeneration + 1 - : 1; + const previous = + typeof entry.writeGeneration === "number" && Number.isFinite(entry.writeGeneration) + ? entry.writeGeneration + : 0; + return Math.max(Date.now(), previous + 1); } constructor(filePath?: string) { @@ -1527,6 +1544,13 @@ export class ExtensionMetadataService { await this.finalizeRecreatedLeftover(claimPath); return; } + // Upper bound on the write time of EVERY entry in the stranded + // snapshot: rename() preserves mtime, so the claim file still carries + // the owner's last save time. The equal-recency ordering below compares + // generation-carrying entries (epoch-ms stamps) against it. Stat + // failures on the process-unique claim are transient: propagate + // (retryable; the claim stays discoverable). + const strandedMtimeMs = (await stat(claimPath)).mtimeMs; // Preliminary main read, only to bound the probe set below: which // candidate ids are MISSING from the current main. Never saved. let preliminary: ExtensionMetadataFile; @@ -1655,22 +1679,33 @@ export class ExtensionMetadataService { // order metadata changes because status/goal/streaming writers // deliberately preserve it); // - strict recency otherwise; - // - at EQUAL recency, the generation-LESS side wins. A downgraded + // - at EQUAL recency with a generation-carrying TARGET facing a + // generation-LESS candidate, the stranded file's mtime decides: the + // stamp is epoch-ms (see nextWriteGeneration) and rename preserves + // mtime, so a target stamp strictly above strandedMtimeMs proves + // the target's write postdates every byte of the stranded snapshot + // (e.g. this build mutated goal/status AFTER a pre-generation main + // was stranded) — keep the target; no later mutation is guaranteed + // to repair a wrong overwrite. Otherwise the order is genuinely + // unknowable and the generation-less candidate wins: a downgraded // build's writers drop writeGeneration from the entry they mutate, - // so a generation-less copy facing a generation-carrying one may be - // that build's LATER goal/status write whose only surviving copy is - // here — dropping it (and unlinking the claim) would lose the - // downgrade's update permanently (upgrade↔downgrade preservation), - // while wrongly preferring an ancient pre-generation copy only - // resurrects stale metadata that the next status write or - // regeneration self-heals. The same preference keeps a - // generation-less TARGET against a generation-carrying candidate. - // Ties with no generation information keep the target, so crash - // replay of an already-merged claim stays a no-op (after adopting a - // generation-less candidate the main entry is generation-less too); - // the replay window can re-adopt over an interleaved same-recency - // local write, but it is bounded by the claim's lifetime (crash - // between merge and unlink) and self-heals like any stale status. + // so the candidate may be that build's LATER goal/status write + // whose only surviving copy is here — dropping it (and unlinking + // the claim) would lose the downgrade's update permanently + // (upgrade↔downgrade preservation), while wrongly preferring an + // ancient copy only resurrects stale metadata that the next status + // write or regeneration self-heals. Coarse-mtime filesystems only + // widen the ambiguous branch, never the destructive one. A + // generation-less TARGET keeps against a generation-carrying + // candidate for the same downgrade-preservation reason (the main + // path is also the actively-written copy the next local mutation + // lands on). Ties with no generation information keep the target, + // so crash replay of an already-merged claim stays a no-op (after + // adopting a generation-less candidate the main entry is + // generation-less too); the replay window can re-adopt over an + // interleaved same-recency local write, but it is bounded by the + // claim's lifetime (crash between merge and unlink) and self-heals + // like any stale status. const candidateNewer = typeof candidateGeneration === "number" && typeof targetGeneration === "number" && @@ -1680,7 +1715,9 @@ export class ExtensionMetadataService { ? true : targetRecency !== candidateRecency ? candidateRecency > targetRecency - : typeof targetGeneration === "number" && typeof candidateGeneration !== "number"; + : typeof targetGeneration === "number" && + typeof candidateGeneration !== "number" && + targetGeneration <= strandedMtimeMs; if (!candidateNewer) { continue; } From a9ddb97a2babaa306559e35cf1a2ad4330cd9eb6 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 17:21:59 +0000 Subject: [PATCH 71/72] =?UTF-8?q?fix:=20round-66=20review=20=E2=80=94=20po?= =?UTF-8?q?st-load=20adoption=20evidence,=20raw-removal=20completeness,=20?= =?UTF-8?q?unique=20empty=20temp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Stranded-leftover recovery gathers adoption registration evidence strictly after the main read (pre-load pass now handles only generation-checked tombstone lifts), so a workspace deregistered during the probe awaits can no longer be resurrected on a stale positive. 2. Raw-visible→raw-invisible transitions count as removal only while the raw view is complete registration evidence: with id-less legacy entries present, the fresh (mid-list) or final (post-probe) authoritative enumeration must deny the id — an affirmation or a failed enumeration retains the entry instead of dropping a revived workspace and republishing the tombstone that evidence just cleared. 3. Empty-file quarantine recovery writes through a process-unique .empty--.tmp with finally-scoped cleanup, so concurrent recoveries can no longer truncate or unlink each other's in-flight temp (the shared fixed name could alias the canonical file). --- .../services/ExtensionMetadataService.test.ts | 54 +++++++ src/node/services/ExtensionMetadataService.ts | 151 +++++++++--------- src/node/services/workspaceService.test.ts | 109 +++++++++++-- src/node/services/workspaceService.ts | 66 ++++++-- 4 files changed, 283 insertions(+), 97 deletions(-) diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index 66ec1c3d31..c62058c647 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1396,6 +1396,60 @@ describe("ExtensionMetadataService", () => { expect(stranded).toEqual([]); }); + test("stranded adoption samples registration strictly after the main read", async () => { + // The post-load evidence rule: a positive probe captured BEFORE the + // main read can go stale when another backend deregisters the + // workspace during the probe awaits — adopting on it would resurrect + // the removed workspace's metadata. Modeled with a probe whose answer + // flips between the tombstone-revalidation sample (registered: lifts + // the tombstone) and the adoption sample (deregistered meanwhile): + // only a post-load adoption sample sees the flip. + await service.updateRecency("ws-other", 1); + service.suppressForeignRemoval("ws-x"); + await writeFile( + `${filePath}.recreated-4242-cafe`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-x": { + recency: 500, + streaming: false, + todoStatus: { emoji: "s", message: "resurrected status" }, + }, + }, + }) + ); + let probeCalls = 0; + service.setRegistrationProbe(() => { + probeCalls += 1; + return Promise.resolve(probeCalls === 1); + }); + const snapshots = await service.getAllSnapshots(); + // The first (registration) sample lifted the tombstone… + expect(service.isWorkspaceDeleted("ws-x")).toBe(false); + // …but the adoption decision used a fresh post-load sample and saw the + // deregistration: proven removed, not resurrected, claim consumed. + expect(snapshots.has("ws-x")).toBe(false); + expect(snapshots.get("ws-other")?.recency).toBe(1); + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("a crash-stranded empty-recovery temp file is inert", async () => { + // Empty-file quarantine recovery writes through a process-unique + // `.empty--.tmp` (a shared fixed name lets concurrent + // recoveries truncate or unlink each other's in-flight temp). A crash + // strands at most one such file, and no scan may consume it — sweeping + // another process's in-flight temp is the same race the unique name + // prevents. + await service.updateRecency("ws-live", 7); + const staleTmp = `${filePath}.empty-12345-dead.tmp`; + await writeFile(staleTmp, "{}"); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-live")?.recency).toBe(7); + expect(await readFile(staleTmp, "utf-8")).toBe("{}"); + }); + test("persisted mutations advance the per-entry write generation", async () => { // The durable ordering contract behind the merge above: metadata // mutations advance the generation even when they preserve recency (so diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index c18b4c44d5..270b968047 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1551,18 +1551,33 @@ export class ExtensionMetadataService { // failures on the process-unique claim are transient: propagate // (retryable; the claim stays discoverable). const strandedMtimeMs = (await stat(claimPath)).mtimeMs; - // Preliminary main read, only to bound the probe set below: which - // candidate ids are MISSING from the current main. Never saved. - let preliminary: ExtensionMetadataFile; + // Tombstone revalidation for candidate ids, following the sidecar + // reconcile's lift-or-suppress generation contract. Safe to sample + // before the main read below: lifts only update the in-memory + // suppression map (never the file), and the merge re-checks live + // tombstone state per entry. + if (this.registrationProbe != null) { + for (const workspaceId of Object.keys(parsed.workspaces)) { + if (!this.deletedWorkspaceIds.has(workspaceId)) { + continue; + } + const generationBefore = this.deletedWorkspaceIds.get(workspaceId); + const registered = await this.registrationProbe(workspaceId); + if (registered && this.deletedWorkspaceIds.get(workspaceId) === generationBefore) { + this.liftTombstone(workspaceId); + } + } + } + let main: ExtensionMetadataFile; try { - const preliminaryParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; - if (!ExtensionMetadataService.isValidMetadataFileShape(preliminaryParsed)) { + const mainParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; + if (!ExtensionMetadataService.isValidMetadataFileShape(mainParsed)) { // Corrupt or newer-schema main: leave the claim for a later pass // (it stays under the scanned prefix) rather than merging across // schemas. return; } - preliminary = preliminaryParsed; + main = mainParsed; } catch (readError) { if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { // Missing-main window: the resumable sidecar recovery owns the @@ -1574,62 +1589,38 @@ export class ExtensionMetadataService { } return; } - // Registration evidence, resolved BEFORE the mergeable main read below - // (probes await config, and holding a pre-probe main snapshot across - // those awaits would let the save clobber a concurrent backend's newer - // write). Two decisions share the probe: - // - tombstoned ids follow the sidecar reconcile's lift-or-suppress - // generation contract; - // - ids MISSING from the main are adopted only with registration - // evidence: the stranded snapshot predates the current main, so a - // missing entry may mean another backend REMOVED the workspace after - // the file was stranded — the local tombstones cannot know — and - // restoring it would resurrect deleted metadata indefinitely (an - // unscoped older build exposes it, and a re-registered deterministic - // legacy id would inherit the stale goal/status). - // Without a wired probe, missing-target adoption stays fail-open (the - // sidecar reconcile's sidecar-only adoption contract; dropping is - // destructive and production always wires the probe). A FAILING probe - // propagates (claim retained, retryable) rather than consuming the - // stranded bytes on unknowable evidence. + // Adoption evidence for candidates MISSING from the loaded main, + // gathered strictly AFTER the load (the concurrency contract's + // post-load evidence rule for destructive/resurrecting decisions): the + // stranded snapshot predates the current main, so a missing entry may + // mean another backend REMOVED the workspace — and a positive probe + // sampled BEFORE the load can go stale when that removal lands during + // the probe awaits, letting the stale positive adopt (resurrect) the + // removed workspace's goal/status indefinitely (an unscoped older + // build exposes it, and a re-registered deterministic legacy id would + // inherit it). These awaits hold the loaded snapshot across config + // reads, widening the window in which the save below clobbers a + // concurrent backend's write — that window exists on every load→save + // slot here and is the contract's documented out-of-scope gap, while + // stale-evidence resurrection is exactly the harm the probe exists to + // prevent, so evidence freshness wins. Without a wired probe, + // missing-target adoption stays fail-open (the sidecar reconcile's + // sidecar-only adoption contract; dropping is destructive and + // production always wires the probe). A FAILING probe propagates + // (claim retained, retryable) rather than consuming the stranded bytes + // on unknowable evidence. const registrationEvidence = new Map(); for (const workspaceId of Object.keys(parsed.workspaces)) { - const tombstoned = this.deletedWorkspaceIds.has(workspaceId); - if (this.registrationProbe == null) { - registrationEvidence.set(workspaceId, !tombstoned); + if (this.deletedWorkspaceIds.has(workspaceId) || workspaceId in main.workspaces) { + // Tombstoned ids are skipped by the merge below; existing targets + // are ordered newest-wins, which needs no registration evidence. continue; } - if (!tombstoned && workspaceId in preliminary.workspaces) { - // No decision needs evidence: the target exists and no local - // tombstone stands. + if (this.registrationProbe == null) { + registrationEvidence.set(workspaceId, true); continue; } - const generationBefore = this.deletedWorkspaceIds.get(workspaceId); - const registered = await this.registrationProbe(workspaceId); - if ( - tombstoned && - registered && - this.deletedWorkspaceIds.get(workspaceId) === generationBefore - ) { - this.liftTombstone(workspaceId); - } - registrationEvidence.set(workspaceId, registered); - } - let main: ExtensionMetadataFile; - try { - const mainParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; - if (!ExtensionMetadataService.isValidMetadataFileShape(mainParsed)) { - return; - } - main = mainParsed; - } catch (readError) { - if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { - return; - } - if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { - throw readError; - } - return; + registrationEvidence.set(workspaceId, await this.registrationProbe(workspaceId)); } let modified = false; for (const [workspaceId, entry] of Object.entries(parsed.workspaces)) { @@ -1649,10 +1640,11 @@ export class ExtensionMetadataService { if (!(workspaceId in main.workspaces)) { const registered = registrationEvidence.get(workspaceId); if (registered === undefined) { - // The entry vanished between the preliminary read and this one - // (deleted during the probes), so no evidence was gathered for - // it. Abort the pass without saving or consuming: the claim - // stays discoverable and the next scan probes the id. Earlier + // No post-load evidence for this missing target: its tombstone + // was lifted between the evidence pass above and this check + // (writers' pre-queue rechecks run outside this queue slot). + // Abort without saving or consuming: the claim stays + // discoverable and the next scan probes the id afresh. Earlier // candidates' merges replay idempotently. return; } @@ -1850,28 +1842,43 @@ export class ExtensionMetadataService { // concurrent writer re-created first), and load()'s sidecar check // turns any remaining missing-main window into a retryable failure // instead of an empty read. - const emptyTmpPath = `${this.filePath}.empty.tmp`; + // Process-unique temp path: with a fixed shared name, a concurrent + // recovery's writeFile could truncate the inode right after this one + // links it into the canonical path (the shared temp path then aliases + // the canonical file, so the truncate empties BOTH and strict readers + // observe empty/partial JSON), or its cleanup unlink could remove the + // temp between this writeFile and link, failing a recovery whose + // sidecar is still recoverable. Crash leftovers are inert: the suffix + // matches no probe or scan prefix, each crashed recovery leaves at most + // one tiny file, and no sweeper may reclaim them (unlinking another + // process's in-flight temp is exactly the race this name prevents). + const emptyTmpPath = `${this.filePath}.empty-${process.pid}-${randomUUID()}.tmp`; await writeFile( emptyTmpPath, JSON.stringify({ version: 1, workspaces: {} } satisfies ExtensionMetadataFile, null, 2), "utf-8" ); try { - await link(emptyTmpPath, this.filePath); - } catch (linkError) { - if (!ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { - try { - await copyFile(emptyTmpPath, this.filePath, constants.COPYFILE_EXCL); - } catch (copyError) { - if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { - // Main path still missing: rethrow (retryable) so the caller's - // re-read hits the sidecar guard instead of reading ENOENT. - throw copyError; + try { + await link(emptyTmpPath, this.filePath); + } catch (linkError) { + if (!ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + try { + await copyFile(emptyTmpPath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + // Main path still missing: rethrow (retryable) so the caller's + // re-read hits the sidecar guard instead of reading ENOENT. + throw copyError; + } } } } + } finally { + // Caller-owned path only; best-effort on every exit so throw paths do + // not strand the temp. + await unlink(emptyTmpPath).catch(() => undefined); } - await unlink(emptyTmpPath).catch(() => undefined); log.error( `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` ); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index a96590363c..d1f122b0cb 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9,7 +9,7 @@ import { createDisplayUsage } from "@/common/utils/tokens/displayUsage"; import { askUserQuestionManager } from "./askUserQuestionManager"; import { WorkspaceLifecycleHooks } from "./workspaceLifecycleHooks"; import { EventEmitter } from "events"; -import { existsSync, writeFileSync } from "fs"; +import { existsSync } from "fs"; import * as fsPromises from "fs/promises"; import { tmpdir } from "os"; import path from "path"; @@ -5847,34 +5847,33 @@ describe("WorkspaceService activity list scoping", () => { // backend" registers a new workspace: its id enters the fresh raw // view outside the initial scope, forcing the late-candidate probes // and with them the final post-probe views this test exercises. + // The final-phase snapshot re-read (call 3) marks the start of the + // post-probe views: the concurrent deregistration lands there and + // every later raw evidence read fails transiently, so only the + // fallback enumeration can prove the removal. const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); let snapshotCalls = 0; + let failRawEvidenceReads = false; const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( async (options?: { throwOnError?: boolean }) => { snapshotCalls += 1; if (snapshotCalls === 2) { await fsPromises.writeFile(configPath, configFor([removedId, survivorId, lateId])); } + if (snapshotCalls === 3) { + await fsPromises.writeFile(configPath, configFor([survivorId, lateId])); + failRawEvidenceReads = true; + } return realGetAllSnapshots(options); } ); - // Raw superset reads: #1 initial baseline, #2 prune enumeration - // union, #3 post-prune scope refresh. Call #4 is the post-probe read - // — the concurrent deregistration lands just before it and every - // later raw read fails transiently, so only the fallback enumeration - // can prove the removal. - const realSuperset = config.readPersistedWorkspaceIdSuperset.bind(config); - let supersetCalls = 0; - const supersetSpy = spyOn(config, "readPersistedWorkspaceIdSuperset").mockImplementation( + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( () => { - supersetCalls += 1; - if (supersetCalls >= 4) { - if (supersetCalls === 4) { - writeFileSync(configPath, configFor([survivorId, lateId])); - } + if (failRawEvidenceReads) { throw new Error("transient raw config read failure"); } - return realSuperset(); + return realEvidence(); } ); try { @@ -5891,7 +5890,7 @@ describe("WorkspaceService activity list scoping", () => { expect(activityList?.[removedId]).toBeUndefined(); } finally { snapshotsSpy.mockRestore(); - supersetSpy.mockRestore(); + evidenceSpy.mockRestore(); } } finally { await cleanup(); @@ -6138,6 +6137,84 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("a raw-removed id affirmed by the fresh enumeration is retained, not tombstoned", async () => { + // A downgraded backend can remove an inline-id workspace entry and + // re-register the SAME deterministic id as an id-less legacy entry + // while this list awaits. The id then vanishes from every fresh raw + // view (its identity lives in session metadata.json) while the fresh + // authoritative enumeration — the very evidence that clears the id's + // tombstone — still resolves it. Treating the raw disappearance alone + // as removal would drop the revived workspace's activity and republish + // the tombstone that evidence just cleared, suppressing it again. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "raw-invisible-revival"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 42); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const internals = workspaceService as unknown as { + pruneStaleExtensionMetadataOnce(): Promise; + enumerateAuthoritativeWorkspaceIds(): Promise>; + }; + const realPrune = internals.pruneStaleExtensionMetadataOnce.bind(workspaceService); + let revivedIdless = false; + internals.pruneStaleExtensionMetadataOnce = async () => { + const prefetched = await realPrune(); + // The removal + id-less re-registration lands after the initial + // baseline and the prune, before the fresh evidence read. + revivedIdless = true; + return prefetched; + }; + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + const evidence = realEvidence(); + if (!revivedIdless) { + return evidence; + } + // The downgraded backend rewrote the entry without an inline id: + // the id disappears from the raw view, and the id-less entry + // marks that view incomplete. + const ids = new Set(evidence.ids); + ids.delete(workspaceId); + return { ids, hasWorkspaceEntriesWithoutIds: true }; + } + ); + const realEnumerate = internals.enumerateAuthoritativeWorkspaceIds.bind(workspaceService); + internals.enumerateAuthoritativeWorkspaceIds = async () => { + // The enumeration resolves the id-less entry's stable identity. + const ids = await realEnumerate(); + ids.add(workspaceId); + return ids; + }; + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.recency).toBe(42); + // No republished tombstone suppressing the revived workspace. + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + } finally { + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap admits snapshotless ids registered after the prune enumerated config", async () => { // A workspace another backend registers after the prune captured its id // set may have workflow- or bash-monitor-only activity and therefore no diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 05b18a215e..eaaa394294 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13776,6 +13776,27 @@ export class WorkspaceService extends EventEmitter { authoritativeIds != null && (scopeEnumerationIds?.has(workspaceId) ?? false) && !authoritativeIds.has(workspaceId)); + // Raw-visible→raw-invisible is verifiable removal only while the + // fresh raw view is COMPLETE registration evidence. With id-less + // legacy entries present, the id may have been removed and + // RE-REGISTERED by a downgraded backend as an id-less entry during + // the awaits (raw-invisible from then on) — the raw comparison alone + // would drop the revived workspace and republish the very tombstone + // the authoritative evidence just cleared, suppressing its activity + // again. In that case the authoritative enumeration (which resolves + // id-less identities, and is attempted whenever id-less entries + // exist — see the third trigger above) must DENY the id before the + // transition counts as removal; an affirmation or a failed + // enumeration retains the entry (keeping a stale entry briefly is + // recoverable, wrongly suppressing a live workspace is not). A + // revival landing after the enumeration but before the raw re-read + // is the contract's out-of-scope window: the next list's initial + // baseline no longer contains the id, so the raw arm cannot re-fire + // and its fresh evidence clears the republished tombstone. + const isVerifiablyRemovedFromRawConfig = (workspaceId: string): boolean => + isRemovedFromConfig(workspaceId) && + (!freshConfigHasRawInvisibleEntries || + (authoritativeIds != null && !authoritativeIds.has(workspaceId))); // Tombstones are process-local removal knowledge; the shared config is // the authority. A downgraded concurrent backend can legitimately // re-register a deterministic legacy id this process pruned earlier — @@ -13836,7 +13857,7 @@ export class WorkspaceService extends EventEmitter { : authoritativeIds != null)) || // Deregistered from the shared config mid-list — also covers // workflow/bash-monitor-only entries with no persisted snapshot. - isRemovedFromConfig(workspaceId) || + isVerifiablyRemovedFromRawConfig(workspaceId) || // Raw-invisible (legacy stable) ids: authoritative-lookup // counterpart of the raw-superset comparison above. isRemovedPerAuthoritativeIdentity(workspaceId); @@ -13974,10 +13995,17 @@ export class WorkspaceService extends EventEmitter { // can prove the removal. Modern deployments never pay this walk. let finalAuthoritativeIds: ReadonlySet | null = null; let finalConfigIds: ReadonlySet | null = null; + // Completeness of the final raw view, mirroring the mid-list + // fresh read: id-less legacy entries make raw denial insufficient + // removal evidence (see isVerifiablyRemovedFromRawConfig). + let finalConfigHasRawInvisibleEntries = false; try { - finalConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + const finalEvidence = this.config.readPersistedWorkspaceIdEvidence(); + finalConfigIds = finalEvidence.ids; + finalConfigHasRawInvisibleEntries = finalEvidence.hasWorkspaceEntriesWithoutIds; } catch { finalConfigIds = null; + finalConfigHasRawInvisibleEntries = true; } if ( Array.from(probedWorkflowRunIds.keys()).some(isRawInvisible) || @@ -13993,7 +14021,15 @@ export class WorkspaceService extends EventEmitter { // no cross-process event to repair it. The enumeration // substitutes as removal evidence for ids the scope enumeration // vouched for. - finalConfigIds == null + finalConfigIds == null || + // Id-less legacy entries in the final raw view: a raw-visible + // id deregistered during the probes is indistinguishable from + // one removed-and-revived as an id-less entry by a downgraded + // backend, so the raw deregistration guards below need the + // enumeration to tell them apart (same rule as the mid-list + // isVerifiablyRemovedFromRawConfig). Modern deployments (every + // id inline) never pay this walk. + finalConfigHasRawInvisibleEntries ) { try { finalAuthoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); @@ -14022,7 +14058,10 @@ export class WorkspaceService extends EventEmitter { // repeat failure keep the earlier successful read (still a // valid post-probe view) rather than degrading to null. try { - finalConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + const refreshedFinalEvidence = this.config.readPersistedWorkspaceIdEvidence(); + finalConfigIds = refreshedFinalEvidence.ids; + finalConfigHasRawInvisibleEntries = + refreshedFinalEvidence.hasWorkspaceEntriesWithoutIds; } catch { // Keep the pre-enumeration read (possibly null). } @@ -14055,11 +14094,16 @@ export class WorkspaceService extends EventEmitter { : finalConfigIds != null)) || // Verifiably deregistered from the raw config during the // probes: visible in an earlier raw view, gone from the - // post-probe one. + // post-probe one. With id-less entries in the final raw view + // the id may instead have been removed-and-revived id-less, + // so the final enumeration must deny it (same completeness + // rule as isVerifiablyRemovedFromRawConfig). (finalConfigIds != null && !finalConfigIds.has(workspaceId) && ((initialConfigIds?.has(workspaceId) ?? false) || - (freshConfigIds?.has(workspaceId) ?? false))) || + (freshConfigIds?.has(workspaceId) ?? false)) && + (!finalConfigHasRawInvisibleEntries || + (finalAuthoritativeIds != null && !finalAuthoritativeIds.has(workspaceId)))) || // Raw-invisible retained ids: post-probe authoritative // counterpart of the raw guard above. (isRawInvisible(workspaceId) && @@ -14101,7 +14145,7 @@ export class WorkspaceService extends EventEmitter { ? (freshSnapshots.get(workspaceId) ?? null) : (snapshots.get(workspaceId) ?? null); const lateForeignRemoved = - isRemovedFromConfig(workspaceId) || + isVerifiablyRemovedFromRawConfig(workspaceId) || isRemovedPerAuthoritativeIdentity(workspaceId) || // Persisted snapshot vanished during the probes — also covers // legacy ids the raw views cannot see. Same corruption-reset @@ -14123,11 +14167,15 @@ export class WorkspaceService extends EventEmitter { // definition) — and is gone from the post-probe view. During // the normal gap between config deregistration and metadata // cleanup the snapshot still exists, so the vanish check - // above cannot catch this. + // above cannot catch this. Same raw-view completeness rule as + // the retained-entry arm: with id-less entries present the + // final enumeration must deny a possibly-revived id. (finalConfigIds != null && !finalConfigIds.has(workspaceId) && ((initialConfigIds?.has(workspaceId) ?? false) || - (freshConfigIds?.has(workspaceId) ?? false))) || + (freshConfigIds?.has(workspaceId) ?? false)) && + (!finalConfigHasRawInvisibleEntries || + (finalAuthoritativeIds != null && !finalAuthoritativeIds.has(workspaceId)))) || // Raw-invisible candidates: post-probe authoritative // counterpart of the raw guard above — a legacy workspace // deregistered during the probes is invisible to every raw From bb84a86762e0bf0dfeedb7f0db68156d0ef27eaf Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 27 Aug 2026 17:41:31 +0000 Subject: [PATCH 72/72] fix: re-enumerate when the post-enumeration raw refresh reveals id-less entries An id removed before the authoritative enumeration and re-registered id-less right after it was dropped (and tombstoned) on the stale denial, even though the post-enumeration raw refresh proved the enumeration's completeness could have been invalidated. Both the mid-list and final-phase guards now run one bounded re-enumeration when the refreshed raw view reports id-less entries, so enumeration-backed removal arms always consult a denial that postdates that refresh; on failure they retain instead of trusting the stale denial set. Raw/snapshot views stay at their earlier reads because their staleness errs toward retention, never a wrong drop. --- src/node/services/workspaceService.test.ts | 84 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 53 ++++++++++++-- 2 files changed, 132 insertions(+), 5 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index d1f122b0cb..885d25ad4a 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6215,6 +6215,90 @@ describe("WorkspaceService activity list scoping", () => { } }); + test("a revival landing between the enumeration and the raw refresh is re-checked, not dropped", async () => { + // The staleness window the post-refresh re-enumeration closes: the id + // is removed BEFORE the mid-list enumeration runs (so that enumeration + // denies it) and re-registered id-less right after it. The raw refresh + // then reports id-less entries — proof the earlier denial may be + // stale — so the removal arms must consult a fresh enumeration (which + // resolves the revived identity) instead of dropping the workspace on + // the stale denial and tombstoning it. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "revived-between-reads"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 42); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const internals = workspaceService as unknown as { + pruneStaleExtensionMetadataOnce(): Promise; + enumerateAuthoritativeWorkspaceIds(): Promise>; + }; + const realPrune = internals.pruneStaleExtensionMetadataOnce.bind(workspaceService); + let removed = false; + internals.pruneStaleExtensionMetadataOnce = async () => { + const prefetched = await realPrune(); + // The cross-process removal lands after the initial baseline and + // the prune. + removed = true; + return prefetched; + }; + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + const evidence = realEvidence(); + if (!removed) { + return evidence; + } + // Post-removal raw views: the id is gone, and an unrelated + // id-less legacy entry keeps the view incomplete throughout. + const ids = new Set(evidence.ids); + ids.delete(workspaceId); + return { ids, hasWorkspaceEntriesWithoutIds: true }; + } + ); + const realEnumerate = internals.enumerateAuthoritativeWorkspaceIds.bind(workspaceService); + let postRemovalEnumerations = 0; + internals.enumerateAuthoritativeWorkspaceIds = async () => { + const ids = await realEnumerate(); + if (!removed) { + return ids; + } + postRemovalEnumerations += 1; + if (postRemovalEnumerations === 1) { + // First post-removal enumeration: the removal is visible, the + // id-less re-registration has not landed yet — a stale denial. + ids.delete(workspaceId); + } + // Later enumerations resolve the revived id-less identity. + return ids; + }; + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.recency).toBe(42); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + } finally { + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("first bootstrap admits snapshotless ids registered after the prune enumerated config", async () => { // A workspace another backend registers after the prune captured its id // set may have workflow- or bash-monitor-only activity and therefore no diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index eaaa394294..f350eed887 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -13754,6 +13754,27 @@ export class WorkspaceService extends EventEmitter { freshConfigIds = null; freshConfigHasRawInvisibleEntries = true; } + if (freshConfigHasRawInvisibleEntries) { + // The refresh postdates the enumeration and reveals id-less + // entries, so the enumeration's DENIALS may already be stale: an + // id removed before the enumeration can have been re-registered + // id-less (with fresh cross-process activity) in the gap, and + // trusting the stale denial would drop the revived workspace and + // tombstone it. Re-enumerate ONCE so the enumeration-backed + // removal arms use the freshest capable view; a revival landing + // after this last read is the contract's out-of-scope window + // (the next list's evidence clears any republished tombstone). + // The raw view deliberately stays at its pre-re-enumeration + // read: staleness there errs toward retention (an id present in + // the older view is kept), never toward a wrong drop. On failure + // fall back to retention, not the stale denial set. + try { + authoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); + } catch (error) { + log.debug("Failed to re-enumerate authoritative ids after raw refresh", { error }); + authoritativeIds = null; + } + } } const isRemovedPerAuthoritativeIdentity = (workspaceId: string): boolean => (initialConfigIds != null && @@ -13788,11 +13809,14 @@ export class WorkspaceService extends EventEmitter { // exist — see the third trigger above) must DENY the id before the // transition counts as removal; an affirmation or a failed // enumeration retains the entry (keeping a stale entry briefly is - // recoverable, wrongly suppressing a live workspace is not). A - // revival landing after the enumeration but before the raw re-read - // is the contract's out-of-scope window: the next list's initial - // baseline no longer contains the id, so the raw arm cannot re-fire - // and its fresh evidence clears the republished tombstone. + // recoverable, wrongly suppressing a live workspace is not). The + // enumeration consulted here postdates the raw refresh whenever that + // refresh reports id-less entries (see the re-enumeration above), so + // a stale pre-refresh denial can never veto a revived id; a revival + // landing after that last enumeration is the contract's out-of-scope + // window — the next list's initial baseline no longer contains the + // id, so the raw arm cannot re-fire and its fresh evidence clears + // the republished tombstone. const isVerifiablyRemovedFromRawConfig = (workspaceId: string): boolean => isRemovedFromConfig(workspaceId) && (!freshConfigHasRawInvisibleEntries || @@ -14065,6 +14089,25 @@ export class WorkspaceService extends EventEmitter { } catch { // Keep the pre-enumeration read (possibly null). } + if (finalConfigHasRawInvisibleEntries) { + // Same staleness rule as the mid-list re-enumeration: the + // refresh revealed id-less entries, so the enumeration's + // denials may predate an id-less re-registration — the + // enumeration-backed drops below must use the freshest + // capable view (a revival after this last read is the + // contract's out-of-scope window). The raw and snapshot + // views deliberately stay at their earlier reads: their + // staleness errs toward retention, never a wrong drop. On + // failure retain rather than trust the stale denial set. + try { + finalAuthoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); + } catch (error) { + log.debug("Failed to re-enumerate authoritative ids after final raw refresh", { + error, + }); + finalAuthoritativeIds = null; + } + } } // The retained-entry filter above ran BEFORE the workflow probes, // so a removal landing during those awaits is invisible to every