diff --git a/src/cli/trust.test.ts b/src/cli/trust.test.ts index 77932a8fdf..11b969f707 100644 --- a/src/cli/trust.test.ts +++ b/src/cli/trust.test.ts @@ -227,7 +227,14 @@ describe("xum trust CLI", () => { .quiet(); expect(result.exitCode).not.toBe(0); - expect(result.stderr.toString()).toContain("Failed to persist trust change"); + // Either failure surface is acceptable: the corrupt-config write gate + // (config.json exists but cannot be read, so editConfig refuses to write + // defaults over it) or the post-write trust verification (the write was + // silently swallowed). Both must fail loudly instead of reporting + // success. + expect(result.stderr.toString()).toMatch( + /Failed to persist trust change|Skipping config write/ + ); expect(result.stdout.toString()).toBe(""); }, 15_000); diff --git a/src/node/config.test.ts b/src/node/config.test.ts index 484f9fb306..7753c6012c 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -903,6 +903,287 @@ describe("Config", () => { }); }); + describe("strict structural validation (throwOnError)", () => { + // Destructive callers (extension-metadata pruning, orphan session-dir + // cleanup) must never receive a lenient-normalized empty/partial workspace + // view for a parseable but structurally invalid config: they would treat + // the omitted live workspaces as removed and delete their data. + const invalidShapes: Array<[string, unknown]> = [ + ["non-array projects", { projects: {} }], + ["non-pair projects entry", { projects: ["not-a-pair"] }], + ["non-object project config", { projects: [["/repo", null]] }], + ["non-array workspaces", { projects: [["/repo", { workspaces: "bogus" }]] }], + // ProjectConfigSchema always persists `workspaces`; a present project + // entry without the key is mangled state that raw evidence flags as + // incomplete — strict mode must not vouch "authoritatively empty" for + // it (the prune would delete every snapshot of that project). + ["missing workspaces key", { projects: [["/repo", {}]] }], + // The lenient path-filter silently drops the WHOLE project for empty + // or non-string keys, and an id-less legacy workspace inside it is + // raw-invisible too (its stable id lives only in session + // metadata.json) — strict mode must fail closed rather than hand the + // prune an id set missing that workspace. + ["null project key", { projects: [[null, { workspaces: [] }]] }], + ["empty-string project key", { projects: [["", { workspaces: [] }]] }], + // A truthy non-string workspace id would ride getAllWorkspaceMetadata's + // modern-entry branch as the authoritative id, omitting the workspace's + // REAL string identity from the prune's known set; non-object entries + // have no establishable identity at all. + ["non-object workspace entry", { projects: [["/repo", { workspaces: ["bogus"] }]] }], + [ + "numeric workspace id", + { projects: [["/repo", { workspaces: [{ id: 42, path: "/repo/ws" }] }]] }, + ], + [ + "empty-string workspace id", + { projects: [["/repo", { workspaces: [{ id: "", path: "/repo/ws" }] }]] }, + ], + ]; + + for (const [label, shape] of invalidShapes) { + it(`rejects ${label} in strict mode while lenient mode still loads`, async () => { + fs.writeFileSync(path.join(tempDir, "config.json"), JSON.stringify(shape)); + const strictConfig = new Config(tempDir); + expect(() => strictConfig.loadConfigOrDefault({ throwOnError: true })).toThrow(); + // try/catch instead of rejects.toThrow: the node-side type-aware lint + // flags awaiting bun's expect() chain as await-thenable. + let strictMetadataRejected = false; + try { + await new Config(tempDir).getAllWorkspaceMetadata({ throwOnError: true }); + } catch { + strictMetadataRejected = true; + } + expect(strictMetadataRejected).toBe(true); + // Ordinary loads keep the historical self-healing behavior: they + // never throw for these shapes (normalized stubs may survive). + expect(() => new Config(tempDir).loadConfigOrDefault()).not.toThrow(); + }); + } + + it("accepts an absent projects key in strict mode (healthy empty config)", () => { + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ defaultProjectDir: "/tmp" }) + ); + const loaded = new Config(tempDir).loadConfigOrDefault({ throwOnError: true }); + expect(loaded.projects.size).toBe(0); + }); + + it("keeps the canonical legacy identity when a secondary alias file is unreadable in lenient loads", async () => { + // An id-less legacy entry with a HEALTHY canonical (generated-legacy) + // metadata file and an unreadable basename-backed second candidate: + // lenient loads must keep the canonical identity instead of + // discarding it for skeletal path-id fallback metadata (which would + // surface the workspace under the WRONG id with its session history + // apparently missing). Strict enumeration still fails closed — the + // unreadable alias may hide a registered identity. + const projectPath = "/repo"; + const workspacePath = "/repo/legacy-ws"; + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + // Migration flags pre-seeded so the first load never schedules the + // async settings-migration persist mid-test. + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }) + ); + const config = new Config(tempDir); + const canonicalId = ( + config as unknown as { + generateLegacyId(projectPath: string, workspacePath: string): string; + } + ).generateLegacyId(projectPath, workspacePath); + const canonicalDir = config.getSessionDir(canonicalId); + fs.mkdirSync(canonicalDir, { recursive: true }); + fs.writeFileSync( + path.join(canonicalDir, "metadata.json"), + JSON.stringify({ id: "stable-canonical-id", name: "legacy-ws" }) + ); + // Basename-backed second candidate is unreadable: a directory at the + // metadata.json path fails reads with EISDIR (non-ENOENT). + fs.mkdirSync(path.join(config.getSessionDir("legacy-ws"), "metadata.json"), { + recursive: true, + }); + + let strictRejected = false; + try { + await config.getAllWorkspaceMetadata({ throwOnError: true }); + } catch { + strictRejected = true; + } + expect(strictRejected).toBe(true); + + const lenient = await config.getAllWorkspaceMetadata(); + const lenientIds = lenient.map((metadata) => metadata.id); + expect(lenientIds).toContain("stable-canonical-id"); + expect(lenientIds).not.toContain(canonicalId); + }); + }); + + describe("readPersistedWorkspaceIdSuperset", () => { + it("ignores nested workspaces arrays inside workspace entries", () => { + // Workspace entries (and unknown newer-build extension objects) can + // carry nested `workspaces`-keyed fields whose ids reference OTHER — + // including removed — workspaces. Treating them as registered would + // lift removed ids' tombstones and recreate stale metadata + // indefinitely; only `projects[*][1].workspaces` direct entries are + // registration evidence. + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ + projects: [ + [ + "/repo", + { + workspaces: [ + { + id: "real-ws", + path: "/repo/ws", + workspaces: [{ id: "phantom-ws", path: "/x" }], + }, + ], + }, + ], + ], + }) + ); + const evidence = new Config(tempDir).readPersistedWorkspaceIdEvidence(); + expect([...evidence.ids]).toEqual(["real-ws"]); + expect(evidence.hasWorkspaceEntriesWithoutIds).toBe(false); + }); + + it("collects ids from entries that lenient normalization discards", () => { + // `[null, ...]` fails the project-path filter and vanishes from the + // normalized view; the raw superset must still surface its workspace id + // so destructive callers never treat that live workspace as removed. + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ + projects: [ + [null, { workspaces: [{ id: "live-discarded", path: "/tmp/x" }] }], + ["/repo", { workspaces: [{ id: "live-normal", path: "/repo/ws", name: "ws" }] }], + ], + }) + ); + const superset = new Config(tempDir).readPersistedWorkspaceIdSuperset(); + expect(superset.has("live-discarded")).toBe(true); + expect(superset.has("live-normal")).toBe(true); + }); + + it("resolves empty for a missing file and throws on unparseable content", () => { + expect(new Config(tempDir).readPersistedWorkspaceIdSuperset().size).toBe(0); + fs.writeFileSync(path.join(tempDir, "config.json"), "{not json"); + expect(() => new Config(tempDir).readPersistedWorkspaceIdSuperset()).toThrow(); + fs.writeFileSync(path.join(tempDir, "config.json"), JSON.stringify(["array-root"])); + expect(() => new Config(tempDir).readPersistedWorkspaceIdSuperset()).toThrow(); + }); + + it("collects only workspace-entry ids, not nested id-bearing objects", () => { + // Workspace entries carry nested id-bearing objects (e.g. + // taskPendingGuidance items) whose ids can reference OTHER — including + // removed — workspaces. Treating those as registered would corrupt + // registration evidence (aborted deletions, lifted tombstones) and + // unbound the activity scope. + fs.writeFileSync( + path.join(tempDir, "config.json"), + JSON.stringify({ + projects: [ + [ + "/repo", + { + workspaces: [ + { + id: "ws-live", + path: "/repo/ws", + taskPendingGuidance: [{ id: "removed-workspace-id", message: "hi" }], + parentWorkspaceId: "some-parent", + }, + ], + }, + ], + ], + }) + ); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(["ws-live"]), + hasWorkspaceEntriesWithoutIds: false, + }); + }); + + it("reports whether any workspace entry lacks an inline id", () => { + // Completeness signal for registration evidence: with inline ids + // everywhere, the raw view is complete and callers may skip the + // per-workspace authoritative enumeration; a single id-less (legacy) + // entry means a raw-invisible stable id may exist. + const configPath = path.join(tempDir, "config.json"); + fs.writeFileSync( + configPath, + JSON.stringify({ + projects: [["/repo", { workspaces: [{ id: "modern", path: "/repo/ws" }] }]], + }) + ); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(["modern"]), + hasWorkspaceEntriesWithoutIds: false, + }); + fs.writeFileSync( + configPath, + JSON.stringify({ + projects: [ + ["/repo", { workspaces: [{ id: "modern", path: "/repo/ws" }, { path: "/repo/old" }] }], + ], + }) + ); + const evidence = new Config(tempDir).readPersistedWorkspaceIdEvidence(); + expect(evidence.hasWorkspaceEntriesWithoutIds).toBe(true); + expect(evidence.ids.has("modern")).toBe(true); + // A PRESENT but malformed (non-array) container is uninterpretable: + // the original entries may have been mangled, so the id set must not + // be treated as complete evidence for destructive decisions. + for (const malformed of [null, "mangled", 7]) { + fs.writeFileSync( + configPath, + JSON.stringify({ projects: [["/repo", { workspaces: malformed }]] }) + ); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(), + hasWorkspaceEntriesWithoutIds: true, + }); + } + // Missing file: healthy empty evidence (fresh install). + fs.rmSync(configPath); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(), + hasWorkspaceEntriesWithoutIds: false, + }); + // Malformed OUTER structure is incomplete evidence too: a mangled + // projects container / pair / project config may be the remnant of + // real registrations. Only an absent projects key is healthy empty. + for (const projects of [ + null, + {}, + "mangled", + [null], + ["not-a-pair"], + [["/repo", null]], + [["/repo", ["array-config"]]], + [["/repo", {}]], // project config with no workspaces key at all + ]) { + fs.writeFileSync(configPath, JSON.stringify({ projects })); + expect( + new Config(tempDir).readPersistedWorkspaceIdEvidence().hasWorkspaceEntriesWithoutIds + ).toBe(true); + } + fs.writeFileSync(configPath, JSON.stringify({ defaultProjectDir: "/tmp" })); + expect(new Config(tempDir).readPersistedWorkspaceIdEvidence()).toEqual({ + ids: new Set(), + hasWorkspaceEntriesWithoutIds: false, + }); + }); + }); + describe("legacy task variant compatibility", () => { it("loads variant children as ordinary sub-agents without destroying downgrade metadata", async () => { const configFile = path.join(tempDir, "config.json"); @@ -3037,6 +3318,94 @@ describe("Config", () => { expect(workspace.name).toBe(workspaceName); expect(workspace.createdAt).toBe("2025-01-01T00:00:00.000Z"); }); + + it("enumerates basename-backed legacy stable ids like findWorkspace", async () => { + // Oldest layout: an id-less config entry whose stable id lives in + // sessions//metadata.json. findWorkspace resolves + // it (basename candidate first), so the enumeration must report the + // same identity — destructive callers (the extension-metadata prune) + // classify ids as stale against the enumeration, and a mismatch would + // delete the live workspace's activity data. + const projectPath = "/fake/project"; + const workspaceName = "old-feature"; + const workspacePath = path.join(config.srcDir, "project", workspaceName); + fs.mkdirSync(workspacePath, { recursive: true }); + + const sessionDir = config.getSessionDir(workspaceName); + fs.mkdirSync(sessionDir, { recursive: true }); + fs.writeFileSync( + path.join(sessionDir, "metadata.json"), + JSON.stringify({ + id: "stable-basename-id", + name: workspaceName, + projectName: "project", + projectPath, + createdAt: "2025-01-01T00:00:00.000Z", + }) + ); + + await config.editConfig((cfg) => { + cfg.projects.set(projectPath, { + workspaces: [{ path: workspacePath }], + }); + return cfg; + }); + + const allMetadata = await config.getAllWorkspaceMetadata({ throwOnError: true }); + expect(allMetadata.map((metadata) => metadata.id)).toContain("stable-basename-id"); + }); + + it("surfaces the second resolvable compatibility file's id as a legacy alias", async () => { + // Both supported layouts exist with DIFFERENT ids (e.g. a stale + // basename-side file next to the live generated-legacy metadata). + // findWorkspace resolves either id, so destructive known-id sets must + // retain both — the GENERATED-LEGACY record stays canonical (its id + // feeds the read-time config migration; a stale basename-side primary + // would rewrite the persisted stable id on upgrade and orphan session + // history) while the basename id is reported through the + // legacyAliasIds out-parameter. + const projectPath = "/fake/project"; + const workspaceName = "aliased-feature"; + const workspacePath = path.join(config.srcDir, "project", workspaceName); + fs.mkdirSync(workspacePath, { recursive: true }); + + const basenameSessionDir = config.getSessionDir(workspaceName); + fs.mkdirSync(basenameSessionDir, { recursive: true }); + fs.writeFileSync( + path.join(basenameSessionDir, "metadata.json"), + JSON.stringify({ id: "stale-basename-id", name: workspaceName }) + ); + const legacyId = config.generateLegacyId(projectPath, workspacePath); + const legacySessionDir = config.getSessionDir(legacyId); + fs.mkdirSync(legacySessionDir, { recursive: true }); + fs.writeFileSync( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: "live-generated-id", name: workspaceName }) + ); + + await config.editConfig((cfg) => { + cfg.projects.set(projectPath, { + workspaces: [{ path: workspacePath }], + }); + return cfg; + }); + + const legacyAliasIds = new Set(); + const allMetadata = await config.getAllWorkspaceMetadata({ + throwOnError: true, + legacyAliasIds, + }); + expect(allMetadata.map((metadata) => metadata.id)).toContain("live-generated-id"); + expect(legacyAliasIds.has("stale-basename-id")).toBe(true); + // The read-time migration must persist the CANONICAL id — a stale + // basename-side id here would change the workspace's stable identity + // on upgrade and make its session history appear missing. + const persisted = config + .loadConfigOrDefault() + .projects.get(projectPath) + ?.workspaces.find((workspace) => workspace.path === workspacePath); + expect(persisted?.id).toBe("live-generated-id"); + }); }); describe("transcriptOnly derivation", () => { diff --git a/src/node/config.ts b/src/node/config.ts index 596c1388a3..5d2a408ea0 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -80,6 +80,11 @@ import { coerceThinkingLevel, type ThinkingLevel } from "@/common/types/thinking export type { Workspace, ProjectConfig, ProjectsConfig, ProviderConfig, CanonicalProvidersConfig }; export type ProvidersConfig = CanonicalProvidersConfig | Record; +/** True only for fs errors whose errno code is ENOENT (genuinely missing path). */ +function isEnoentError(error: unknown): boolean { + return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT"; +} + function isValidHeartbeatIntervalMs(intervalMs: unknown): intervalMs is number { return ( typeof intervalMs === "number" && @@ -1107,14 +1112,144 @@ export class Config { ); } + /** + * Maximal superset of workspace ids present in the RAW persisted config, + * for destructive "id is not in config" decisions (extension-metadata + * pruning, orphan session-dir cleanup). + * + * loadConfigOrDefault's validation/normalization is LOSSY: entries it + * filters or discards (invalid project paths, malformed pairs, entries a + * migration rewrites) simply vanish from the normalized view, so a pruner + * keyed on that view would treat their live workspaces as removed and + * delete their data. This scan instead walks the raw `projects` subtree + * and collects every string `id` it can find, without judging validity — + * over-collection merely retains a stale entry a little longer, while + * under-collection destroys live data. Callers should union this with the + * normalized view (which contains ids created by in-memory migrations). + * + * Throws when the file exists but cannot be read/parsed or the root is not + * a plain object; a missing file resolves as an empty set (fresh install). + */ + readPersistedWorkspaceIdSuperset(): Set { + return this.readPersistedWorkspaceIdEvidence().ids; + } + + /** + * readPersistedWorkspaceIdSuperset plus a completeness signal: whether any + * persisted workspace entry lacks an inline string id. Only such id-less + * (legacy) entries can be registered "raw-invisibly" — their stable id + * lives in the session metadata.json, which only the authoritative + * enumeration resolves. When this reports false, the raw id set is + * complete registration evidence and callers can skip that per-workspace + * enumeration. Detection is conservative in the destructive direction: + * any workspaces container whose entries cannot be verified to carry ids + * reports true (over-reporting merely costs an extra enumeration, while + * under-reporting could let a pruner delete a live legacy workspace). + */ + readPersistedWorkspaceIdEvidence(): { + ids: Set; + hasWorkspaceEntriesWithoutIds: boolean; + } { + const ids = new Set(); + let hasWorkspaceEntriesWithoutIds = false; + let raw: string; + try { + raw = fs.readFileSync(this.configFile, "utf-8"); + } catch (error) { + // No existsSync probe: it also returns false for EACCES/ENOTDIR/EIO, + // which would report a transiently unreadable config as an empty id + // set and let destructive callers treat every workspace as removed. + // Only a genuinely missing file is a healthy empty set (fresh install). + if (isEnoentError(error)) { + return { ids, hasWorkspaceEntriesWithoutIds }; + } + throw error; + } + const parsedValue: unknown = JSON.parse(raw); + if (!parsedValue || typeof parsedValue !== "object" || Array.isArray(parsedValue)) { + throw new Error("Config root must be a JSON object"); + } + const hasInlineStringId = (value: unknown): boolean => { + if (value === null || typeof value !== "object") { + return false; + } + const id = (value as { id?: unknown }).id; + return typeof id === "string" && id.length > 0; + }; + // Collect ids ONLY from the direct entries of `projects[*][1].workspaces` + // — never from arbitrary nested objects. Workspace entries carry nested + // id-bearing objects (e.g. taskPendingGuidance items) and can even carry + // nested `workspaces`-keyed fields written by other builds; ids found + // there can reference OTHER (including removed) workspaces, and treating + // them as registered corrupts registration evidence (aborted deletions, + // lifted tombstones, ghost activity probes) and unbounds the activity + // scope. Under-collection stays safe: any workspaces container whose + // entries cannot be verified (id-less entry, non-array container) flips + // the incompleteness flag, routing callers to the strict enumeration. + // + // The OUTER structure must be interpretable too, or the id set cannot be + // proven complete: a present non-array `projects`, a non-pair element, a + // non-object project config, or a project config with no workspaces key + // at all may be a mangled remnant of real registrations. Only an absent + // projects key is healthy emptiness (the strict loader accepts it). + // A malformed sibling project only flips the flag — ids from the + // remaining well-formed projects are still collected. + const projects = (parsedValue as { projects?: unknown }).projects; + if (projects !== undefined) { + if (!Array.isArray(projects)) { + hasWorkspaceEntriesWithoutIds = true; + } else { + for (const pair of projects) { + const projectConfig: unknown = Array.isArray(pair) ? pair[1] : undefined; + if ( + projectConfig === null || + typeof projectConfig !== "object" || + Array.isArray(projectConfig) + ) { + hasWorkspaceEntriesWithoutIds = true; + continue; + } + const workspaces = (projectConfig as { workspaces?: unknown }).workspaces; + if (!Array.isArray(workspaces)) { + // A missing key or a PRESENT container in any non-array shape is + // uninterpretable evidence — the original entries may have been + // mangled, so the raw id set cannot be proven complete. + hasWorkspaceEntriesWithoutIds = true; + continue; + } + for (const entry of workspaces) { + if (hasInlineStringId(entry)) { + ids.add((entry as { id: string }).id); + } else { + hasWorkspaceEntriesWithoutIds = true; + } + } + } + } + } + return { ids, hasWorkspaceEntriesWithoutIds }; + } + loadConfigOrDefault(options?: { throwOnError?: boolean }): ProjectsConfig { // Read as a Buffer and hand the same snapshot to the failure handler: backing up via a // second read could preserve a concurrent writer's replacement instead of the bytes that // actually failed parsing. let rawBytes: Buffer | undefined; try { - if (fs.existsSync(this.configFile)) { + try { rawBytes = fs.readFileSync(this.configFile); + } catch (readError) { + // No existsSync probe: it also returns false for EACCES/ENOTDIR/EIO, + // which would silently select the fresh-install default while the + // config is merely transiently unreadable — in strict mode that + // empty view feeds destructive "not in config" decisions. Route + // non-ENOENT failures through the shared failure path below + // (throwOnError callers rethrow); only ENOENT means missing. + if (!isEnoentError(readError)) { + throw readError; + } + } + if (rawBytes !== undefined) { const parsedValue: unknown = JSON.parse(rawBytes.toString("utf-8")); if (!parsedValue || typeof parsedValue !== "object" || Array.isArray(parsedValue)) { throw new Error("Config root must be a JSON object"); @@ -1266,6 +1401,84 @@ export class Config { // Config is stored as array of [path, config] pairs. // Older/newer files may omit `projects`; treat missing/invalid values as an empty map // so top-level settings (provider/runtime/server preferences) still load. + // + // Strict mode must NOT accept that lenient normalization: callers that + // make destructive "id is not in config" decisions (extension-metadata + // pruning, orphan session-dir cleanup) would interpret a structurally + // invalid-but-parseable file (e.g. `projects: {}` or a non-array + // `workspaces`) as an empty/partial workspace set and delete live + // data. A genuinely ABSENT `projects` key stays valid in strict mode: + // it is how older/newer builds persist a config with no projects. + if (options?.throwOnError && parsed.projects !== undefined) { + if (!Array.isArray(parsed.projects)) { + throw new Error("Config projects must be an array of [path, config] pairs"); + } + for (const pair of parsed.projects) { + if (!Array.isArray(pair)) { + throw new Error("Config projects entries must be [path, config] pairs"); + } + const projectKey: unknown = pair[0]; + // The lenient normalization below silently drops the WHOLE + // project when its key is empty or non-string ("Filtering out + // project with invalid path"), and an id-less legacy workspace + // inside it is raw-invisible too (its stable id lives only in + // session metadata.json, which only the normalized enumeration + // resolves). Accepting the pair here would hand destructive + // strict callers an authoritative id set missing every one of + // that project's workspaces — the startup prune would then + // permanently delete their recency/goal/status snapshots. + if (typeof projectKey !== "string" || projectKey.length === 0) { + throw new Error("Config project entries must have a non-empty string path"); + } + const projectConfig: unknown = pair[1]; + // Arrays pass typeof "object": lenient normalization would turn + // an array-valued project config into a project with no + // workspaces, and destructive strict callers would then classify + // every one of its workspaces as removed. + if ( + projectConfig === null || + typeof projectConfig !== "object" || + Array.isArray(projectConfig) + ) { + throw new Error("Config project entries must be objects"); + } + const workspaces = (projectConfig as { workspaces?: unknown }).workspaces; + // ProjectConfigSchema persists `workspaces` as a REQUIRED array, + // so a present project entry without the key is mangled state + // (readPersistedWorkspaceIdEvidence flags it incomplete too). + // Accepting it here would hand destructive callers an + // authoritatively-empty workspace set for that project. + if (!Array.isArray(workspaces)) { + throw new Error("Config project workspaces must be an array"); + } + for (const workspaceEntry of workspaces) { + // WorkspaceSchema persists workspace entries as objects; a + // non-object entry is mangled state whose identity cannot be + // established. + if ( + workspaceEntry === null || + typeof workspaceEntry !== "object" || + Array.isArray(workspaceEntry) + ) { + throw new Error("Config workspace entries must be objects"); + } + const workspaceId = (workspaceEntry as { id?: unknown }).id; + // A truthy non-string id (42, {}) would ride the modern-entry + // branch of getAllWorkspaceMetadata as the authoritative id, + // so the prune's known set would omit the workspace's REAL + // string identity and delete its activity snapshot. Nullish + // ids stay valid: legacy entries resolve their stable id + // through session metadata.json, and the strict guard there + // fails closed when that resolution yields no usable id. + if ( + workspaceId != null && + !(typeof workspaceId === "string" && workspaceId.length > 0) + ) { + throw new Error("Config workspace ids must be non-empty strings"); + } + } + } + } const rawPairs = Array.isArray(parsed.projects) ? parsed.projects : []; // Migrate: normalize project paths by stripping trailing slashes // This fixes configs created with paths like "/home/user/project/" @@ -2105,7 +2318,19 @@ export class Config { * Find a workspace by ID. * @returns Stored config project key plus a separate attribution project path, or null */ - findWorkspace(workspaceId: string): { + findWorkspace( + workspaceId: string, + options?: { + /** + * Propagate failures that hide a workspace's identity (unreadable or + * unparseable config / legacy session metadata.json) instead of + * skipping the entry. Callers making destructive "id is not + * registered" decisions must use this: a lenient miss is + * indistinguishable from a genuine absence. + */ + throwOnError?: boolean; + } + ): { workspacePath: string; projectPath: string; attributionProjectPath?: string; @@ -2114,7 +2339,7 @@ export class Config { parentWorkspaceId?: string; pendingAutoTitle?: boolean; } | null { - const config = this.loadConfigOrDefault(); + const config = this.loadConfigOrDefault({ throwOnError: options?.throwOnError }); for (const [projectPath, project] of config.projects) { for (const workspace of project.workspaces) { @@ -2143,28 +2368,90 @@ export class Config { // Try loading metadata with basename as ID (works for old workspaces) const metadataPath = path.join(this.getSessionDir(workspaceBasename), "metadata.json"); - if (fs.existsSync(metadataPath)) { - try { - const data = fs.readFileSync(metadataPath, "utf-8"); - const metadata = JSON.parse(data) as WorkspaceMetadata; - this.rememberLegacyTaskVariantWorkspace(projectPath, metadata, "metadata"); - if (metadata.id === workspaceId) { - return { - workspacePath: workspace.path, - projectPath, - attributionProjectPath, - projects: metadata.projects ?? workspace.projects, - workspaceName: undefined, - parentWorkspaceId: undefined, - }; - } - } catch { - // Ignore parse errors, try legacy ID + try { + const data = fs.readFileSync(metadataPath, "utf-8"); + const metadata = JSON.parse(data) as WorkspaceMetadata; + this.rememberLegacyTaskVariantWorkspace(projectPath, metadata, "metadata"); + // Parseable-but-id-less metadata (e.g. `{}`) leaves this entry's + // identity unknowable: strict callers (the extension-metadata + // discard) must not conclude "not registered" from it, or a live + // workspace whose stable id lived only here gets its activity + // deleted and write-tombstoned. Mirrors the strict enumeration + // guard in getAllWorkspaceMetadata. The catch below rethrows this + // for strict callers and keeps ignoring it for lenient ones. + if ( + options?.throwOnError && + !(typeof metadata.id === "string" && metadata.id.length > 0) + ) { + throw new Error( + `Legacy workspace metadata at ${metadataPath} resolved without a usable id` + ); } + if (metadata.id === workspaceId) { + return { + workspacePath: workspace.path, + projectPath, + attributionProjectPath, + projects: metadata.projects ?? workspace.projects, + workspaceName: undefined, + parentWorkspaceId: undefined, + }; + } + } catch (error) { + // A genuinely missing file is the common case (most entries have + // no legacy metadata.json). The entry's identity may live in an + // unreadable/unparseable one though, so strict callers must not + // conclude "absent" from a failed lookup. + if (!isEnoentError(error) && options?.throwOnError) { + throw error; + } + // Ignore errors, try legacy ID } - // Try legacy ID format as last resort + // Authoritative legacy path: getAllWorkspaceMetadata resolves an + // id-less entry's stable id from sessions// + // metadata.json (NOT the basename path above). Callers verifying + // "is this id still registered" (e.g. the extension-metadata + // discard) must see the same identity, or a stable id that lives + // only in that file would be reported absent while its workspace + // remains registered. const legacyId = this.generateLegacyId(projectPath, workspace.path); + const legacyMetadataPath = path.join(this.getSessionDir(legacyId), "metadata.json"); + try { + const legacyData = fs.readFileSync(legacyMetadataPath, "utf-8"); + const legacyMetadata = JSON.parse(legacyData) as WorkspaceMetadata; + this.rememberLegacyTaskVariantWorkspace(projectPath, legacyMetadata, "metadata"); + // Same unknowable-identity guard as the basename lookup above: + // this is the authoritative file getAllWorkspaceMetadata resolves + // stable ids from, so an id-less parse here must fail closed in + // strict mode rather than fall through to "not registered". + if ( + options?.throwOnError && + !(typeof legacyMetadata.id === "string" && legacyMetadata.id.length > 0) + ) { + throw new Error( + `Legacy workspace metadata at ${legacyMetadataPath} resolved without a usable id` + ); + } + if (legacyMetadata.id === workspaceId) { + return { + workspacePath: workspace.path, + projectPath, + attributionProjectPath, + projects: legacyMetadata.projects ?? workspace.projects, + workspaceName: undefined, + parentWorkspaceId: undefined, + }; + } + } catch (error) { + // Same strict-mode contract as the basename lookup above. + if (!isEnoentError(error) && options?.throwOnError) { + throw error; + } + // Ignore errors, try legacy ID + } + + // Try legacy ID format as last resort if (legacyId === workspaceId) { return { workspacePath: workspace.path, @@ -2224,8 +2511,30 @@ export class Config { * If missing from config or legacy metadata, a new timestamp is assigned and * saved to config for subsequent loads. */ - async getAllWorkspaceMetadata(): Promise { - const config = this.loadConfigOrDefault(); + async getAllWorkspaceMetadata(options?: { + /** + * Throw on config read/parse failure instead of silently resolving with + * the empty default. Callers that make destructive decisions based on + * "workspace is not in config" (e.g. extension-metadata pruning) must use + * this: the swallowed-failure default is indistinguishable from a truly + * empty config. A missing config file still resolves as empty (healthy + * fresh install), matching loadConfigOrDefault. + */ + throwOnError?: boolean; + /** + * Out-parameter collecting ADDITIONAL stable ids that findWorkspace() + * can resolve for id-less legacy entries but that are not the returned + * entry's primary id: when both compatibility files + * (sessions//metadata.json and + * sessions//metadata.json) exist with different + * ids, only the first becomes the metadata entry, yet the second + * identity remains registered for targeted lookups. Destructive callers + * building "known id" sets must include these aliases or they would + * delete activity data findWorkspace still vouches for. + */ + legacyAliasIds?: Set; + }): Promise { + const config = this.loadConfigOrDefault({ throwOnError: options?.throwOnError }); const workspaceMetadata: FrontendWorkspaceMetadata[] = []; // Read-time migrations recorded here are re-applied to a FRESH config snapshot inside // editConfig below. Persisting the local `config` snapshot directly (the old @@ -2394,15 +2703,90 @@ export class Config { continue; // Skip metadata file lookup } - // LEGACY FORMAT: Fall back to reading metadata.json - // Try legacy ID format first (project-workspace) - used by E2E tests and old workspaces + // LEGACY FORMAT: Fall back to reading metadata.json. + // findWorkspace resolves an id-less entry's stable id from EITHER + // sessions//metadata.json or + // sessions//metadata.json (old layout). + // Enumerate BOTH candidates: destructive callers (the + // extension-metadata prune) classify ids as stale against this + // enumeration, so a stable id that findWorkspace can resolve but + // this walk cannot would be deleted as unknown. + // The generated-legacy record stays CANONICAL when both exist: + // this walk's primary id feeds the read-time config migration, and + // it historically consulted only the generated-legacy file — making + // a (potentially stale) basename-side file primary would rewrite + // the workspace's persisted stable id on upgrade and orphan its + // session history. The basename id is surfaced as an alias below; + // it becomes primary only when it is the sole surviving record. const legacyId = this.generateLegacyId(projectPath, workspace.path); - const metadataPath = path.join(this.getSessionDir(legacyId), "metadata.json"); let metadataFound = false; - if (fs.existsSync(metadataPath)) { - const data = fs.readFileSync(metadataPath, "utf-8"); - const metadata = JSON.parse(data) as WorkspaceMetadata; + let metadataPath = ""; + let legacyMetadataRaw: string | undefined; + const candidateIds = + workspaceBasename === legacyId ? [legacyId] : [legacyId, workspaceBasename]; + for (const candidateId of candidateIds) { + const candidatePath = path.join(this.getSessionDir(candidateId), "metadata.json"); + let candidateRaw: string | undefined; + try { + candidateRaw = fs.readFileSync(candidatePath, "utf-8"); + } catch (readError) { + // Missing is normal (most entries never had a legacy + // metadata.json). Any other failure means the workspace's + // authoritative stable id is unknowable right now — surface it + // to the catch below instead of silently substituting the + // generated legacy path id via the !metadataFound branch. + if (!isEnoentError(readError)) { + // Secondary-candidate failure with a usable canonical + // record already in hand: lenient loads keep the canonical + // identity instead of discarding it for skeletal path-id + // fallback metadata (which would surface the workspace + // under the WRONG id with its session history apparently + // missing). Destructive strict enumeration still fails + // closed — the unreadable alias file may hide a registered + // identity. + if (legacyMetadataRaw !== undefined && !options?.throwOnError) { + continue; + } + throw readError; + } + continue; + } + if (legacyMetadataRaw === undefined) { + legacyMetadataRaw = candidateRaw; + metadataPath = candidatePath; + continue; + } + // A SECOND resolvable compatibility file: findWorkspace() tries + // every candidate, so its id stays registered for targeted + // lookups even though only the first file becomes the metadata + // entry. Surface it through the legacyAliasIds out-parameter so + // destructive known-id sets retain it. findWorkspace consults + // candidate files only for id-less entries; for a partially + // migrated entry (inline id, no name) the alias is + // over-inclusive, which errs toward retention, never deletion. + // A corrupt or id-less second file leaves the alias identity + // unknowable — fail closed in strict mode, mirroring the + // primary lookup guards. + let aliasMetadata: WorkspaceMetadata | undefined; + try { + aliasMetadata = JSON.parse(candidateRaw) as WorkspaceMetadata; + } catch (parseError) { + if (options?.throwOnError) { + throw parseError; + } + } + const aliasId = aliasMetadata?.id; + if (typeof aliasId === "string" && aliasId.length > 0) { + options?.legacyAliasIds?.add(aliasId); + } else if (aliasMetadata !== undefined && options?.throwOnError) { + throw new Error( + `Legacy workspace metadata at ${candidatePath} resolved without a usable id` + ); + } + } + if (legacyMetadataRaw !== undefined) { + const metadata = JSON.parse(legacyMetadataRaw) as WorkspaceMetadata; this.rememberLegacyTaskVariantWorkspace(projectPath, metadata, "metadata"); // Ensure required fields are present @@ -2461,6 +2845,21 @@ export class Config { // different value here would hand the UI an ID that findWorkspace cannot // resolve until the next reload. metadata.id = workspace.id ?? metadata.id; + // Strict callers make destructive "id is not known" decisions (the + // extension-metadata prune): metadata.json that parses but carries + // no usable id (e.g. `{}`, or a non-object like an array) resolves + // to an id-less entry here, and the workspace's REAL stable id — + // recorded nowhere else — would be classified as stale and its + // activity data permanently deleted. Successful JSON parsing does + // not establish identity; fail closed instead. + if ( + options?.throwOnError && + !(typeof metadata.id === "string" && metadata.id.length > 0) + ) { + throw new Error( + `Legacy workspace metadata at ${metadataPath} resolved without a usable id` + ); + } metadata.name = workspace.name ?? metadata.name; metadata.createdAt = workspace.createdAt ?? metadata.createdAt; metadata.runtimeConfig = workspace.runtimeConfig ?? metadata.runtimeConfig; @@ -2576,6 +2975,14 @@ export class Config { ); } } catch (error) { + // Strict callers make destructive "id is not known" decisions (the + // extension-metadata prune): for a legacy entry whose stable id + // lives only in its unreadable/unparseable metadata.json, the + // generated-path-id fallback below would classify the real id's + // entries as stale. Propagate the identity-lookup failure instead. + if (options?.throwOnError) { + throw error; + } log.error(`Failed to load/migrate workspace metadata:`, error); // Fallback to basic metadata if migration fails const legacyId = this.generateLegacyId(projectPath, workspace.path); diff --git a/src/node/services/ExtensionMetadataService.test.ts b/src/node/services/ExtensionMetadataService.test.ts index f9813902ca..c62058c647 100644 --- a/src/node/services/ExtensionMetadataService.test.ts +++ b/src/node/services/ExtensionMetadataService.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test"; -import { mkdtemp, rm, writeFile } from "fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rename, rm, utimes, writeFile } from "fs/promises"; import { tmpdir } from "os"; import * as path from "path"; import { ExtensionMetadataService } from "./ExtensionMetadataService"; @@ -377,4 +377,1970 @@ describe("ExtensionMetadataService", () => { const snapshots = await service.getAllSnapshots(); expect(snapshots.get("workspace-2")).toEqual(cleared); }); + + test("pruneMissingWorkspaces drops unknown entries and keeps known ones", async () => { + await service.updateRecency("known-workspace", 100); + await service.updateRecency("stale-workspace", 200); + await service.updateRecency("another-stale", 300); + + const prunedCount = await service.pruneMissingWorkspaces(() => + Promise.resolve(new Set(["known-workspace"])) + ); + expect(prunedCount).toBe(2); + + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("known-workspace")?.recency).toBe(100); + expect(snapshots.has("stale-workspace")).toBe(false); + expect(snapshots.has("another-stale")).toBe(false); + }); + + test("pruneMissingWorkspaces preserves unrecognized fields on surviving entries", async () => { + // Upgrade↔downgrade: a newer build may persist fields this build does not + // know about; pruning stale siblings must not strip them. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "known-workspace": { + recency: 100, + streaming: false, + lastModel: null, + lastThinkingLevel: null, + agentStatus: null, + fieldFromNewerBuild: { nested: true }, + }, + "stale-workspace": { recency: 200, streaming: false }, + }, + }) + ); + + expect( + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set(["known-workspace"]))) + ).toBe(1); + + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(Object.keys(persisted.workspaces)).toEqual(["known-workspace"]); + expect( + (persisted.workspaces["known-workspace"] as { fieldFromNewerBuild?: unknown }) + .fieldFromNewerBuild + ).toEqual({ nested: true }); + }); + + test("pruneMissingWorkspaces fetches known ids only after loading the file", async () => { + // The cross-process loss-safety argument (see the pruneMissingWorkspaces + // doc comment) requires the known-ids fetch to observe every workspace + // registration that preceded an entry visible in the loaded file — i.e. + // load first, fetch second. Fetch-first would misclassify a concurrently + // created workspace's fresh entry as stale. + await service.updateRecency("known-workspace", 100); + const order: string[] = []; + const internals = service as unknown as ExtensionMetadataServiceInternals; + const originalLoad = internals.load.bind(service); + internals.load = async () => { + order.push("load"); + return originalLoad(); + }; + try { + await service.pruneMissingWorkspaces(() => { + order.push("fetch-known-ids"); + return Promise.resolve(new Set(["known-workspace"])); + }); + } finally { + internals.load = originalLoad; + } + expect(order).toEqual(["load", "fetch-known-ids"]); + }); + + test("deleteWorkspace removes malformed falsy persisted entries", async () => { + // Key presence, not truthiness: a null entry must not survive removal + // (it would leak the key until the next process-start prune). + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "malformed-workspace": null, "other-workspace": { recency: 1 } }, + }) + ); + await service.deleteWorkspace("malformed-workspace"); + + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect("malformed-workspace" in persisted.workspaces).toBe(false); + expect("other-workspace" in persisted.workspaces).toBe(true); + }); + + test("late writers cannot resurrect a deleted workspace entry", async () => { + // Removal cannot drain every in-flight metadata producer (e.g. a + // stream-abort's fire-and-forget stop-status write), so writes landing + // after deleteWorkspace must not recreate the entry on disk. + await service.updateRecency("removed-workspace", 100); + await service.deleteWorkspace("removed-workspace"); + + const lateStreaming = await service.setStreaming("removed-workspace", false, { + hasTodos: false, + }); + // Callers still get a computed snapshot; it just is not persisted. + expect(lateStreaming.streaming).toBe(false); + await service.updateRecency("removed-workspace", 200); + await service.setTodoStatus("removed-workspace", { emoji: "x", message: "late" }, true); + expect( + await service.setSidebarStatus("removed-workspace", { emoji: "x", message: "late" }) + ).toBeNull(); + + expect((await service.getAllSnapshots()).has("removed-workspace")).toBe(false); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(persisted.workspaces["removed-workspace"]).toBeUndefined(); + + // Unrelated workspaces keep writing normally. + await service.updateRecency("other-workspace", 300); + expect((await service.getAllSnapshots()).get("other-workspace")?.recency).toBe(300); + }); + + test("pruneMissingWorkspaces preserves entries written by another process mid-prune", async () => { + // XUM_ALLOW_MULTIPLE_INSTANCES: a second backend can register a fresh + // workspace and complete its first metadata write between this pass's + // snapshot load and its pruned rewrite. The deletion-only merge against a + // fresh reload must preserve that entry instead of clobbering the file + // with the pre-write snapshot. + await service.updateRecency("known-workspace", 100); + await service.updateRecency("stale-workspace", 200); + + const internals = service as unknown as ExtensionMetadataServiceInternals; + const originalLoad = internals.load.bind(service); + let foreignWriteInjected = false; + internals.load = async () => { + const data = await originalLoad(); + if (!foreignWriteInjected) { + foreignWriteInjected = true; + // Simulate the foreign process's write landing after our snapshot. + const foreign = await originalLoad(); + foreign.workspaces["foreign-new"] = { + recency: 300, + streaming: false, + lastModel: null, + lastThinkingLevel: null, + agentStatus: null, + }; + await writeFile(filePath, JSON.stringify(foreign)); + } + return data; + }; + try { + expect( + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set(["known-workspace"]))) + ).toBe(1); + } finally { + internals.load = originalLoad; + } + + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(persisted.workspaces["foreign-new"]).toBeDefined(); + expect(persisted.workspaces["known-workspace"]).toBeDefined(); + expect(persisted.workspaces["stale-workspace"]).toBeUndefined(); + }); + + test("pruneMissingWorkspaces preserves writes landing during the re-registration recheck", async () => { + // The recheck can perform a full legacy enumeration — the longest await + // in the prune. A concurrent backend's write landing during it must not + // be rolled back by the pruned rewrite: the fresh snapshot is loaded + // strictly AFTER the recheck resolves. + await service.updateRecency("known-workspace", 100); + await service.updateRecency("stale-workspace", 200); + const recheck = async (): Promise> => { + // Foreign process write landing while the recheck enumerates. + const onDisk = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + (onDisk.workspaces["known-workspace"] as { recency: number }).recency = 999; + await writeFile(filePath, JSON.stringify(onDisk)); + return new Set(["known-workspace"]); + }; + expect( + await service.pruneMissingWorkspaces( + () => Promise.resolve(new Set(["known-workspace"])), + recheck + ) + ).toBe(1); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect((persisted.workspaces["known-workspace"] as { recency?: number }).recency).toBe(999); + expect(persisted.workspaces["stale-workspace"]).toBeUndefined(); + }); + + test("pruneMissingWorkspaces spares a stale entry rewritten during the recheck", async () => { + // Inverse race of the post-recheck reload: an id re-registered and + // written by another backend after the recheck read its registration + // evidence is still classified deletable while holding fresh activity. + // The unchanged-bytes guard proves the writer and fails closed. + await service.updateRecency("stale-workspace", 200); + const recheck = async (): Promise> => { + const onDisk = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + (onDisk.workspaces["stale-workspace"] as { recency: number }).recency = 999; + await writeFile(filePath, JSON.stringify(onDisk)); + return new Set(); // Registration evidence predates the write. + }; + expect( + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set()), recheck) + ).toBe(0); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect((persisted.workspaces["stale-workspace"] as { recency?: number }).recency).toBe(999); + }); + + test("late writers cannot resurrect entries reclaimed by pruneMissingWorkspaces", async () => { + await service.updateRecency("stale-workspace", 100); + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set())); + + await service.updateRecency("stale-workspace", 200); + expect((await service.getAllSnapshots()).has("stale-workspace")).toBe(false); + }); + + test("a writer enqueued while the prune is mid-fetch cannot resurrect a reclaimed entry", async () => { + // The prune publishes tombstones only while its queued mutation runs. A + // writer that passes its pre-queue tombstone check during the prune's + // known-ids fetch enqueues BEHIND the prune, so only the in-queue + // re-check stops it from recreating the entry the prune just reclaimed. + await service.updateRecency("stale-workspace", 100); + let releaseKnownIds!: () => void; + const gate = new Promise((resolve) => { + releaseKnownIds = resolve; + }); + const prune = service.pruneMissingWorkspaces(async () => { + await gate; + return new Set(); + }); + const lateWriter = service.updateRecency("stale-workspace", 200); + releaseKnownIds(); + expect(await prune).toBe(1); + await lateWriter; + + expect((await service.getAllSnapshots()).has("stale-workspace")).toBe(false); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(persisted.workspaces["stale-workspace"]).toBeUndefined(); + }); + + test("pruneMissingWorkspaces does not rewrite the file when nothing is stale", async () => { + // Compact (non-pretty) serialization: any rewrite through save() would + // change the raw bytes, so byte-equality proves no write happened. + const rawContent = JSON.stringify({ + version: 1, + workspaces: { + "known-workspace": { recency: 100, streaming: false }, + }, + }); + await writeFile(filePath, rawContent); + + expect( + await service.pruneMissingWorkspaces(() => Promise.resolve(new Set(["known-workspace"]))) + ).toBe(0); + expect(await readFile(filePath, "utf-8")).toBe(rawContent); + }); + + test("pruneMissingWorkspaces spares ids re-registered mid-prune", async () => { + // With multiple instances, a downgraded backend can re-register a + // deterministic legacy id (and write its activity) between the prune's + // enumeration and its deletion pass. The recheck against a second fresh + // enumeration must spare that entry and lift its write tombstone — + // deleting on the stale classification would destroy data a later + // tombstone-clear cannot restore. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "known-workspace": { recency: 100, streaming: false }, + "revived-workspace": { recency: 200, streaming: false }, + }, + }) + ); + let fetches = 0; + const prunedCount = await service.pruneMissingWorkspaces(() => { + fetches += 1; + // First enumeration: revived-workspace looks stale. Recheck: it was + // re-registered concurrently. + return Promise.resolve( + fetches === 1 + ? new Set(["known-workspace"]) + : new Set(["known-workspace", "revived-workspace"]) + ); + }); + + expect(prunedCount).toBe(0); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(Object.keys(persisted.workspaces).sort()).toEqual([ + "known-workspace", + "revived-workspace", + ]); + // Tombstone lifted: subsequent writes for the revived id persist. + expect(service.isWorkspaceDeleted("revived-workspace")).toBe(false); + }); + + test("a newer build's metadata version is unsupported, never quarantined or self-healed", async () => { + // Downgrade safety: a syntactically valid file with version !== 1 was + // written by a newer schema. Quarantining/resetting it (or a lenient + // writer self-healing to {} and saving version-1 bytes over it) would + // make the upgrade back find an empty canonical file and lose all + // activity state. Both read modes must propagate and leave the bytes + // untouched. + const newerFile = JSON.stringify({ version: 2, workspaces: {}, futureField: true }); + await writeFile(filePath, newerFile); + + // Strict read: retryable failure, no quarantine. + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + + // Lenient write: the mutation fails instead of clobbering the file. + let writeError: unknown = null; + try { + await service.updateRecency("ws-1", 123); + } catch (error) { + writeError = error; + } + expect(writeError).not.toBeNull(); + + // Bytes untouched, no sidecar created. + expect(await readFile(filePath, "utf-8")).toBe(newerFile); + const sidecarExists = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => true, + () => false + ); + expect(sidecarExists).toBe(false); + }); + + test("a strict read resumes a crash-interrupted quarantine of corrupt bytes", async () => { + // Crash window: quarantine renamed main -> sidecar and died before + // writing the empty replacement. The next strict read must finish the + // recovery — never return an authoritative {} while the main path is + // missing, and never keep failing until an unrelated writer saves. + await writeFile(`${filePath}.corrupt`, "{not json"); + // Main file intentionally absent (beforeEach never creates it). + + // The moved bytes really were corrupt: recovery resets the main path to + // a valid empty file and keeps the bytes quarantined for inspection. + expect((await service.getAllSnapshots({ throwOnError: true })).size).toBe(0); + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toBe("{not json"); + expect(JSON.parse(await readFile(filePath, "utf-8"))).toEqual({ + version: 1, + workspaces: {}, + }); + }); + + test("an ENOENT read races a completed recovery in another process and re-reads", async () => { + // Post-recovery TOCTOU: our read hits ENOENT while another process holds + // the file mid-quarantine, and by the time we probe for the sidecar that + // process has already restored the healthy main file AND consumed the + // sidecar. The absent sidecar proves nothing about the stale ENOENT — + // load must re-read the main path instead of returning authoritative {}. + const healthy = { + version: 1, + workspaces: { "ws-1": { recency: 42, streaming: false } }, + }; + // Main file intentionally absent at first read (beforeEach never creates + // it). Simulate the other process's completed recovery at probe time. + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(service); + internals.probeQuarantineSidecar = async () => { + // The concurrent process restored the healthy file and unlinked the + // sidecar before our probe ran. + await writeFile(filePath, JSON.stringify(healthy)); + internals.probeQuarantineSidecar = originalProbe; + return false; + }; + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-1")?.recency).toBe(42); + }); + + test("a failing sidecar probe keeps a missing-main read retryable", async () => { + // Main file absent and the sidecar probe fails with EACCES: the + // sidecar's existence is unknowable, so the read must not resolve as an + // authoritative empty file — recoverable metadata may sit in the + // unprobeable sidecar. Only a verified ENOENT counts as absence. + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + internals.probeQuarantineSidecar = () => { + const error = new Error("permission denied") as NodeJS.ErrnoException; + error.code = "EACCES"; + return Promise.reject(error); + }; + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + }); + + test("a lenient writer completes a crash-interrupted quarantine instead of clobbering it", async () => { + // Same crash window as above, but hit by a normal (lenient) mutation. + // Treating it as an empty file would save a partial one-entry file at + // the main path — and the pending restore (deliberately no-overwrite) + // would then strand every other workspace's metadata in the sidecar. + // The writer must instead complete the recovery inline and apply its + // mutation on top of the restored data. + const healthy = { + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }; + await writeFile(`${filePath}.corrupt`, JSON.stringify(healthy)); + // Main file intentionally absent (beforeEach never creates it). + + await service.updateRecency("ws-new", 100); + + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as ExtensionMetadataFile; + expect(Object.keys(persisted.workspaces).sort()).toEqual(["ws-new", "ws-other"]); + // Sidecar consumed by the restore. + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("recovery restores an unsupported-version sidecar instead of resetting it", async () => { + // A newer build's file can end up in the sidecar (crash-interrupted + // quarantine on a downgraded install, or a newer backend saving between + // the corruption check and the rename). It is preserved data, not + // corruption: recovery must put it back at the main path — installing + // the empty version-1 reset would hand the newer build an empty + // canonical file and lose all its activity state. + const newerFile = JSON.stringify({ version: 2, workspaces: {}, futureField: true }); + await writeFile(`${filePath}.corrupt`, newerFile); + // Main file intentionally absent (crash window). + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + // The restored file still fails the CURRENT build's read — but with the + // non-destructive unsupported-version signal, not an empty reset. + expect(strictError).not.toBeNull(); + expect(await readFile(filePath, "utf-8")).toBe(newerFile); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("recovery reconciles a partial main file recreated during the crash window", async () => { + // An older backend can observe the missing-main window as an empty file + // and save a PARTIAL snapshot before recovery restores the sidecar. + // EEXIST is not success: the sidecar's other entries must be merged into + // the recreated file (main wins per key) instead of being abandoned. + const sidecar = { + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }; + await writeFile(`${filePath}.corrupt`, JSON.stringify(sidecar)); + // Main file intentionally absent; the probe hook below recreates it as a + // partial file mid-recovery (modeling the concurrent older backend). + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(service); + internals.probeQuarantineSidecar = async () => { + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 300, streaming: false } }, + }) + ); + internals.probeQuarantineSidecar = originalProbe; + return true; + }; + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-new")?.recency).toBe(300); + expect(snapshots.get("ws-other")?.recency).toBe(42); + // Sidecar consumed by the reconcile. + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("reconcile restores an unsupported-version sidecar over a recreated partial main", async () => { + // Downgrade-overlap variant of the reconcile: a NEWER build's schema is + // stranded in the sidecar while an older-schema backend recreates a + // partial main during the crash window. Accepting the partial file would + // lose the newer data permanently — nothing re-inspects the sidecar once + // the main path exists, and a later quarantine's rename would destroy + // it. The newer bytes must win the canonical path; the recreated file is + // preserved as its own leftover. + const newerFile = JSON.stringify({ version: 2, workspaces: {}, futureField: true }); + await writeFile(`${filePath}.corrupt`, newerFile); + // Main file intentionally absent; the probe hook recreates it as a + // partial version-1 file mid-recovery (modeling the older backend). + const partial = JSON.stringify({ + version: 1, + workspaces: { "ws-partial": { recency: 300, streaming: false } }, + }); + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(service); + internals.probeQuarantineSidecar = async () => { + await writeFile(filePath, partial); + internals.probeQuarantineSidecar = originalProbe; + return true; + }; + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + // The swapped-in file still fails the CURRENT build's read — but with + // the non-destructive unsupported-version signal, not the partial data. + expect(strictError).not.toBeNull(); + expect(await readFile(filePath, "utf-8")).toBe(newerFile); + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe(partial); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("an unreadable sidecar keeps reconcile retryable instead of accepting the partial main", async () => { + // Transient I/O failure reading the sidecar during reconcile: reporting + // success would make the read accept the recreated partial main while + // the (possibly healthy) sidecar is never inspected again. A directory + // at the sidecar path yields a deterministic errno (EISDIR) standing in + // for EACCES/EIO-class failures. + await mkdir(`${filePath}.corrupt`); + const partial = JSON.stringify({ + version: 1, + workspaces: { "ws-partial": { recency: 300, streaming: false } }, + }); + const internals = service as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(service); + internals.probeQuarantineSidecar = async () => { + await writeFile(filePath, partial); + internals.probeQuarantineSidecar = originalProbe; + return true; + }; + + let strictError: unknown = null; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + expect((strictError as NodeJS.ErrnoException).code).toBe("EISDIR"); + // Once the sidecar becomes readable (here: gone), the retry proceeds. + await rm(`${filePath}.corrupt`, { recursive: true }); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-partial")?.recency).toBe(300); + }); + + test("mutations reconcile a stranded sidecar before saving and emitting", async () => { + // Normal recency/status/goal writes save and BROADCAST their snapshot. + // With a valid partial main beside a healthy full sidecar, the mutation + // must reconcile first — otherwise it persists and emits the partial + // view, clearing goal/status in the renderer until some read recovers. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 1, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-live": { recency: 1, streaming: false }, + "ws-other": { recency: 5, streaming: false }, + }, + }) + ); + + await service.updateRecency("ws-live", 999); + + const main = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + expect(main.workspaces["ws-other"]?.recency).toBe(5); + expect(main.workspaces["ws-live"]?.recency).toBe(999); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("consumption is scoped to the sidecar generation that was reconciled", async () => { + // With multiple processes recovering the fixed .corrupt path, another + // backend can consume the generation this process read and strand a NEW + // snapshot at the same path before the removal runs. A path-only unlink + // would destroy that unreconciled generation permanently; the + // claim-then-verify consume must instead take exactly one generation off + // the shared path, detect the identity mismatch on the claimed file, and + // reconcile the foreign generation into the main file. + const quarantinePath = `${filePath}.corrupt`; + const statics = ExtensionMetadataService as unknown as { + statQuarantineToken(path: string): Promise; + }; + const internals = service as unknown as { + consumeQuarantineSidecar(path: string, token: unknown): Promise; + }; + await writeFile(filePath, JSON.stringify({ version: 1, workspaces: {} })); + await writeFile( + quarantinePath, + JSON.stringify({ version: 1, workspaces: { a: { recency: 1, streaming: false } } }) + ); + const staleToken = await statics.statQuarantineToken(quarantinePath); + // Concurrent recovery consumes that generation and quarantines a new + // snapshot at the same path (different content => different identity). + await rm(quarantinePath); + await writeFile( + quarantinePath, + JSON.stringify({ version: 1, workspaces: { b: { recency: 22222, streaming: false } } }) + ); + + await internals.consumeQuarantineSidecar(quarantinePath, staleToken); + // The newer generation was reconciled into the main file, not destroyed. + const main = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + expect(main.workspaces.b?.recency).toBe(22222); + const sidecarGone = await readFile(quarantinePath, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + // No claim file left behind (unique names; discovered by prefix). + const leftovers = (await readdir(tempDir)).filter((name) => name.includes(".corrupt-claim-")); + expect(leftovers).toEqual([]); + }); + + test("quarantine preserves an existing healthy sidecar instead of clobbering it", async () => { + // Crash strands the full snapshot at .corrupt; another backend recreates + // the main file, which later becomes corrupt too. The next strict read's + // quarantine must not rename() over the healthy sidecar (POSIX rename + // replaces silently) — recovery would then reset the canonical file to + // empty and permanently destroy every stranded entry. The corrupt main + // moves aside as the bounded fixed-name leftover instead and the healthy + // sidecar restores. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + await writeFile(filePath, "{corrupt json"); + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-stranded")?.recency).toBe(700); + // Corrupt main preserved as the bounded leftover; sidecar consumed. + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{corrupt json"); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("recovery replaces a stale .recreated leftover from an earlier pass", async () => { + // A second recovery cycle can find the fixed-name leftover already + // occupied by an earlier pass. The move-aside must replace it portably + // (unlink first — Windows rename onto an existing file is not reliably + // a replace), keeping the LATEST superseded file: a recovery that fails + // on the occupied destination would keep every strict read failing + // until the leftover was removed by hand. + await writeFile(`${filePath}.recreated`, "stale leftover from an earlier recovery"); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + await writeFile(filePath, "{corrupt json"); + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-stranded")?.recency).toBe(700); + // The leftover now holds the LATEST superseded file, not the stale one. + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{corrupt json"); + }); + + test("a healthy main raced into the quarantine move-aside is restored and merged, not lost", async () => { + // Multi-instance race: the in-queue corruption check sees a corrupt main + // and an existing healthy sidecar, then another backend's atomic save + // lands a NEWER healthy main before the move-aside. Nothing ever reads + // the .recreated leftover, so without revalidating the moved bytes the + // newer update would be silently lost while the OLDER sidecar restores. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-old": { recency: 700, streaming: false } }, + }) + ); + await writeFile(filePath, "{corrupt json"); + const internals = service as unknown as { + moveMainAsideAsRecreatedLeftover: () => Promise; + }; + const realMove = internals.moveMainAsideAsRecreatedLeftover.bind(service); + internals.moveMainAsideAsRecreatedLeftover = async () => { + // The concurrent backend's save landing inside the race window. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 900, streaming: false } }, + }) + ); + internals.moveMainAsideAsRecreatedLeftover = realMove; + return realMove(); + }; + try { + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-new")?.recency).toBe(900); + expect(snapshots.get("ws-old")?.recency).toBe(700); + } finally { + internals.moveMainAsideAsRecreatedLeftover = realMove; + } + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + // The restored bytes were never superseded: the in-flight moved-aside + // file is dropped after the restore rather than finalized (or stranded) + // as a leftover. + const strayLeftovers = (await readdir(tempDir)).filter((name) => name.includes(".recreated")); + expect(strayLeftovers).toEqual([]); + }); + + test("a crash-stranded in-flight moved-aside main is recovered, not orphaned", async () => { + // Another backend crashed mid-recovery after moving a raced HEALTHY + // main aside under its unique in-flight name (the rename is the commit + // point; revalidation happens after it). Unique names are invisible to + // every fixed-path probe, so without the stranded-leftover scan the + // newer snapshot would be orphaned forever while the older sidecar + // restores. The scan must merge it back instead. + const foreignInflight = `${filePath}.recreated-99999-f0e1d2c3`; + await writeFile( + foreignInflight, + JSON.stringify({ + version: 1, + workspaces: { "ws-newer": { recency: 900, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-old": { recency: 100, streaming: false } }, + }) + ); + await writeFile(filePath, "{corrupt json"); + const healed = await service.getAllSnapshots({ throwOnError: true }); + // Sidecar restored AND the stranded newer snapshot merged on top. + expect(healed.get("ws-old")?.recency).toBe(100); + expect(healed.get("ws-newer")?.recency).toBe(900); + // The corrupt main finalized as the bounded fixed-name leftover; the + // stranded unique file was consumed, not left to accumulate. + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{corrupt json"); + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("stranded leftover entries merge by strictly newer recency with streaming cleared", async () => { + // The stranded bytes are a complete main snapshot of unknowable age: + // per-entry recency orders the copies, a stale duplicate must never + // overwrite newer main data (this also makes crash replay a no-op), + // and a truthy streaming flag from a crashed process must not pin the + // workspace "streaming" forever. + await service.updateRecency("ws-current", 500); + await writeFile( + `${filePath}.recreated-4242-cafebabe`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-current": { recency: 300, streaming: false }, + "ws-imported": { recency: 900, streaming: true }, + }, + }) + ); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-current")?.recency).toBe(500); + expect(snapshots.get("ws-imported")?.recency).toBe(900); + expect(snapshots.get("ws-imported")?.streaming).toBe(false); + // Consumed cleanly: no stranded unique file and no garbage finalized to + // the fixed leftover name. + const leftovers = (await readdir(tempDir)).filter((name) => name.includes(".recreated")); + expect(leftovers).toEqual([]); + }); + + test("a raced healthy main losing the restore collision is merged, not parked", async () => { + // EEXIST during the restore does not prove the main-path owner is + // newer: a competing recovery can restore the OLDER sidecar to the + // vacant main path first and consume the sidecar. The valid moved + // bytes must stay at their unique in-flight name so the + // stranded-leftover scan merges them — finalizing them to the fixed + // (unscanned) leftover would silently drop the newer update. + const sidecarPath = `${filePath}.corrupt`; + const olderRestored = JSON.stringify({ + version: 1, + workspaces: { "ws-old": { recency: 100, streaming: false } }, + }); + await writeFile(sidecarPath, olderRestored); + await writeFile(filePath, "{corrupt json"); + const internals = service as unknown as { + moveMainAsideAsRecreatedLeftover: () => Promise; + }; + const realMove = internals.moveMainAsideAsRecreatedLeftover.bind(service); + internals.moveMainAsideAsRecreatedLeftover = async () => { + // The raced newer save lands before the move-aside… + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 900, streaming: false } }, + }) + ); + internals.moveMainAsideAsRecreatedLeftover = realMove; + const inflightPath = await realMove(); + // …and the competing recovery restores the older sidecar to the + // vacant main path and consumes it before this recovery's restore. + await writeFile(filePath, olderRestored); + await rm(sidecarPath); + return inflightPath; + }; + try { + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-old")?.recency).toBe(100); + expect(snapshots.get("ws-new")?.recency).toBe(900); + } finally { + internals.moveMainAsideAsRecreatedLeftover = realMove; + } + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("an unsupported sidecar never displaces a same-or-newer unsupported canonical file", async () => { + // Multi-version overlap: a v3 writer re-created the canonical file + // while an older v2 sidecar remained. A v1 build cannot order or merge + // foreign schemas, so it must keep the canonical copy in place and + // retain the sidecar for a build that understands both — not park v3 + // at the unscanned fixed leftover and restore older v2 data over it. + const v3Main = JSON.stringify({ version: 3, workspaces: {}, futureField: true }); + const v2Sidecar = JSON.stringify({ version: 2, workspaces: {} }); + await writeFile(filePath, v3Main); + await writeFile(`${filePath}.corrupt`, v2Sidecar); + await service.getSnapshot("any", { throwOnError: true }).catch(() => null); + expect(await readFile(filePath, "utf-8")).toBe(v3Main); + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toBe(v2Sidecar); + const leftovers = (await readdir(tempDir)).filter((name) => name.includes(".recreated")); + expect(leftovers).toEqual([]); + }); + + test("a strictly newer unsupported sidecar still replaces an older unsupported canonical file", async () => { + // Guard for the inverse direction: the version comparison must not + // block the legitimate upgrade-preservation swap when the sidecar + // schema is strictly newer than the canonical one. + const v2Main = JSON.stringify({ version: 2, workspaces: {} }); + const v3Sidecar = JSON.stringify({ version: 3, workspaces: {} }); + await writeFile(filePath, v2Main); + await writeFile(`${filePath}.corrupt`, v3Sidecar); + await service.getSnapshot("any", { throwOnError: true }).catch(() => null); + expect(await readFile(filePath, "utf-8")).toBe(v3Sidecar); + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe(v2Main); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("stranded leftover merge orders by write generation, not recency alone", async () => { + // recency is a user-interaction timestamp that status/goal writers + // deliberately preserve: a newer status write captured in the stranded + // copy can share its recency with the older restored main and must + // still win via the per-entry write generation — and the reverse copy + // (older generation, equal recency) must lose. + const entry = (writeGeneration: number, message: string) => ({ + recency: 500, + streaming: false, + writeGeneration, + lastModel: null, + lastThinkingLevel: null, + agentStatus: null, + lastStatusUrl: null, + todoStatus: { emoji: "s", message }, + }); + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "ws-status": entry(3, "old status"), + "ws-keep": entry(5, "current status"), + }, + }) + ); + await writeFile( + `${filePath}.recreated-31337-0ddba11`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-status": entry(7, "newer status"), + "ws-keep": entry(2, "stale status"), + }, + }) + ); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-status")?.todoStatus?.message).toBe("newer status"); + expect(snapshots.get("ws-keep")?.todoStatus?.message).toBe("current status"); + }); + + test("equal-recency stranded entries from generation-less builds are preserved", async () => { + // A downgraded build's writers drop writeGeneration when mutating an + // entry, and its later goal/status write can share recency with an + // older generation-carrying copy restored by recovery. When the + // generation-carrying stamp does NOT postdate the stranded file's + // mtime (here: legacy counter-sized stamps far below any mtime), the + // order is unknowable and the generation-less side must win in BOTH + // positions — as the stranded candidate (or the downgrade's update is + // permanently lost when the claim is consumed) and as the target (a + // stale generation-carrying stranded copy must not displace it). + const entry = (message: string, writeGeneration?: number) => ({ + recency: 500, + streaming: false, + ...(writeGeneration !== undefined ? { writeGeneration } : {}), + lastModel: null, + lastThinkingLevel: null, + agentStatus: null, + lastStatusUrl: null, + todoStatus: { emoji: "s", message }, + }); + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "ws-downgrade": entry("old status", 4), + "ws-target": entry("downgrade status"), + }, + }) + ); + await writeFile( + `${filePath}.recreated-2468-beefcafe`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-downgrade": entry("downgrade newer status"), + "ws-target": entry("stale status", 9), + }, + }) + ); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-downgrade")?.todoStatus?.message).toBe("downgrade newer status"); + expect(snapshots.get("ws-target")?.todoStatus?.message).toBe("downgrade status"); + }); + + test("an equal-recency generation-less stranded copy does not revert a later local write", async () => { + // The decidable side of the equal-recency tie: a pre-generation main + // (no writeGeneration anywhere — e.g. written before the upgrade to + // this build) is stranded, and THIS build then mutates the entry's + // goal/status, preserving recency but stamping an epoch-ms + // writeGeneration that postdates the stranded file's mtime. The merge + // must keep the newer local write — no later mutation is guaranteed to + // repair a wrong overwrite — instead of blindly preferring the + // generation-less candidate. + await service.updateRecency("ws-fresh", 500); + await service.setTodoStatus("ws-fresh", { emoji: "s", message: "current status" }, true); + const strandedPath = `${filePath}.recreated-8642-deadbea7`; + await writeFile( + strandedPath, + JSON.stringify({ + version: 1, + workspaces: { + "ws-fresh": { + recency: 500, + streaming: false, + todoStatus: { emoji: "s", message: "stale pre-upgrade status" }, + }, + }, + }) + ); + // Backdate the stranded file's mtime below the mutation stamps above + // (rename preserves mtime in production; writeFile here stamps "now", + // which could tie with the mutation's stamp at ms granularity). + const past = new Date(Date.now() - 60_000); + await utimes(strandedPath, past, past); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-fresh")?.todoStatus?.message).toBe("current status"); + // Consumed, not retried forever. + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("stranded adoption samples registration strictly after the main read", async () => { + // The post-load evidence rule: a positive probe captured BEFORE the + // main read can go stale when another backend deregisters the + // workspace during the probe awaits — adopting on it would resurrect + // the removed workspace's metadata. Modeled with a probe whose answer + // flips between the tombstone-revalidation sample (registered: lifts + // the tombstone) and the adoption sample (deregistered meanwhile): + // only a post-load adoption sample sees the flip. + await service.updateRecency("ws-other", 1); + service.suppressForeignRemoval("ws-x"); + await writeFile( + `${filePath}.recreated-4242-cafe`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-x": { + recency: 500, + streaming: false, + todoStatus: { emoji: "s", message: "resurrected status" }, + }, + }, + }) + ); + let probeCalls = 0; + service.setRegistrationProbe(() => { + probeCalls += 1; + return Promise.resolve(probeCalls === 1); + }); + const snapshots = await service.getAllSnapshots(); + // The first (registration) sample lifted the tombstone… + expect(service.isWorkspaceDeleted("ws-x")).toBe(false); + // …but the adoption decision used a fresh post-load sample and saw the + // deregistration: proven removed, not resurrected, claim consumed. + expect(snapshots.has("ws-x")).toBe(false); + expect(snapshots.get("ws-other")?.recency).toBe(1); + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("a crash-stranded empty-recovery temp file is inert", async () => { + // Empty-file quarantine recovery writes through a process-unique + // `.empty--.tmp` (a shared fixed name lets concurrent + // recoveries truncate or unlink each other's in-flight temp). A crash + // strands at most one such file, and no scan may consume it — sweeping + // another process's in-flight temp is the same race the unique name + // prevents. + await service.updateRecency("ws-live", 7); + const staleTmp = `${filePath}.empty-12345-dead.tmp`; + await writeFile(staleTmp, "{}"); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-live")?.recency).toBe(7); + expect(await readFile(staleTmp, "utf-8")).toBe("{}"); + }); + + test("persisted mutations advance the per-entry write generation", async () => { + // The durable ordering contract behind the merge above: metadata + // mutations advance the generation even when they preserve recency (so + // cross-copy ordering never regresses to the recency tiebreak for + // entries written by this build), and the stamp is wall-clock epoch-ms + // so recovery can order it against a stranded file's mtime. + const readGeneration = async () => { + const raw = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + return raw.workspaces["ws-gen"]; + }; + const before = Date.now(); + await service.updateRecency("ws-gen", 100); + const first = await readGeneration(); + await service.setTodoStatus("ws-gen", { emoji: "s", message: "working" }, true); + const second = await readGeneration(); + expect(first.writeGeneration).toBeGreaterThanOrEqual(before); + expect(second.writeGeneration).toBeGreaterThan(first.writeGeneration ?? 0); + expect(second.recency).toBe(100); + }); + + test("stranded entries for foreign-removed workspaces are not resurrected", async () => { + // The stranded snapshot predates the current main: an entry missing + // from the main may have been REMOVED by another backend after the + // file was stranded (local tombstones cannot know). Adoption requires + // registration evidence — a probed-unregistered id is dropped instead + // of resurrected, while registered ids still recover. + service.setRegistrationProbe((workspaceId) => Promise.resolve(workspaceId !== "ws-removed")); + await service.updateRecency("ws-live", 100); + await writeFile( + `${filePath}.recreated-1234-abcd12`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-removed": { recency: 900, streaming: false }, + "ws-created": { recency: 800, streaming: false }, + }, + }) + ); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.has("ws-removed")).toBe(false); + expect(snapshots.get("ws-created")?.recency).toBe(800); + expect(snapshots.get("ws-live")?.recency).toBe(100); + // Consumed on evidence, not left for an endless retry loop. + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("a corrupt stranded in-flight leftover is finalized, not merged", async () => { + // Corrupt stranded bytes are exactly the validated-corrupt main their + // owner moved aside: the scan keeps them as the bounded fixed-name + // leftover (the owner's own terminal state) instead of merging garbage + // or leaving unique files to accumulate. + await service.updateRecency("ws-live", 100); + await writeFile(`${filePath}.recreated-777-feedface`, "{not json"); + const snapshots = await service.getAllSnapshots(); + expect(snapshots.get("ws-live")?.recency).toBe(100); + expect(await readFile(`${filePath}.recreated`, "utf-8")).toBe("{not json"); + const stranded = (await readdir(tempDir)).filter((name) => name.includes(".recreated-")); + expect(stranded).toEqual([]); + }); + + test("a failing sidecar token probe during consumption propagates instead of reporting success", async () => { + // consumeQuarantineSidecar's post-merge stat() guards which sidecar + // generation gets unlinked. If a transient EACCES/EIO there were + // swallowed as success, the caller would proceed — e.g. the one-time + // prune deletes stale entries, the NEXT snapshot read re-merges them + // from the retained sidecar and consumes it, and with the prune latch + // already set the resurrected entries stay indefinitely. It must + // propagate (retryable) instead. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + const statics = ExtensionMetadataService as unknown as { + statQuarantineToken: (quarantinePath: string) => Promise; + }; + const realStat = statics.statQuarantineToken.bind(ExtensionMetadataService); + // Call #1 captures the reconcile's generation token, call #2 binds the + // token to the read bytes (both real); call #3 is the consumption-time + // identity probe this test degrades. + let statCalls = 0; + statics.statQuarantineToken = async (quarantinePath: string) => { + statCalls += 1; + if (statCalls === 3) { + const error = new Error("EACCES: permission denied, stat") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + } + return realStat(quarantinePath); + }; + try { + let strictRejected = false; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch { + strictRejected = true; + } + expect(strictRejected).toBe(true); + // Bytes retained for the retry: the consume claimed the sidecar before + // the failing identity probe, so they now sit at a unique claim name + // and are never deleted on unverifiable identity. + const claims = (await readdir(tempDir)).filter((name) => name.includes(".corrupt-claim-")); + expect(claims.length).toBe(1); + expect(await readFile(path.join(tempDir, claims[0]), "utf-8")).toContain("ws-stranded"); + } finally { + statics.statQuarantineToken = realStat; + } + // Retry with the probe healthy: the stranded-claim discovery re-merges + // idempotently and consumption completes. + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-stranded")?.recency).toBe(700); + expect(snapshots.get("ws-live")?.recency).toBe(900); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + expect((await readdir(tempDir)).filter((name) => name.includes(".corrupt-claim-"))).toEqual([]); + }); + + test("stranded claims whose bytes match their embedded token are deleted without replay", async () => { + // A crash between a matched claim's rename and its unlink strands a + // claim whose file identity equals the token embedded in its name — + // proof its bytes were already merged into the main file before the + // consume began. Replaying the merge would re-fill fields another + // backend explicitly cleared to null since (the null-fill merge is not + // idempotent across clears) and resurrect entries pruning reclaimed. + // Discovery must delete it, never merge it. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + const strandedTmp = `${filePath}.stranded-tmp`; + await writeFile( + strandedTmp, + JSON.stringify({ + version: 1, + workspaces: { "ws-reclaimed": { recency: 700, streaming: false } }, + }) + ); + const token = await ( + ExtensionMetadataService as unknown as { + statQuarantineToken(p: string): Promise<{ ino: bigint; mtimeNs: bigint; size: bigint }>; + } + ).statQuarantineToken(strandedTmp); + await rename( + strandedTmp, + `${filePath}.corrupt-claim-${token.ino}-${token.mtimeNs}-${token.size}-123-stranded` + ); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.has("ws-reclaimed")).toBe(false); + expect(snapshots.get("ws-live")?.recency).toBe(900); + expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-claim-"))).toEqual([]); + }); + + test("a retained corrupt sidecar does not starve stranded claim recovery", async () => { + // A deterministically corrupt sidecar is intentionally retained by its + // reconcile. If the leftover pass returned early on it, a stranded + // mismatched claim (unreconciled foreign generation) would never be + // discovered — its recency/goal/status hidden indefinitely. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + await writeFile(`${filePath}.corrupt`, "{corrupt sidecar"); + await writeFile( + `${filePath}.corrupt-claim-99-99-99-123-foreign`, + JSON.stringify({ + version: 1, + workspaces: { "ws-claim": { recency: 700, streaming: false } }, + }) + ); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-claim")?.recency).toBe(700); + expect(snapshots.get("ws-live")?.recency).toBe(900); + // Corrupt sidecar intentionally retained (for inspection); claim consumed. + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toBe("{corrupt sidecar"); + expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-claim-"))).toEqual([]); + }); + + test("a sidecar swapped between stat and read is not merged under the stale token", async () => { + // Another recovery consumes the captured generation and installs a NEW + // one at the fixed path between this reconcile's stat and read. Merging + // the new bytes under the old token would double-apply them (the + // consume claims the new generation, sees the mismatch, and replays), + // re-filling fields another backend explicitly cleared to null in + // between. The post-read binding must abort instead. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + const statics = ExtensionMetadataService as unknown as { + statQuarantineToken(p: string): Promise; + }; + const realStat = statics.statQuarantineToken.bind(ExtensionMetadataService); + let statCalls = 0; + statics.statQuarantineToken = async (p: string) => { + statCalls += 1; + if (statCalls === 1) { + // The capture observed a generation that was then consumed by + // another recovery, which installed the CURRENT file before our + // read. + return { ino: 1n, mtimeNs: 2n, size: 3n }; + } + return realStat(p); + }; + try { + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + // Not merged under the stale token; left for its own recovery pass. + expect(snapshots.has("ws-stranded")).toBe(false); + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toContain("ws-stranded"); + } finally { + statics.statQuarantineToken = realStat; + } + const healed = await service.getAllSnapshots({ throwOnError: true }); + expect(healed.get("ws-stranded")?.recency).toBe(700); + }); + + test("deleteWorkspace reconciles a stranded sidecar so the removed entry cannot be resurrected", async () => { + // Full sidecar beside a recreated partial main: deleting from the + // partial main alone would leave the removed workspace's complete entry + // in the sidecar, where a concurrent backend without this + // process-local tombstone could reconcile it back onto disk (visible + // immediately on unscoped builds; inherited as stale goal/status by a + // deterministic legacy-id re-registration). + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 300, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-other": { recency: 300, streaming: false }, + "ws-removed": { recency: 700, streaming: false }, + }, + }) + ); + service.setRegistrationProbe(() => Promise.resolve(false)); + await service.deleteWorkspace("ws-removed"); + // Sidecar consumed (tombstoned entry skipped by the merge); the removed + // entry survives on NEITHER file. + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + expect("ws-removed" in persisted.workspaces).toBe(false); + expect("ws-other" in persisted.workspaces).toBe(true); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("a sidecar swapped during the missing-main restore is not restored under the stale token", async () => { + // Same stat-to-read race as the recreated-main reconcile, on the + // missing-main restore path (completeQuarantineRecovery): another + // recovery consumes the captured generation and installs a newer one + // before the read. Restoring under the stale token would double-apply + // the bytes through the consume-side mismatch replay. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-stranded": { recency: 700, streaming: false } }, + }) + ); + const statics = ExtensionMetadataService as unknown as { + statQuarantineToken(p: string): Promise; + }; + const realStat = statics.statQuarantineToken.bind(ExtensionMetadataService); + let statCalls = 0; + statics.statQuarantineToken = async (p: string) => { + statCalls += 1; + if (statCalls === 1) { + return { ino: 1n, mtimeNs: 2n, size: 3n }; + } + return realStat(p); + }; + try { + let strictRejected = false; + try { + await service.getAllSnapshots({ throwOnError: true }); + } catch { + strictRejected = true; + } + // Not restored under the stale token: the read stays retryable with + // the sidecar retained for the next pass. + expect(strictRejected).toBe(true); + expect(await readFile(`${filePath}.corrupt`, "utf-8")).toContain("ws-stranded"); + } finally { + statics.statQuarantineToken = realStat; + } + const healed = await service.getAllSnapshots({ throwOnError: true }); + expect(healed.get("ws-stranded")?.recency).toBe(700); + }); + + test("stranded claims that do not match their embedded token are replayed", async () => { + // Crash between a MISMATCH claim (foreign generation another backend + // installed at the sidecar path) and its reconcile: nobody merged + // those bytes, so discovery must replay them into the main file. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-live": { recency: 900, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt-claim-99-99-99-123-foreign`, + JSON.stringify({ + version: 1, + workspaces: { "ws-foreign": { recency: 700, streaming: false } }, + }) + ); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-foreign")?.recency).toBe(700); + expect(snapshots.get("ws-live")?.recency).toBe(900); + expect((await readdir(tempDir)).filter((n) => n.includes(".corrupt-claim-"))).toEqual([]); + }); + + test("a strict per-workspace read self-heals deterministic corruption", async () => { + // Live emissions read per-workspace snapshots after the subscription + // bootstraps. If the file becomes deterministically corrupt afterwards, + // the workflow-run/bash-monitor handlers drop their emissions on error — + // and nothing else is guaranteed to repair the file on an idle process, + // pinning stale activity in the renderer indefinitely. The strict read + // must route corruption through the same quarantine-and-reread recovery + // getAllSnapshots uses instead of failing every retry forever. + await writeFile(filePath, "{corrupt json"); + const snapshot = await service.getSnapshot("ws-any", { throwOnError: true }); + expect(snapshot).toBeNull(); + // The canonical file was quarantined and reset to a valid empty file. + const healed = JSON.parse(await readFile(filePath, "utf-8")) as { version?: number }; + expect(healed.version).toBe(1); + }); + + test("a strict snapshot read propagates a failed sidecar reconcile instead of the partial main", async () => { + // Live emissions read per-workspace snapshots after the subscription + // bootstraps. With a sidecar stranded next to a recreated partial main + // and the reconcile failing transiently, emitting the partial main + // would clear goal/status in the renderer with no guaranteed + // strict-list retry — strict readers (the emit paths) must skip the + // emit by propagating, while lenient readers (settings/eligibility) + // keep availability. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-partial": { recency: 300, streaming: false } }, + }) + ); + // A directory at the sidecar path yields a deterministic errno (EISDIR) + // standing in for EACCES/EIO-class reconcile failures. + await mkdir(`${filePath}.corrupt`); + + let strictError: unknown = null; + try { + await service.getSnapshot("ws-partial", { throwOnError: true }); + } catch (error) { + strictError = error; + } + expect((strictError as NodeJS.ErrnoException | null)?.code).toBe("EISDIR"); + // Lenient readers keep availability on the same state. + expect((await service.getSnapshot("ws-partial"))?.recency).toBe(300); + // Once the sidecar becomes readable (here: gone), the strict read heals. + await rm(`${filePath}.corrupt`, { recursive: true }); + expect((await service.getSnapshot("ws-partial", { throwOnError: true }))?.recency).toBe(300); + }); + + test("a strict read reconciles a leftover sidecar next to a recreated valid main", async () => { + // Crash between quarantine's rename and its completion, then another + // backend recreates a VALID partial main from the missing-main window + // before this process starts: no read ever sees ENOENT or corruption, + // so without the once-per-process sidecar check the full snapshot would + // stay stranded in the sidecar forever. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 300, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }) + ); + // Fresh instance: models the restarted process. + const restarted = new ExtensionMetadataService(filePath); + + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-new")?.recency).toBe(300); + expect(snapshots.get("ws-other")?.recency).toBe(42); + // Sidecar consumed by the reconcile. + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("reconcile preserves sidecar fields a partial recreated entry did not supersede", async () => { + // The recreated main entry is commonly a partial self-heal (a recency + // write initializes every other field to its default). Key-level + // main-wins would discard the sidecar entry's model/goal/unknown fields; + // the merge must be per field. Crash-stranded streaming flags must not + // leak back either: initialize()'s stale-streaming cleanup ran against + // the main file before this reconcile. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { + "ws-1": { recency: 300, streaming: false, lastModel: null, lastThinkingLevel: null }, + }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-1": { + recency: 42, + streaming: true, + lastModel: "anthropic:claude", + lastThinkingLevel: "high", + futureField: { fromNewerBuild: true }, + }, + "ws-2": { recency: 7, streaming: true, lastModel: null, lastThinkingLevel: null }, + }, + }) + ); + const restarted = new ExtensionMetadataService(filePath); + + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-1")?.recency).toBe(300); + expect(snapshots.get("ws-2")?.recency).toBe(7); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record>; + }; + // Main's newer write wins the fields it actually carries... + expect(persisted.workspaces["ws-1"]?.recency).toBe(300); + // ...while unaffected sidecar fields (including unknown newer-build + // fields) survive the reconcile instead of being discarded. + expect(persisted.workspaces["ws-1"]?.lastModel).toBe("anthropic:claude"); + expect(persisted.workspaces["ws-1"]?.lastThinkingLevel).toBe("high"); + expect(persisted.workspaces["ws-1"]?.futureField).toEqual({ fromNewerBuild: true }); + // Crash-leftover streaming flags never come back from the sidecar. + expect(persisted.workspaces["ws-1"]?.streaming).toBe(false); + expect(persisted.workspaces["ws-2"]?.streaming).toBe(false); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("a malformed version value quarantines instead of masquerading as a newer schema", async () => { + // Only a structurally plausible forward version (integer > 1) earns + // non-destructive preservation. Corruption that mangles the version + // field (null/string/object, or a numeric no schema lineage can produce: + // 0/-1/2.5) must quarantine and self-heal — preserving it would fail + // every read and write forever on a file no build can read. + for (const version of [null, "two", {}, 0, -1, 2.5]) { + await writeFile( + filePath, + JSON.stringify({ version, workspaces: { "ws-1": { recency: 1, streaming: false } } }) + ); + const restarted = new ExtensionMetadataService(filePath); + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + // Quarantined to empty (bytes preserved in the sidecar), not stuck. + expect(snapshots.size).toBe(0); + await rm(`${filePath}.corrupt`, { force: true }); + } + // A plausible forward version stays preserved (non-destructive signal). + const newerFile = JSON.stringify({ version: 2, workspaces: {} }); + await writeFile(filePath, newerFile); + const restarted = new ExtensionMetadataService(filePath); + let strictError: unknown = null; + try { + await restarted.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect(strictError).not.toBeNull(); + expect(await readFile(filePath, "utf-8")).toBe(newerFile); + }); + + test("a write proceeds when another path lifts the tombstone mid-probe", async () => { + await service.updateRecency("ws-1", 100); + await service.deleteWorkspace("ws-1"); + let resolveProbe: ((registered: boolean) => void) | null = null; + service.setRegistrationProbe( + () => + new Promise((resolve) => { + resolveProbe = resolve; + }) + ); + const write = service.updateRecency("ws-1", 200); + expect(resolveProbe).not.toBeNull(); + // The activity bootstrap clears the (re-registered) tombstone while the + // write's probe is still in flight. + service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), service.getTombstonedIds()); + expect(service.isWorkspaceDeleted("ws-1")).toBe(false); + resolveProbe!(true); + await write; + // The write must have PERSISTED: with the tombstone gone, the caller's + // snapshot is broadcast, and a transient (unpersisted) result would + // leave renderer state ahead of disk until restart. + expect((await service.getAllSnapshots()).get("ws-1")?.recency).toBe(200); + }); + + test("reconcile restores a healthy sidecar entry over an uncoercible main entry", async () => { + // The recreated main can carry a malformed value (null/primitive/array) + // under the same key as a healthy sidecar entry: the field merge cannot + // repair it, and the sidecar is consumed — leaving it would permanently + // lose the only valid copy. + await writeFile( + filePath, + JSON.stringify({ version: 1, workspaces: { "ws-1": null, "ws-2": 7 } }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + "ws-1": { recency: 42, streaming: true, lastModel: "m", lastThinkingLevel: null }, + "ws-2": { recency: 43, streaming: false, lastModel: null, lastThinkingLevel: null }, + }, + }) + ); + const restarted = new ExtensionMetadataService(filePath); + + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-1")?.recency).toBe(42); + expect(snapshots.get("ws-2")?.recency).toBe(43); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record>; + }; + expect(persisted.workspaces["ws-1"]?.lastModel).toBe("m"); + // Crash-leftover streaming stays cleared on restore. + expect(persisted.workspaces["ws-1"]?.streaming).toBe(false); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("prune's re-registration spare keeps tombstones republished mid-recheck", async () => { + // The recheck enumeration awaits disk; a same-process removal landing in + // that window republishes the tombstone with a newer generation. The + // enumeration's pre-removal positive must clear only the prune's own + // tombstone — clearing the newer one would let a late writer recreate + // the removed entry right after the removal's queued deletion. + await service.updateRecency("ws-stale", 100); + let removal: Promise | null = null; + await service.pruneMissingWorkspaces( + () => Promise.resolve(new Set()), + () => { + // Removed again in this process while the recheck awaited: + // deleteWorkspace publishes its newer tombstone synchronously and + // queues the deletion behind the running prune. + removal = service.deleteWorkspace("ws-stale"); + // The (stale) enumeration still reports the id registered. + return Promise.resolve(new Set(["ws-stale"])); + } + ); + await removal!; + expect(service.isWorkspaceDeleted("ws-stale")).toBe(true); + // A late writer stays suppressed instead of recreating the entry. + await service.updateRecency("ws-stale", 200); + expect((await service.getAllSnapshots()).has("ws-stale")).toBe(false); + }); + + test("deleteWorkspace revalidates registration inside the queued deletion", async () => { + await service.updateRecency("ws-legacy", 100); + // Probe models a downgraded backend re-registering the id between the + // caller's deregistration checks and the queued deletion: the fresh + // snapshot must survive and the tombstone lift. + service.setRegistrationProbe(() => Promise.resolve(true)); + await service.deleteWorkspace("ws-legacy"); + expect((await service.getAllSnapshots()).get("ws-legacy")?.recency).toBe(100); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(false); + // An unknowable probe keeps the tombstone but never deletes on lossy + // evidence: the entry stays on disk (recoverable) while writes and + // lists are suppressed. + service.setRegistrationProbe(() => Promise.reject(new Error("io"))); + await service.deleteWorkspace("ws-legacy"); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(true); + const persisted = JSON.parse(await readFile(filePath, "utf-8")) as { + workspaces: Record; + }; + expect("ws-legacy" in persisted.workspaces).toBe(true); + // A verified-unregistered id deletes as before (the normal removal). + service.setRegistrationProbe(() => Promise.resolve(false)); + await service.deleteWorkspace("ws-legacy"); + expect((await service.getAllSnapshots()).has("ws-legacy")).toBe(false); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(true); + }); + + test("a write persists when the tombstone is lifted mid-probe despite a negative probe", async () => { + // The registration probe and a concurrent activity bootstrap race: the + // bootstrap proves re-registration and lifts the tombstone while this + // write's own probe resolves negative (stale evidence) or fails. Once + // the tombstone is gone, broadcasts are un-suppressed — suppressing + // only this in-flight write would broadcast an unpersisted transient + // snapshot, leaving renderer state ahead of disk until restart. The + // write must persist. + await service.updateRecency("ws-revived", 100); + service.setRegistrationProbe(() => Promise.resolve(false)); + await service.deleteWorkspace("ws-revived"); + expect(service.isWorkspaceDeleted("ws-revived")).toBe(true); + service.setRegistrationProbe(() => { + // Concurrent bootstrap lifts the tombstone while the probe is in + // flight, on registration evidence of its own. + service.clearTombstonesForRegisteredIds(new Set(["ws-revived"]), service.getTombstonedIds()); + return Promise.resolve(false); // This probe's own (stale) negative outcome. + }); + const snapshot = await service.updateRecency("ws-revived", 500); + expect(snapshot.recency).toBe(500); + expect((await service.getAllSnapshots()).get("ws-revived")?.recency).toBe(500); + expect(service.isWorkspaceDeleted("ws-revived")).toBe(false); + }); + + test("live snapshot reads reconcile a stranded sidecar too", async () => { + // After the activity subscription bootstraps, live metadata/workflow + // emissions read via getSnapshot — a healthy subscription never issues + // another list read, so getSnapshot must run the same leftover-sidecar + // reconcile or a recreated partial main would feed emitted snapshots + // indefinitely. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 300, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }) + ); + const restarted = new ExtensionMetadataService(filePath); + + const snapshot = await restarted.getSnapshot("ws-other"); + expect(snapshot?.recency).toBe(42); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("a strict read reconciles a sidecar stranded after earlier successful reads", async () => { + // Quarantines are cross-process: another backend can crash mid-quarantine + // (stranding a healthy sidecar) at any point in this process's lifetime, + // not just before startup. A process-lifetime latch would hide that + // sidecar until restart; the probe must run on every authoritative read. + await service.updateRecency("ws-main", 100); + expect((await service.getAllSnapshots({ throwOnError: true })).has("ws-main")).toBe(true); + // Stranded AFTER the first successful read. + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }) + ); + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-main")?.recency).toBe(100); + expect(snapshots.get("ws-other")?.recency).toBe(42); + const sidecarGone = await readFile(`${filePath}.corrupt`, "utf-8").then( + () => false, + () => true + ); + expect(sidecarGone).toBe(true); + }); + + test("reconcile revalidates tombstoned sidecar entries against registration", async () => { + // A tombstone may be stale (id re-registered by a downgraded backend) + // and the sidecar holds that workspace's ONLY copy — dropping is + // permanent, unlike write suppression. With the probe denying + // registration the entry is dropped; with the probe confirming it, the + // entry is merged and the tombstone lifted. + await service.updateRecency("ws-legacy", 100); + await service.deleteWorkspace("ws-legacy"); + let registered = false; + service.setRegistrationProbe(() => Promise.resolve(registered)); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-legacy": { recency: 42, streaming: false } }, + }) + ); + expect((await service.getAllSnapshots({ throwOnError: true })).has("ws-legacy")).toBe(false); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(true); + // Re-registered: a re-stranded sidecar entry must survive the reconcile. + registered = true; + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-legacy": { recency: 43, streaming: false } }, + }) + ); + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-legacy")?.recency).toBe(43); + expect(service.isWorkspaceDeleted("ws-legacy")).toBe(false); + }); + + test("a tombstone published during the registration probe survives a stale positive", async () => { + await service.updateRecency("ws-1", 100); + await service.deleteWorkspace("ws-1"); + let resolveProbe: ((registered: boolean) => void) | null = null; + let probeCalls = 0; + service.setRegistrationProbe(() => { + probeCalls += 1; + if (probeCalls === 1) { + // The write's probe: parked until the test resolves it below. + return new Promise((resolve) => { + resolveProbe = resolve; + }); + } + // The removal's own in-queue revalidation reads FRESH config + // (deregistered), unlike the write's stale parked probe. + return Promise.resolve(false); + }); + // The write parks on the probe (invoked synchronously up to the await). + const write = service.updateRecency("ws-1", 200); + expect(resolveProbe).not.toBeNull(); + // While the probe is in flight, the workspace is removed AGAIN: a newer + // tombstone generation is published. The probe's positive answer is + // stale (it read config before the deregistration) and must not clear + // the newer tombstone — otherwise the parked write recreates the entry + // right after the removal's queued deletion. + await service.deleteWorkspace("ws-1"); + resolveProbe!(true); + await write; + expect(service.isWorkspaceDeleted("ws-1")).toBe(true); + expect((await service.getAllSnapshots()).has("ws-1")).toBe(false); + }); + + test("an unprobeable sidecar during resumed recovery keeps the strict read retryable", async () => { + // The per-read leftover check found the sidecar, but the re-probe + // INSIDE the queued recovery transiently fails (EACCES/EIO class): + // reporting it absent would silently accept the recreated partial main. + // The failure must propagate (retryable) so a later read reconciles. + await writeFile( + filePath, + JSON.stringify({ + version: 1, + workspaces: { "ws-new": { recency: 300, streaming: false } }, + }) + ); + await writeFile( + `${filePath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { "ws-other": { recency: 42, streaming: false } }, + }) + ); + const restarted = new ExtensionMetadataService(filePath); + const internals = restarted as unknown as { + probeQuarantineSidecar: () => Promise; + }; + const originalProbe = internals.probeQuarantineSidecar.bind(restarted); + let probeCalls = 0; + internals.probeQuarantineSidecar = async () => { + probeCalls += 1; + if (probeCalls === 2) { + // Second probe = the one inside resumeQuarantineRecovery's queue. + internals.probeQuarantineSidecar = originalProbe; + const error = new Error("probe blocked") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + } + return originalProbe(); + }; + + let strictError: unknown = null; + try { + await restarted.getAllSnapshots({ throwOnError: true }); + } catch (error) { + strictError = error; + } + expect((strictError as NodeJS.ErrnoException | null)?.code).toBe("EACCES"); + // Latch was reset: the retry probes again and reconciles the sidecar. + const snapshots = await restarted.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-new")?.recency).toBe(300); + expect(snapshots.get("ws-other")?.recency).toBe(42); + }); + + test("a registration probe clears a stale tombstone and lets the write persist", async () => { + await service.updateRecency("ws-1", 100); + await service.deleteWorkspace("ws-1"); + // No probe wired: tombstones stay strictly write-suppressing. + await service.updateRecency("ws-1", 200); + expect((await service.getAllSnapshots()).has("ws-1")).toBe(false); + // Probe reporting the id NOT registered: still suppressed. + let registered = false; + service.setRegistrationProbe(() => Promise.resolve(registered)); + await service.updateRecency("ws-1", 300); + expect((await service.getAllSnapshots()).has("ws-1")).toBe(false); + expect(service.isWorkspaceDeleted("ws-1")).toBe(true); + // Re-registered (e.g. by a downgraded concurrent backend): the write + // must persist without waiting for an activity bootstrap, and the + // tombstone lifts so broadcasts resume too. + registered = true; + await service.updateRecency("ws-1", 400); + expect((await service.getAllSnapshots()).get("ws-1")?.recency).toBe(400); + expect(service.isWorkspaceDeleted("ws-1")).toBe(false); + }); + + test("clearTombstonesForRegisteredIds only clears tombstones the evidence postdates", async () => { + await service.updateRecency("ws-1", 100); + // Evidence snapshot captured BEFORE the removal: a tombstone published + // afterwards (same-process removal racing the activity list's evidence + // reads) must survive stale evidence claiming the id is registered. + const preEvidence = service.getTombstonedIds(); + await service.deleteWorkspace("ws-1"); + service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), preEvidence); + expect(service.isWorkspaceDeleted("ws-1")).toBe(true); + // A tombstone REPUBLISHED while the evidence was being gathered must + // also survive: the snapshot carries the old generation, so the stale + // positive clears only the exact tombstone it captured. + const staleGenerationEvidence = service.getTombstonedIds(); + await service.updateRecency("ws-1", 150); // suppressed (still tombstoned) + await service.deleteWorkspace("ws-1"); // republish: newer generation + service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), staleGenerationEvidence); + expect(service.isWorkspaceDeleted("ws-1")).toBe(true); + // The next bootstrap snapshots the tombstone before its evidence reads, + // so a genuinely re-registered id is cleared then. + service.clearTombstonesForRegisteredIds(new Set(["ws-1"]), service.getTombstonedIds()); + expect(service.isWorkspaceDeleted("ws-1")).toBe(false); + }); + + test("a strict read restores healthy bytes stranded in the sidecar by a crashed quarantine", async () => { + // The crash can also strand a HEALTHY file in the sidecar: a concurrent + // writer repaired the main file right before quarantine's rename moved + // it aside. Recovery must restore it — resetting to empty would discard + // live activity data. + const healthy = { + version: 1, + workspaces: { "ws-1": { recency: 42, streaming: false } }, + }; + await writeFile(`${filePath}.corrupt`, JSON.stringify(healthy)); + + const snapshots = await service.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get("ws-1")?.recency).toBe(42); + // Restored to the main path (sidecar consumed by the restore). + expect(JSON.parse(await readFile(filePath, "utf-8"))).toEqual(healthy); + }); }); diff --git a/src/node/services/ExtensionMetadataService.ts b/src/node/services/ExtensionMetadataService.ts index 02f2e35346..270b968047 100644 --- a/src/node/services/ExtensionMetadataService.ts +++ b/src/node/services/ExtensionMetadataService.ts @@ -1,5 +1,17 @@ -import { dirname } from "path"; -import { mkdir, readFile, access } from "fs/promises"; +import { basename, dirname, join } from "path"; +import { randomUUID } from "crypto"; +import { + mkdir, + readFile, + readdir, + access, + rename, + link, + unlink, + copyFile, + writeFile, + stat, +} from "fs/promises"; import { constants } from "fs"; import writeFileAtomic from "write-file-atomic"; import { @@ -16,6 +28,25 @@ import type { WorkspaceActivitySnapshot } from "@/common/types/workspace"; import type { GoalSnapshot } from "@/common/types/goal"; import { log } from "@/node/services/log"; +/** + * Identity of one quarantine-sidecar generation: with multiple processes + * recovering the fixed `.corrupt` path, consumption must be scoped to + * exactly the file generation a recovery read (see consumeQuarantineSidecar). + */ +interface QuarantineSidecarToken { + ino: bigint; + mtimeNs: bigint; + size: bigint; +} + +/** + * Marker code for "file written by a newer schema version" load failures. + * Shaped like an errno code so the corruption/quarantine classification + * (isDeterministicCorruption: errors WITH a code are not quarantinable) + * treats downgrade encounters as retryable, never as resettable corruption. + */ +const UNSUPPORTED_METADATA_VERSION_CODE = "XUM_UNSUPPORTED_METADATA_VERSION"; + /** * Stateless service for managing workspace metadata used by VS Code extension integration. * @@ -47,7 +78,124 @@ export interface ExtensionMetadataStreamingUpdate { export class ExtensionMetadataService { private readonly filePath: string; + + /** + * Name infix for per-consume claim files + * (`
----`). Unique names keep + * concurrent recoveries from replacing or clearing each other's live + * claims, and the embedded identity token makes every crash point + * replay-safe: the consumer only calls consume AFTER merging the + * generation identified by the token, so a stranded claim whose file + * identity MATCHES its embedded token is proven represented at the main + * path (rename preserves ino/mtime/size) and is deleted by discovery — + * replaying it would re-fill fields another backend explicitly cleared + * to null (the null-fill merge is not idempotent across clears). A + * mismatched or unparseable claim holds a foreign generation nobody + * merged and is replayed. + */ + private static readonly CLAIM_INFIX = ".corrupt-claim-"; + /** + * Prefix (after the main file name) for uniquely named in-flight + * moved-aside mains AND the scan claims that consume them when stranded. + * The trailing dash keeps the bounded fixed-name `.recreated` leftover + * (finalized, proven-superseded bytes) out of the stranded-file scan. + */ + private static readonly RECREATED_INFIX = ".recreated-"; + + /** Parse the identity token embedded in a claim file name; null when the + * name does not carry one (fail toward replay — the pre-token behavior). */ + private static parseClaimToken(claimName: string): QuarantineSidecarToken | null { + const match = /\.corrupt-claim-(\d+)-(\d+)-(\d+)-/.exec(claimName); + if (match == null) { + return null; + } + return { ino: BigInt(match[1]), mtimeNs: BigInt(match[2]), size: BigInt(match[3]) }; + } + + private static tokensMatch(a: QuarantineSidecarToken, b: QuarantineSidecarToken): boolean { + return a.ino === b.ino && a.mtimeNs === b.mtimeNs && a.size === b.size; + } private mutationQueue: Promise = Promise.resolve(); + /** + * Per-process write tombstones for removed workspaces. Workspace removal + * cannot drain every in-flight metadata producer (e.g. a stream-abort's + * fire-and-forget stop-status handler that is still reading todos when the + * entry is deleted), so late writers would silently recreate entries for + * removed workspaces and re-leak stale keys until the next process-start + * prune. Ids are added synchronously in deleteWorkspace/prune, so combined + * with the FIFO mutation queue there is no gap: a writer enqueued before + * the delete lands first and its entry is deleted; one enqueued after + * no-ops on this set. Workspace ids are never reused (generateStableId), + * so a tombstone never blocks a legitimate new workspace; recreations of + * legacy fixed-id workspaces happen in a different (downgraded) process. + */ + // Map value is the tombstone's GENERATION (monotonic per process): + // clearing paths that await between reading registration evidence and + // deleting must only clear the exact tombstone the evidence preceded — a + // same-process removal republishing the tombstone mid-probe must survive + // the stale positive (see recheckTombstonedRegistration). + private readonly deletedWorkspaceIds = new Map(); + private tombstoneGeneration = 0; + + private publishTombstone(workspaceId: string): number { + const generation = ++this.tombstoneGeneration; + this.deletedWorkspaceIds.set(workspaceId, generation); + return generation; + } + + /** + * In-memory suppression for a CROSS-PROCESS removal proven by the + * activity list's config-corroborated guards. The foreign backend's + * removal cannot publish a tombstone in this process, so without one a + * late local producer (workflow-run or bash-monitor completion) would + * repopulate the just-evicted caches and emitWorkspaceActivity — whose + * isWorkspaceDeleted check only knows local removals — would broadcast + * the removed incarnation's activity right after an authoritative list + * dropped it. Reuses the standard tombstone lifecycle, so suppression + * ends exactly on fresh registration evidence: writes re-probe via + * recheckTombstonedRegistration and later lists clear it via + * clearTombstonesForRegisteredIds when the id is genuinely re-registered. + */ + suppressForeignRemoval(workspaceId: string): void { + this.publishTombstone(workspaceId); + } + + /** + * Single exit point for clearing a tombstone (re-registration revival). + * Notifies the owner so process-local activity caches bootstrapped for the + * REMOVED incarnation (active workflow-run ids, bash-monitor seen set) are + * evicted — workspace removal never evicts them, and the revived + * incarnation's session state on disk may differ, so a stale cache would + * otherwise show ghost activity counts indefinitely. + */ + private liftTombstone(workspaceId: string): void { + this.deletedWorkspaceIds.delete(workspaceId); + try { + this.tombstoneClearedListener?.(workspaceId); + } catch { + // Listener failures must not break the clearing path. + } + } + + private tombstoneClearedListener: ((workspaceId: string) => void) | null = null; + + setTombstoneClearedListener(listener: (workspaceId: string) => void): void { + this.tombstoneClearedListener = listener; + } + + /** + * Optional registration probe for writes hitting a tombstoned id (see + * recheckTombstonedRegistration). Wired by the owner that can read the + * shared config (coreServices); absent in bare constructions, where + * tombstones stay strictly write-suppressing. Must answer with CURRENT + * registration evidence or throw — a stale positive would resurrect a + * removed workspace's entry. + */ + private registrationProbe: ((workspaceId: string) => Promise) | null = null; + + setRegistrationProbe(probe: (workspaceId: string) => Promise): void { + this.registrationProbe = probe; + } /** * Serialize all mutating operations on the shared metadata file. @@ -96,20 +244,153 @@ export class ExtensionMetadataService { return normalized ? toWorkspaceActivitySnapshot(normalized) : null; } + /** + * Late write for a removed workspace: compute the snapshot for the caller + * but never persist it, so the deleted entry cannot be resurrected. + */ + private buildTransientSnapshot( + workspaceId: string, + recency: number, + mutate: (workspace: ExtensionMetadata) => void + ): WorkspaceActivitySnapshot { + const transient = this.getOrCreateWorkspaceEntry( + { version: 1, workspaces: {} }, + workspaceId, + recency + ); + mutate(transient); + return toWorkspaceActivitySnapshot(transient); + } + + /** + * A write arrived for a tombstoned id: decide whether the tombstone is + * stale. Tombstones are process-local removal knowledge and the shared + * config is the authority — a downgraded concurrent backend can + * legitimately re-register a deterministic legacy id this process pruned, + * and the renderer only calls the activity bootstrap (which also clears + * stale tombstones) when its subscription needs repair, so a healthy + * long-lived process would otherwise suppress the revived workspace's + * writes and broadcasts indefinitely. The probe reads registration + * evidence NOW, strictly after the tombstone was published, so clearing on + * a positive answer is sound (same ordering contract as + * clearTombstonesForRegisteredIds). A negative, missing, or failing probe + * keeps the tombstone: suppression self-heals on a later successful probe + * or bootstrap, while a wrongly persisted write would resurrect a removed + * workspace's entry on disk. + */ + private async recheckTombstonedRegistration(workspaceId: string): Promise { + if (this.registrationProbe == null) { + return false; + } + // Capture the tombstone's generation before the probe: a same-process + // removal can republish the tombstone while the probe awaits, and a + // probe that read config just before that deregistration returns a + // stale positive that must not clear the NEWER tombstone (the removal's + // queued deletion may run before the caller's queued write, which would + // then recreate the removed entry). + const generationBefore = this.deletedWorkspaceIds.get(workspaceId); + let probeRegistered = false; + try { + probeRegistered = await this.registrationProbe(workspaceId); + } catch { + probeRegistered = false; + } + const generationAfter = this.deletedWorkspaceIds.get(workspaceId); + if (generationAfter === undefined) { + // Another path (activity bootstrap, reconcile) lifted the tombstone + // while the probe was in flight, on registration evidence that + // postdates the tombstone. The write must PERSIST regardless of THIS + // probe's outcome (negative and failing probes included): once the + // tombstone is gone, broadcasts are un-suppressed and every later + // write persists normally — suppressing only this in-flight write + // would hand the caller an unpersisted transient snapshot that + // WorkspaceService still broadcasts, leaving renderer recency/goal/ + // status ahead of disk until restart. A genuinely newer same-process + // removal republishes a tombstone, which the caller's in-queue + // re-check still honors. + return true; + } + if (!probeRegistered) { + // Negative or failing probe with the tombstone still standing: keep + // suppressing. Emits stay suppressed by the same tombstone, so no + // transient state can reach the renderer. + return false; + } + if (generationAfter !== generationBefore) { + // Republished mid-probe: a newer same-process removal wins. + return false; + } + this.liftTombstone(workspaceId); + return true; + } + private async mutateWorkspaceSnapshot( workspaceId: string, recency: number, mutate: (workspace: ExtensionMetadata) => void ): Promise { + if ( + this.deletedWorkspaceIds.has(workspaceId) && + !(await this.recheckTombstonedRegistration(workspaceId)) + ) { + return this.buildTransientSnapshot(workspaceId, recency, mutate); + } + // Write-produced snapshots are broadcast by WorkspaceService: reconcile a + // crash-stranded sidecar BEFORE mutating (outside the queue — the resume + // path serializes itself). Without this, a valid partial main next to a + // healthy full sidecar is saved and EMITTED, clearing goal/status in the + // renderer until some read triggers recovery. Failure propagates: + // failing one write beats persisting and broadcasting the partial view + // (same tradeoff as the unprobeable-sidecar contract in + // probeQuarantineSidecar). + await this.reconcileLeftoverSidecarIfPresent(); return this.withSerializedMutation(async () => { + // Re-check inside the queue: pruneMissingWorkspaces publishes its + // tombstones only while its queued mutation runs, so a writer that + // passed the pre-queue check and enqueued behind the prune must not + // recreate an entry the prune just reclaimed. + if ( + this.deletedWorkspaceIds.has(workspaceId) && + !(await this.recheckTombstonedRegistration(workspaceId)) + ) { + return this.buildTransientSnapshot(workspaceId, recency, mutate); + } const data = await this.load(); const workspace = this.getOrCreateWorkspaceEntry(data, workspaceId, recency); mutate(workspace); + // Every persisted mutation advances the write generation: `recency` + // is a user-interaction timestamp (status/goal/streaming writers + // deliberately preserve it), so recovery merges cannot order metadata + // copies by recency alone — see recoverStrandedRecreatedLeftover. + workspace.writeGeneration = ExtensionMetadataService.nextWriteGeneration(workspace); + // (The stamp is epoch-ms so the stranded-leftover merge can order a + // generation-carrying entry against a generation-LESS copy via the + // stranded file's mtime — see nextWriteGeneration.) await this.save(data); return toWorkspaceActivitySnapshot(workspace); }); } + /** + * Next per-entry write stamp: wall-clock epoch milliseconds, floored to + * strictly exceed the previous stamp (same-millisecond mutations and + * backward clock steps stay monotonic per entry). A plain counter would + * order two generation-carrying copies just as well, but the stranded- + * leftover merge must also order a generation-carrying entry against a + * generation-LESS copy (written by a pre-generation or downgraded build): + * with a wall-clock stamp it can compare the entry's last write time + * against the stranded FILE's mtime — same host, same clock — and prove + * the generation-carrying write postdates every byte of the stranded + * snapshot. See recoverStrandedRecreatedLeftover's ordering comment. + */ + private static nextWriteGeneration(entry: ExtensionMetadata): number { + const previous = + typeof entry.writeGeneration === "number" && Number.isFinite(entry.writeGeneration) + ? entry.writeGeneration + : 0; + return Math.max(Date.now(), previous + 1); + } + constructor(filePath?: string) { this.filePath = filePath ?? getXumExtensionMetadataPath(); } @@ -144,28 +425,313 @@ export class ExtensionMetadataService { } } - private async load(): Promise { + /** + * Consume the quarantine sidecar once every surviving entry is represented + * at the main path. ENOENT means a concurrent recovery already consumed + * it; any other failure must propagate (retryable) — reporting success + * with the sidecar still present would let a later process reconcile the + * stale sidecar again after the main file has moved on, repeatedly + * re-merging entries that pruning or removal already reclaimed. + * + * Claim-then-verify: with multiple processes recovering the fixed sidecar + * path, another backend can consume the generation this caller read and + * quarantine a NEW snapshot at the same path — a stat-compare followed by + * a path unlink would leave a window in which that unreconciled newer + * generation is destroyed. The rename below atomically takes exactly one + * generation off the shared path first; identity is then verified on the + * claimed file. The claim name is UNIQUE per consume (pid + uuid): a fixed + * name would let a concurrent consumer's claim land on (POSIX rename + * replaces) or clear (its own leftover pass) another recovery's live + * claim, re-opening the destroyed-generation window this claim exists to + * close — and a fresh unique destination also never collides on Windows, + * where rename onto an existing file is not reliably a replace. A claimed + * FOREIGN generation (identity mismatch) is reconciled into the main file + * rather than deleted. Invariant: a claim file whose identity matched its + * consumer's token holds already-reconciled bytes (callers merge/restore + * before consuming), so only a crash between a MISMATCH claim and its + * reconcile strands unconsumed data at a claim name — + * reconcileLeftoverSidecarIfPresent discovers stranded claims by prefix + * and resumes the merge on the next authoritative read. + */ + private async consumeQuarantineSidecar( + quarantinePath: string, + token: QuarantineSidecarToken | null + ): Promise { + if (token == null) { + // The caller never observed a sidecar identity (it read the file only + // after a successful probe, so this is unreachable in practice) — + // fail closed by leaving the file rather than unlinking blind. + return; + } + // The claim name embeds the token of the generation the caller just + // merged (see CLAIM_INFIX): a crash at ANY later point leaves a claim + // discovery can classify by identity — matching bytes were merged + // (delete), anything else is a foreign generation (replay). + const claimPath = `${this.filePath}${ExtensionMetadataService.CLAIM_INFIX}${token.ino}-${token.mtimeNs}-${token.size}-${process.pid}-${randomUUID()}`; try { - await access(this.filePath, constants.F_OK); - } catch { - return { version: 1, workspaces: {} }; + await rename(quarantinePath, claimPath); + } catch (renameError) { + if (ExtensionMetadataService.isErrnoCode(renameError, "ENOENT")) { + return; // Already consumed by a concurrent recovery. + } + throw renameError; + } + // Identity check on the CLAIMED file (nothing else references the + // unique name except the stranded-claim discovery, which reconciles + // rather than destroys): non-ENOENT probe failures propagate + // (retryable) with the claim left for that discovery — reporting + // success would let the caller proceed, e.g. the one-time prune deletes + // stale entries, a later read re-merges them from a retained sidecar, + // and with the prune latch already set the resurrected entries stay + // indefinitely. + const current = await ExtensionMetadataService.statQuarantineToken(claimPath); + if (current == null) { + return; // Claim vanished (concurrent discovery consumed it). + } + if (!ExtensionMetadataService.tokensMatch(current, token)) { + // The claim took a NEWER generation another process installed after + // consuming ours: merge it into the main file instead of destroying + // it. The reconcile consumes the claim itself on success; a corrupt + // foreign generation stays at the claim name until the discovery + // pass's reconcile classifies it (bounded: one file per crashed or + // failed recovery, consumed on the next successful pass). + log.debug("Claimed a replaced quarantine sidecar generation; reconciling it", { + quarantinePath, + }); + await this.reconcileRecreatedMainWithSidecar(claimPath); + return; } + // Matched: the claim's bytes are proven represented at the main path, + // and its name says so (the embedded token matches the file), so a + // crash before this unlink strands a file discovery deletes rather + // than replays. + try { + await unlink(claimPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + } + /** + * Identity of one sidecar generation (inode + mtime + size), captured when + * a recovery reads the sidecar and required by consumeQuarantineSidecar so + * consumption is scoped to exactly the generation that was reconciled. + * Resolves null on ENOENT; other stat failures propagate (retryable). + */ + private static async statQuarantineToken( + quarantinePath: string + ): Promise { try { - const content = await readFile(this.filePath, "utf-8"); - const parsed = JSON.parse(content) as ExtensionMetadataFile; + const stats = await stat(quarantinePath, { bigint: true }); + return { ino: stats.ino, mtimeNs: stats.mtimeNs, size: stats.size }; + } catch (statError) { + if (ExtensionMetadataService.isErrnoCode(statError, "ENOENT")) { + return null; + } + throw statError; + } + } - // Validate structure - if (typeof parsed !== "object" || parsed.version !== 1) { - log.error("Invalid metadata file, resetting"); - return { version: 1, workspaces: {} }; + /** + * Shared leftover-sidecar recovery for the read paths (authoritative list + * and per-workspace snapshot reads): probe the fixed sidecar path and run + * the resumable recovery when one exists. Returns true when a sidecar was + * found (callers should re-read). Failures propagate; each caller decides + * whether its read contract is strict (list) or best-effort (live + * emissions). + */ + private async reconcileLeftoverSidecarIfPresent(): Promise { + let reconciledSidecar = false; + if (await this.probeQuarantineSidecar()) { + await this.resumeQuarantineRecovery(); + reconciledSidecar = true; + } + // A crash between consumeQuarantineSidecar's mismatch claim and its + // reconcile strands an unreconciled foreign generation at a unique + // claim name, which the sidecar probe above cannot see. Discover + // stranded claims by prefix (one readdir next to the full-file read the + // caller already does); an unreadable directory propagates so the read + // stays retryable rather than vouching for a possibly partial main. + // Runs even when the fixed sidecar was just processed: a + // deterministically corrupt sidecar is intentionally RETAINED by its + // reconcile, and returning early on it would starve stranded-claim + // recovery indefinitely (recency/goal/status hidden in the claim). + const dirNames = await readdir(dirname(this.filePath)); + const claimNames = dirNames.filter((name) => + name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.CLAIM_INFIX}`) + ); + // Crash-stranded in-flight moved-aside mains (see + // moveMainAsideAsRecreatedLeftover): their unique names make them + // invisible to every fixed-path probe, so without this scan an owner + // crash before revalidation would orphan a raced healthy main's only + // copy forever while the next recovery restores the OLDER sidecar. A + // live owner's file listed here is gone again by the time the queue + // slot below runs its claim rename (ENOENT — see + // recoverStrandedRecreatedLeftover for the steal semantics). + const strandedRecreatedNames = dirNames.filter((name) => + name.startsWith(`${basename(this.filePath)}${ExtensionMetadataService.RECREATED_INFIX}`) + ); + if (claimNames.length === 0 && strandedRecreatedNames.length === 0) { + return reconciledSidecar; + } + await this.withSerializedMutation(async () => { + for (const strandedName of strandedRecreatedNames) { + await this.recoverStrandedRecreatedLeftover(join(dirname(this.filePath), strandedName)); + } + for (const claimName of claimNames) { + const claimPath = join(dirname(this.filePath), claimName); + // Re-probe inside the queue: a concurrent recovery's discovery (or + // the claim's own consume) may have taken the file while this + // caller waited. Only a verified ENOENT skips — an unprobeable + // claim propagates, same contract as the sidecar probe: reporting + // success would let a strict read accept and emit a possibly + // partial main while recoverable fields sit in the claim. + const current = await ExtensionMetadataService.statQuarantineToken(claimPath); + if (current == null) { + continue; + } + // Identity classification (see CLAIM_INFIX): a claim whose file + // matches its embedded token holds bytes its consumer had already + // merged before claiming — delete, never replay (a re-merge would + // re-fill fields another backend explicitly cleared to null + // since). Mismatched or token-less claims hold a foreign + // generation nobody merged: replay it. + const expected = ExtensionMetadataService.parseClaimToken(claimName); + if (expected != null && ExtensionMetadataService.tokensMatch(current, expected)) { + try { + await unlink(claimPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + continue; + } + await this.reconcileRecreatedMainWithSidecar(claimPath); + } + }); + return true; + } + + /** + * Seam for load()'s missing-main handling (and deterministic TOCTOU tests): + * reports whether the quarantine sidecar currently exists. + */ + private probeQuarantineSidecar(): Promise { + return access(`${this.filePath}.corrupt`).then( + () => true, + (error: unknown) => { + if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + return false; + } + // EACCES/EIO/...: the sidecar's existence is unknowable, and only a + // verified absence may let an ENOENT main read resolve as a healthy + // empty file — recoverable metadata may sit in the unprobeable + // sidecar. Propagate so the read stays a retryable failure for + // strict readers and fails the mutation for lenient writers (same + // tradeoff as the in-window recovery: failing one operation beats + // clobbering the sidecar's data with a partial save). + throw error; } + ); + } - return parsed; - } catch (error) { - log.error("Failed to load metadata:", error); - return { version: 1, workspaces: {} }; + private async load(options?: { throwOnError?: boolean }): Promise { + // Bounded because the ENOENT branch below re-reads: each retry only + // happens after a fresh ENOENT, so the loop converges. + const MAX_READ_ATTEMPTS = 3; + let lastError: unknown; + // True when the final failed attempt hit the mid-quarantine window + // (main missing, sidecar present). That failure must never resolve as + // an empty file, even leniently — see the sidecar branch below. + let blockedByQuarantineWindow = false; + for (let attempt = 1; attempt <= MAX_READ_ATTEMPTS; attempt++) { + try { + const content = await readFile(this.filePath, "utf-8"); + const parsed = JSON.parse(content) as ExtensionMetadataFile; + + // A syntactically valid file whose version is not 1 was written by a + // build with a newer schema — NOT corruption. It must never be + // quarantined/reset (upgrading back would find the canonical file + // empty and lose all activity state) and never self-healed to {} by + // a lenient writer (saving version-1 bytes over it is the same data + // loss). Both read modes propagate; see the catch below. + if (ExtensionMetadataService.isUnsupportedVersion(parsed)) { + throw ExtensionMetadataService.unsupportedVersionError(); + } + + // Validate structure, including the workspaces container: a parseable + // file with e.g. an array or primitive `workspaces` would otherwise + // enumerate as zero entries and masquerade as an authoritative empty + // state in strict reads. + if (!ExtensionMetadataService.isValidMetadataFileShape(parsed)) { + throw new Error("Invalid extension metadata file structure"); + } + + return parsed; + } catch (error) { + // Only a genuinely missing file is a healthy empty state. Other read + // failures (EACCES/ENOTDIR/EIO, parse or structure errors) must not + // masquerade as one: throwOnError lets read paths distinguish them + // from an authoritative empty state, while the default self-heals so + // writers can always make progress. + lastError = error; + blockedByQuarantineWindow = false; + if (!ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + break; + } + // A missing main file WITH the quarantine sidecar present is not a + // healthy empty state: it is the (rare) window where quarantine + // moved the file aside and has not yet written the empty + // replacement — the moved bytes may even be a concurrent writer's + // healthy repair awaiting restore. Resolving this window as an + // empty file is never safe, lenient or strict: a lenient WRITER + // would mutate {} and save, recreating the main path — and the + // pending restore (deliberately no-overwrite) would then strand + // every other workspace's metadata in the sidecar for good. + if (await this.probeQuarantineSidecar()) { + blockedByQuarantineWindow = true; + if (options?.throwOnError) { + // Strict readers surface a retryable failure (ENOENT carries an + // errno code, so callers classify it as transient) and + // getAllSnapshots resumes the recovery through the mutation + // queue — never an authoritative {} that a subsequent restore + // cannot retract. + break; + } + // Lenient callers complete the recovery INLINE and re-read. + // Inline (unqueued) is deliberate: most lenient loads run inside + // withSerializedMutation, whose promise-chain queue is not + // reentrant — enqueueing resumeQuarantineRecovery here would + // deadlock. Racing a concurrent recovery is safe because every + // step is no-overwrite (link/COPYFILE_EXCL) and idempotent. A + // recovery failure propagates: failing one mutation in this + // pathological window beats destroying the sidecar's data. + await this.completeQuarantineRecovery(`${this.filePath}.corrupt`); + continue; + } + // No sidecar either. With multiple instances the failed read may + // have raced another process's COMPLETED recovery — main restored + // (or reset to empty) and the sidecar already consumed — so the + // absent sidecar proves nothing about the earlier ENOENT. Re-read + // the main path instead of trusting the stale failure; only a file + // still missing on the final attempt is a healthy empty state. + if (attempt === MAX_READ_ATTEMPTS) { + return { version: 1, workspaces: {} }; + } + } } + if ( + options?.throwOnError || + blockedByQuarantineWindow || + ExtensionMetadataService.isErrnoCode(lastError, UNSUPPORTED_METADATA_VERSION_CODE) + ) { + throw lastError; + } + log.error("Failed to load metadata:", lastError); + return { version: 1, workspaces: {} }; } private async save(data: ExtensionMetadataFile): Promise { @@ -268,7 +834,27 @@ export class ExtensionMetadataService { status: ExtensionAgentStatus | null, options: { skipIfRecencyAdvancedSince?: number | null; inputHash?: string | null } = {} ): Promise { + // See deletedWorkspaceIds: never resurrect a removed workspace's entry. + // A stale tombstone (id re-registered by a concurrent backend) is + // cleared via the registration recheck, same as mutateWorkspaceSnapshot. + if ( + this.deletedWorkspaceIds.has(workspaceId) && + !(await this.recheckTombstonedRegistration(workspaceId)) + ) { + return null; + } + // Same pre-mutation sidecar reconcile as mutateWorkspaceSnapshot: this + // path also saves and returns a snapshot the caller broadcasts. + await this.reconcileLeftoverSidecarIfPresent(); return this.withSerializedMutation(async () => { + // Re-check inside the queue (see mutateWorkspaceSnapshot): tombstones + // published by an already-enqueued prune must be honored here too. + if ( + this.deletedWorkspaceIds.has(workspaceId) && + !(await this.recheckTombstonedRegistration(workspaceId)) + ) { + return null; + } const data = await this.load(); const existing = coerceExtensionMetadata(data.workspaces[workspaceId]); const workspace: ExtensionMetadata = existing ?? { @@ -297,6 +883,9 @@ export class ExtensionMetadataService { delete workspace.todoStatus; delete workspace.sidebarStatusInputHash; } + // Same write-generation contract as mutateWorkspaceSnapshot: this + // writer persists a mutation while deliberately preserving recency. + workspace.writeGeneration = ExtensionMetadataService.nextWriteGeneration(workspace); data.workspaces[workspaceId] = workspace; await this.save(data); return toWorkspaceActivitySnapshot(workspace); @@ -365,26 +954,287 @@ export class ExtensionMetadataService { }); } - async getSnapshot(workspaceId: string): Promise { - const data = await this.load(); + async getSnapshot( + workspaceId: string, + options?: { throwOnError?: boolean } + ): Promise { + // Same leftover-sidecar reconcile as getAllSnapshots (see the comment + // there): live emissions read through this path after the subscription + // bootstraps, so without it a recreated partial main would feed emitted + // snapshots (clearing goal/status in the renderer) while the healthy + // subscription never triggers another list read. + // Strict callers (the emit paths) must also propagate a FAILED + // reconcile instead of reading through it: with a sidecar present the + // main file is suspect (typically a partial recreation), and emitting + // it would clear goal/status in the renderer with no guaranteed + // strict-list retry to repair — skipping the emit retains the + // renderer's last-known snapshot until the reconcile succeeds. Lenient + // callers (settings/eligibility readers) keep availability: a stranded + // sidecar must never block message sending or heartbeats. + try { + await this.reconcileLeftoverSidecarIfPresent(); + } catch (error) { + if (options?.throwOnError) { + throw error; + } + log.debug("Leftover sidecar reconcile failed during snapshot read", { error }); + } + const data = await this.loadWithCorruptionRecovery(options); return this.toSnapshot(data.workspaces[workspaceId]); } + /** + * Whether this process removed the workspace's entry (see + * deletedWorkspaceIds). Lets emit paths suppress late activity broadcasts + * whose disk writes the tombstone already blocked. + */ + isWorkspaceDeleted(workspaceId: string): boolean { + return this.deletedWorkspaceIds.has(workspaceId); + } + + /** + * Drop write tombstones for ids a fresh config view proves are registered. + * Tombstones are process-local removal knowledge and the shared config is + * the authority: with XUM_ALLOW_MULTIPLE_INSTANCES a downgraded concurrent + * backend can legitimately re-register a deterministic legacy id this + * process pruned earlier, and without this hook the stale tombstone would + * suppress every one of the revived workspace's metadata writes (and + * filter it from activity lists) until restart. Called from the activity + * bootstrap with fresh config-derived id sets only — never with snapshot + * or in-memory cache keys, which do not prove registration. + * + * `eligibleIds` must be a getTombstonedIds() snapshot captured BEFORE the + * registration evidence was gathered: clearing is sound only when the + * evidence postdates the tombstone. A tombstone published while the + * evidence reads were in flight (same-process removal during the activity + * list's authoritative enumeration await) would otherwise be cleared by a + * stale pre-removal view, un-suppressing late writers and letting the + * removed entry ride back into the renderer. + */ + clearTombstonesForRegisteredIds( + registeredIds: ReadonlySet, + eligibleIds: ReadonlyMap + ): void { + for (const [workspaceId, generation] of eligibleIds) { + // Generation compare (see recheckTombstonedRegistration): a same- + // process removal can republish the tombstone while the caller's + // registration evidence was being gathered — the stale positive must + // clear only the exact tombstone the snapshot captured, never the + // newer removal's. + if ( + registeredIds.has(workspaceId) && + this.deletedWorkspaceIds.get(workspaceId) === generation + ) { + this.liftTombstone(workspaceId); + } + } + } + + /** + * Snapshot of the ids currently write-tombstoned in this process, with + * their generations. Capture it before gathering registration evidence and + * pass it back to clearTombstonesForRegisteredIds as the set of clearable + * tombstones — the generation lets the clear skip tombstones republished + * after the snapshot. + */ + getTombstonedIds(): ReadonlyMap { + return new Map(this.deletedWorkspaceIds); + } + /** * Delete metadata for a workspace. * Call this when a workspace is deleted. */ async deleteWorkspace(workspaceId: string): Promise { + // Synchronously, before the queued mutation: any writer enqueued from now + // on must see the tombstone (see deletedWorkspaceIds). + const publishedGeneration = this.publishTombstone(workspaceId); + // Reconcile a crash-stranded sidecar BEFORE the queued deletion (outside + // the queue — the resume path serializes itself), same as the emitting + // write entry points: deleting from a recreated PARTIAL main would skip + // the removed workspace's complete entry sitting in the sidecar, and a + // concurrent backend without this process-local tombstone could later + // reconcile that sidecar and restore the removed entry (visible + // immediately on unscoped builds; inherited as stale goal/status by a + // deterministic legacy-id re-registration). A failing reconcile + // propagates: the tombstone stays (fail closed, writes suppressed) and + // the disk entry remains recoverable for a retried removal or the + // process-start prune. + await this.reconcileLeftoverSidecarIfPresent(); await this.withSerializedMutation(async () => { const data = await this.load(); - - if (data.workspaces[workspaceId]) { + // In-queue registration revalidation, strictly AFTER the load (same + // ordering as the prune): a workspace is durably registered before its + // first metadata write, so any entry visible in the loaded snapshot + // was persisted before the load — a post-load probe reporting + // "unregistered" therefore postdates that write and proves the entry + // belongs to the removed incarnation. A downgraded backend + // re-registering a deterministic legacy id (after the caller's + // deregistration checks, or while this deletion waited in the queue) + // keeps its fresh snapshot: the probe-confirmed registration aborts + // the deletion and lifts the tombstone THIS call published + // (generation-guarded — a newer removal's tombstone stays). An + // unknowable probe keeps the tombstone and skips the disk deletion: a + // stale entry is recoverable (filtered by the tombstone now, reclaimed + // by a later removal or process-start prune), destroyed re-registered + // data is not. Without a probe the caller's own deregistration + // evidence stands. + if (this.registrationProbe != null) { + let registered: boolean; + try { + registered = await this.registrationProbe(workspaceId); + } catch { + return; + } + if (registered) { + if (this.deletedWorkspaceIds.get(workspaceId) === publishedGeneration) { + this.liftTombstone(workspaceId); + } + return; + } + } + // Key presence, not truthiness: malformed falsy persisted entries + // (e.g. null) must be deleted too, or removal leaves a stale key + // behind until the next process-start prune. + if (workspaceId in data.workspaces) { delete data.workspaces[workspaceId]; await this.save(data); } }); } + /** + * Remove entries whose workspace no longer exists. Removed workspaces and + * sub-agents were historically never pruned here, so long-lived deployments + * accumulate thousands of stale entries that inflate every read and rewrite + * of this file (issue #3959 measured 13,895 entries for 1,513 known + * workspaces). Called once per process; `deleteWorkspace` keeps the file + * bounded afterwards. + * + * Loss safety: `getKnownWorkspaceIds` is invoked INSIDE the serialized + * mutation and strictly AFTER the file is loaded, and the callback reads + * config fresh from disk. A workspace is durably registered in config + * before its first metadata write, so every entry visible in the loaded + * file belongs to a workspace whose registration is already on disk — the + * post-load fetch therefore always includes live entries' ids, even for + * workspaces created concurrently by another backend process + * (XUM_ALLOW_MULTIPLE_INSTANCES). Fetching before the load would leave a + * window where another process registers + writes a fresh entry that the + * stale known-ids set misclassifies as prunable. + * + * Cross-process writers (XUM_ALLOW_MULTIPLE_INSTANCES) are not serialized + * by the in-process queue, so this pass never rewrites its own working + * snapshot: it computes the stale-id set from the first load, then re-loads + * a FRESH snapshot and applies only those deletions before saving. A fresh + * entry another backend wrote between the two loads is preserved (its id + * was not in the first snapshot, so it is never classified stale), and + * since workspace ids are never reused, a stale id cannot have become live + * in between. The residual window (a foreign write landing during the + * final stringify+atomic-write) is the same lost-update window every + * existing writer of this file already has. + * + * Upgrade↔downgrade safety: surviving entries are round-tripped verbatim + * (no coercion), so fields written by other builds are preserved and the + * on-disk format is unchanged. + */ + async pruneMissingWorkspaces( + getKnownWorkspaceIds: () => Promise>, + // Optional cheaper view for the mid-prune re-registration recheck: it + // only needs to answer "is this stale-classified id registered NOW", so + // callers whose full enumeration is expensive (per-workspace filesystem + // walks) can substitute an equally-complete but cheaper read. It must be + // COMPLETE (contain every currently registered id) or throw — resolving + // with a lossy set would let the prune delete a re-registered + // workspace's data. Defaults to getKnownWorkspaceIds. + recheckKnownWorkspaceIds?: () => Promise> + ): Promise { + // Reconcile a crash-stranded sidecar FIRST (outside the mutation queue — + // the resume path serializes itself): the prune classifies stale ids + // against the file it loads, so sidecar-only entries would dodge the + // one-time deletion set and merge back into the main file on the very + // next read — with the prune latched, exactly the stale entries this + // cleanup exists to remove would keep inflating every read and rewrite + // until restart. A failing reconcile propagates so the caller's + // fail-closed abort applies instead of pruning against a partial view. + await this.reconcileLeftoverSidecarIfPresent(); + return this.withSerializedMutation(async () => { + const data = await this.load(); + const knownWorkspaceIds = await getKnownWorkspaceIds(); + const staleWorkspaceIds = Object.keys(data.workspaces).filter( + (workspaceId) => !knownWorkspaceIds.has(workspaceId) + ); + const staleTombstoneGenerations = new Map(); + for (const workspaceId of staleWorkspaceIds) { + // Same guard as deleteWorkspace: a late in-process writer must not + // resurrect an entry this pass reclaims. The generation scopes the + // re-registration spare below to THIS prune's tombstone. + staleTombstoneGenerations.set(workspaceId, this.publishTombstone(workspaceId)); + } + if (staleWorkspaceIds.length === 0) { + return 0; + } + // Re-registration recheck: with multiple instances a downgraded + // backend can re-register a deterministic legacy id (and write new + // activity for it) between the enumeration above and here. Deleting + // on the stale classification would destroy that new entry's + // recency/goal/status — clearing the tombstone later cannot restore + // data. A re-registered id is dropped from the deletion set and its + // write tombstone lifted. If the recheck fails, abort the prune (throw + // to the caller's catch) rather than deleting on stale knowledge. + const recheckedKnownIds = await (recheckKnownWorkspaceIds ?? getKnownWorkspaceIds)(); + // Deletion-only merge against a fresh snapshot loaded strictly AFTER + // the recheck — the LAST await before the save below. The recheck can + // perform a full legacy enumeration, and any recency/goal/status + // another backend writes during that await would be absent from a + // pre-recheck snapshot: save() would silently roll it back while + // deleting the stale keys. The inverse race (an id re-registered + // after the recheck read but before this load) is covered by the + // unchanged-bytes guard below, which is strictly narrower than the + // enumeration-wide window this ordering closes. + const fresh = await this.load(); + let prunedCount = 0; + for (const workspaceId of staleWorkspaceIds) { + if (recheckedKnownIds.has(workspaceId)) { + // Generation-guarded (see clearTombstonesForRegisteredIds): the + // recheck enumeration awaited disk, and a same-process removal can + // republish this tombstone mid-await — the enumeration's + // pre-removal positive must clear only the prune's own tombstone, + // never the newer removal's (a late writer would otherwise pass + // its in-queue check and recreate the removed entry). + if ( + this.deletedWorkspaceIds.get(workspaceId) === staleTombstoneGenerations.get(workspaceId) + ) { + this.liftTombstone(workspaceId); + } + continue; + } + if (!(workspaceId in fresh.workspaces)) { + continue; + } + // Fail-closed bytes guard: the registration evidence above predates + // this load, so an id re-registered in that gap says "unregistered" + // while a concurrent backend may already have written fresh + // activity for it. A stale-classified entry whose bytes CHANGED + // between the two loads proves such a writer — spare it (the entry + // is re-evaluated on the next process start; the write tombstone + // stays until fresh registration evidence clears it through the + // normal revival paths). + if ( + JSON.stringify(fresh.workspaces[workspaceId]) !== + JSON.stringify(data.workspaces[workspaceId]) + ) { + continue; + } + delete fresh.workspaces[workspaceId]; + prunedCount++; + } + if (prunedCount > 0) { + await this.save(fresh); + } + return prunedCount; + }); + } + /** * Clear all streaming flags. * Call this on app startup to clean up stale streaming states from crashes. @@ -411,8 +1261,994 @@ export class ExtensionMetadataService { }); } - async getAllSnapshots(): Promise> { - const data = await this.load(); + /** + * fs errors carry an errno `code` and may be transient (EACCES/EIO/...); + * anything readFile's content produced afterwards (JSON parse or structure + * validation errors) fails identically for the same bytes on every retry. + */ + private static isDeterministicCorruption(error: unknown): boolean { + return !(typeof error === "object" && error != null && "code" in error); + } + + private static isErrnoCode(error: unknown, code: string): boolean { + return typeof error === "object" && error != null && "code" in error && error.code === code; + } + + /** + * A structurally sound file whose version is not 1: written by a newer + * schema, not corrupt. Carries the marker code so isDeterministicCorruption + * classifies it as non-quarantinable and load() refuses to self-heal it. + */ + private static isUnsupportedVersion(parsed: unknown): boolean { + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return false; + } + const version = (parsed as { version?: unknown }).version; + // Only a structurally PLAUSIBLE forward version (an integer greater + // than 1) earns the non-destructive preservation path. A malformed value + // (null, string, object, or a numeric no schema lineage can produce — + // 0, -1, 2.5) must classify as deterministic corruption instead: + // preserving it would leave every strict read and lenient writer failing + // forever on a file no build can ever read, rather than quarantining and + // self-healing. + return typeof version === "number" && Number.isInteger(version) && version > 1; + } + + private static unsupportedVersionError(): NodeJS.ErrnoException { + const error = new Error( + "Unsupported extension metadata file version (written by a newer build)" + ) as NodeJS.ErrnoException; + error.code = UNSUPPORTED_METADATA_VERSION_CODE; + return error; + } + + private static isValidMetadataFileShape(parsed: unknown): parsed is ExtensionMetadataFile { + if (typeof parsed !== "object" || parsed === null) { + return false; + } + const candidate = parsed as { version?: unknown; workspaces?: unknown }; + return ( + candidate.version === 1 && + typeof candidate.workspaces === "object" && + candidate.workspaces !== null && + !Array.isArray(candidate.workspaces) + ); + } + + /** + * Move a deterministically corrupt metadata file aside so strict readers + * stop failing on every retry across process restarts (nothing else + * repairs the file until some unrelated writer happens to replace it). + * The original bytes are preserved at a fixed `.corrupt` path for + * inspection rather than deleted; the fixed name keeps quarantine bounded. + * Serialized with writers, and corruption is re-verified inside the queue + * so a concurrently repaired (just-saved) file is never quarantined. + */ + private async quarantineCorruptFile(): Promise { + return this.withSerializedMutation(async () => { + try { + await this.load({ throwOnError: true }); + return false; // Healed concurrently (or transiently unreadable before). + } catch (error) { + if (!ExtensionMetadataService.isDeterministicCorruption(error)) { + return false; + } + } + const quarantinePath = `${this.filePath}.corrupt`; + // A crash-stranded sidecar may already occupy the quarantine path — + // typically the full healthy snapshot, with the corrupt main being a + // later re-corruption of a recreated file. On POSIX, rename() would + // silently REPLACE those bytes, and recovery would then reset the + // canonical file to empty, permanently destroying the recoverable + // data. Move the corrupt main aside as the bounded fixed-name + // leftover instead (same name the newer-schema swap uses; keeps the + // latest superseded file) and complete the EXISTING sidecar's + // recovery: healthy bytes restore to the canonical path, corrupt + // sidecar bytes reset it to empty exactly as before. A failing probe + // propagates so the read stays retryable on unknowable evidence. + if (await this.probeQuarantineSidecar()) { + const inflightLeftoverPath = await this.moveMainAsideAsRecreatedLeftover(); + // Same cross-process race completeQuarantineRecovery closes for the + // rename-to-.corrupt branch: another backend's atomic save can land + // a NEWER healthy main between the in-queue corruption check above + // and the move — nothing ever reads the leftover, so the newer + // update would be silently lost while the OLDER sidecar restores. + // Re-validate what actually got moved; when it turns out healthy + // (or a preserved newer schema), restore it to the vacant main path + // and merge the existing sidecar into it via the recreated-main + // reconcile. Transient read failures propagate (retryable) with the + // in-flight file left in place — finalizing UNVERIFIED bytes could + // bury a raced healthy save as the superseded leftover. Only + // deterministic corruption proceeds to the sidecar-restore path. + let movedRaw: unknown; + let movedParses = true; + try { + movedRaw = JSON.parse(await readFile(inflightLeftoverPath, "utf-8")) as unknown; + } catch (readError) { + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + movedParses = false; + } + if ( + movedParses && + (ExtensionMetadataService.isValidMetadataFileShape(movedRaw) || + ExtensionMetadataService.isUnsupportedVersion(movedRaw)) + ) { + // Restore without overwriting a file yet another writer re-created + // at the main path (EEXIST): the reconcile below merges the + // sidecar into whichever file now owns the path. + let restored = true; + try { + await link(inflightLeftoverPath, this.filePath); + } catch (linkError) { + if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + restored = false; + } else { + try { + await copyFile(inflightLeftoverPath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + // Main path missing with the raced bytes only in the + // in-flight file: rethrow (retryable) rather than letting + // the sidecar restore over the vacant path and orphan them. + throw copyError; + } + restored = false; + } + } + } + if (restored) { + // Restored to the main path: the in-flight file is a duplicate + // hard link of the live main, not a superseded leftover — drop + // it. Non-ENOENT failures propagate (retryable); the retried + // read finds the main healthy and at worst strands the + // harmless duplicate. + try { + await unlink(inflightLeftoverPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + } + // EEXIST (restored === false) does NOT prove the main-path owner + // is newer than the moved bytes: a competing recovery can restore + // the OLDER sidecar to the vacant main path first (and consume + // the sidecar, making the reconcile below a no-op). Finalizing + // the valid moved bytes to the fixed leftover would exclude them + // from stranded-file discovery and silently drop the newer + // recency/goal/status — leave them at the unique in-flight name + // instead: the stranded-leftover scan merges them by write + // generation/recency against whatever now owns the main path. + return this.reconcileRecreatedMainWithSidecar(quarantinePath); + } + // Deterministically corrupt moved bytes: proven superseded — keep + // them as the bounded fixed-name leftover. + await this.finalizeRecreatedLeftover(inflightLeftoverPath); + return this.completeQuarantineRecovery(quarantinePath); + } + await rename(this.filePath, quarantinePath); + return this.completeQuarantineRecovery(quarantinePath); + }); + } + + /** + * Move the current main file aside to an in-flight uniquely named + * `.recreated--` path and return that path. In-flight + * moved-aside bytes must never live at the shared fixed-name `.recreated` + * leftover: the mutation queue is process-local, so two backends can both + * pass the corrupt-main validation for the same sidecar, and if process A + * had moved a concurrently saved HEALTHY main to the fixed name, process + * B's finalize would unlink A's only copy before A could re-validate and + * restore it — both recoveries would then restore the OLDER sidecar, + * permanently losing the newer update. Callers finalize via + * finalizeRecreatedLeftover once the moved bytes are proven superseded, + * or unlink the in-flight file once they are restored. A crash (or a + * transient revalidation failure) can strand the file mid-recovery — + * possibly holding a raced healthy save's ONLY copy — so the leftover + * scan discovers stranded files by prefix and recovers them (see + * recoverStrandedRecreatedLeftover); the fixed name still never holds + * unverified bytes another recovery could destroy. + */ + private async moveMainAsideAsRecreatedLeftover(): Promise { + const inflightPath = `${this.filePath}${ExtensionMetadataService.RECREATED_INFIX}${process.pid}-${randomUUID()}`; + await rename(this.filePath, inflightPath); + return inflightPath; + } + + /** + * Install proven-superseded moved-aside bytes as the bounded fixed-name + * `.recreated` leftover ("keeps the latest superseded file"). Replacing + * the fixed name is safe: with every in-flight move under a unique name + * (see moveMainAsideAsRecreatedLeftover), the fixed name only ever holds + * FINALIZED superseded bytes that no recovery will re-read. The prior + * leftover is unlinked first because Windows rename onto an existing file + * is not reliably a replace (it can fail with EPERM/EEXIST), which would + * fail every strict activity read's recovery until the user removed the + * leftover by hand. A crash between the unlink and the rename only loses + * the OLDER finalized leftover. Non-ENOENT unlink failures propagate + * (retryable) — the rename would fail on the occupied destination anyway. + */ + private async finalizeRecreatedLeftover(inflightPath: string): Promise { + const leftoverPath = `${this.filePath}.recreated`; + try { + await unlink(leftoverPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + await rename(inflightPath, leftoverPath); + } + + /** + * Recover one stranded in-flight moved-aside main (unique RECREATED_INFIX + * name). The owner recovery's rename is its commit point — revalidation + * happens AFTER the move — so an owner crash (or transient revalidation + * failure) can strand a raced healthy save's ONLY copy at the unique + * name, where no fixed-path probe ever finds it; the next recovery would + * then restore the OLDER sidecar and orphan the newer + * recency/goal/status forever. Claiming may steal a LIVE owner's + * in-flight file: that is deliberate and data-preserving — the owner's + * revalidation read fails with a retryable ENOENT (never reported as + * success) while the bytes are merged or finalized here, so no copy is + * destroyed unmerged and no process-liveness probing is needed. Runs + * inside withSerializedMutation. + */ + private async recoverStrandedRecreatedLeftover(strandedPath: string): Promise { + // Claim to a fresh unique name under the same scanned prefix first: + // concurrent scanners race on the rename (ENOENT = already + // claimed/consumed) and a crash after the claim leaves the claim itself + // rediscoverable by the same prefix scan. No identity token in the name + // (unlike CLAIM_INFIX, whose claims hold VALID bytes whose + // already-merged vs foreign-generation identity is undecidable by + // parsing): here parsing decides everything — corrupt bytes are proven + // superseded, and valid bytes merge idempotently under the + // strictly-newer recency gate below, so replaying a claim after a crash + // between merge and unlink is a no-op rather than a resurrection. + const claimPath = `${this.filePath}${ExtensionMetadataService.RECREATED_INFIX}claim-${process.pid}-${randomUUID()}`; + try { + await rename(strandedPath, claimPath); + } catch (renameError) { + if (ExtensionMetadataService.isErrnoCode(renameError, "ENOENT")) { + return; + } + throw renameError; + } + let parsed: unknown; + try { + parsed = JSON.parse(await readFile(claimPath, "utf-8")) as unknown; + } catch (readError) { + // The claim path is process-unique, so any read failure other than + // deterministic parse corruption is transient: propagate (retryable; + // the claim stays discoverable). Corrupt bytes are exactly the + // validated-corrupt main the owner moved aside — keep them as the + // bounded fixed-name leftover, the same terminal state the owner + // itself would have chosen. + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + await this.finalizeRecreatedLeftover(claimPath); + return; + } + if (ExtensionMetadataService.isUnsupportedVersion(parsed)) { + // A newer build's main stranded mid-recovery: same upgrade- + // preservation swap as an unsupported sidecar (the reconcile treats + // the claim path as its sidecar argument and consumes it). + await this.reconcileRecreatedMainWithSidecar(claimPath); + return; + } + if (!ExtensionMetadataService.isValidMetadataFileShape(parsed)) { + // Parseable but not a metadata file: proven superseded garbage. + await this.finalizeRecreatedLeftover(claimPath); + return; + } + // Upper bound on the write time of EVERY entry in the stranded + // snapshot: rename() preserves mtime, so the claim file still carries + // the owner's last save time. The equal-recency ordering below compares + // generation-carrying entries (epoch-ms stamps) against it. Stat + // failures on the process-unique claim are transient: propagate + // (retryable; the claim stays discoverable). + const strandedMtimeMs = (await stat(claimPath)).mtimeMs; + // Tombstone revalidation for candidate ids, following the sidecar + // reconcile's lift-or-suppress generation contract. Safe to sample + // before the main read below: lifts only update the in-memory + // suppression map (never the file), and the merge re-checks live + // tombstone state per entry. + if (this.registrationProbe != null) { + for (const workspaceId of Object.keys(parsed.workspaces)) { + if (!this.deletedWorkspaceIds.has(workspaceId)) { + continue; + } + const generationBefore = this.deletedWorkspaceIds.get(workspaceId); + const registered = await this.registrationProbe(workspaceId); + if (registered && this.deletedWorkspaceIds.get(workspaceId) === generationBefore) { + this.liftTombstone(workspaceId); + } + } + } + let main: ExtensionMetadataFile; + try { + const mainParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; + if (!ExtensionMetadataService.isValidMetadataFileShape(mainParsed)) { + // Corrupt or newer-schema main: leave the claim for a later pass + // (it stays under the scanned prefix) rather than merging across + // schemas. + return; + } + main = mainParsed; + } catch (readError) { + if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { + // Missing-main window: the resumable sidecar recovery owns the + // path right now; the claim stays discoverable for the next pass. + return; + } + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + return; + } + // Adoption evidence for candidates MISSING from the loaded main, + // gathered strictly AFTER the load (the concurrency contract's + // post-load evidence rule for destructive/resurrecting decisions): the + // stranded snapshot predates the current main, so a missing entry may + // mean another backend REMOVED the workspace — and a positive probe + // sampled BEFORE the load can go stale when that removal lands during + // the probe awaits, letting the stale positive adopt (resurrect) the + // removed workspace's goal/status indefinitely (an unscoped older + // build exposes it, and a re-registered deterministic legacy id would + // inherit it). These awaits hold the loaded snapshot across config + // reads, widening the window in which the save below clobbers a + // concurrent backend's write — that window exists on every load→save + // slot here and is the contract's documented out-of-scope gap, while + // stale-evidence resurrection is exactly the harm the probe exists to + // prevent, so evidence freshness wins. Without a wired probe, + // missing-target adoption stays fail-open (the sidecar reconcile's + // sidecar-only adoption contract; dropping is destructive and + // production always wires the probe). A FAILING probe propagates + // (claim retained, retryable) rather than consuming the stranded bytes + // on unknowable evidence. + const registrationEvidence = new Map(); + for (const workspaceId of Object.keys(parsed.workspaces)) { + if (this.deletedWorkspaceIds.has(workspaceId) || workspaceId in main.workspaces) { + // Tombstoned ids are skipped by the merge below; existing targets + // are ordered newest-wins, which needs no registration evidence. + continue; + } + if (this.registrationProbe == null) { + registrationEvidence.set(workspaceId, true); + continue; + } + registrationEvidence.set(workspaceId, await this.registrationProbe(workspaceId)); + } + let modified = false; + for (const [workspaceId, entry] of Object.entries(parsed.workspaces)) { + if (this.deletedWorkspaceIds.has(workspaceId)) { + // Still tombstoned after the revalidation pass above: the local + // removal knowledge stands. + continue; + } + if (entry === null || typeof entry !== "object" || Array.isArray(entry)) { + continue; + } + const candidateRecency = (entry as { recency?: unknown }).recency; + if (typeof candidateRecency !== "number") { + // Unorderable candidate: keep the main entry (fail closed). + continue; + } + if (!(workspaceId in main.workspaces)) { + const registered = registrationEvidence.get(workspaceId); + if (registered === undefined) { + // No post-load evidence for this missing target: its tombstone + // was lifted between the evidence pass above and this check + // (writers' pre-queue rechecks run outside this queue slot). + // Abort without saving or consuming: the claim stays + // discoverable and the next scan probes the id afresh. Earlier + // candidates' merges replay idempotently. + return; + } + if (!registered) { + // Proven removed: do not resurrect the deleted entry. + continue; + } + } + const target: unknown = main.workspaces[workspaceId]; + const targetIsObject = + target !== null && typeof target === "object" && !Array.isArray(target); + const targetRecency = targetIsObject ? (target as { recency?: unknown }).recency : undefined; + const targetGeneration = targetIsObject + ? (target as { writeGeneration?: unknown }).writeGeneration + : undefined; + const candidateGeneration = (entry as { writeGeneration?: unknown }).writeGeneration; + // Entry-level newest-wins, NOT the sidecar reconcile's main-wins + // field merge: the stranded bytes are a complete healthy main + // snapshot whose age relative to the CURRENT main is unknowable per + // field (the current main may be an older sidecar restore plus newer + // writes). Ordering, in precedence order: + // - per-entry writeGeneration when both copies carry distinct ones + // (advanced by EVERY persisted mutation — recency alone cannot + // order metadata changes because status/goal/streaming writers + // deliberately preserve it); + // - strict recency otherwise; + // - at EQUAL recency with a generation-carrying TARGET facing a + // generation-LESS candidate, the stranded file's mtime decides: the + // stamp is epoch-ms (see nextWriteGeneration) and rename preserves + // mtime, so a target stamp strictly above strandedMtimeMs proves + // the target's write postdates every byte of the stranded snapshot + // (e.g. this build mutated goal/status AFTER a pre-generation main + // was stranded) — keep the target; no later mutation is guaranteed + // to repair a wrong overwrite. Otherwise the order is genuinely + // unknowable and the generation-less candidate wins: a downgraded + // build's writers drop writeGeneration from the entry they mutate, + // so the candidate may be that build's LATER goal/status write + // whose only surviving copy is here — dropping it (and unlinking + // the claim) would lose the downgrade's update permanently + // (upgrade↔downgrade preservation), while wrongly preferring an + // ancient copy only resurrects stale metadata that the next status + // write or regeneration self-heals. Coarse-mtime filesystems only + // widen the ambiguous branch, never the destructive one. A + // generation-less TARGET keeps against a generation-carrying + // candidate for the same downgrade-preservation reason (the main + // path is also the actively-written copy the next local mutation + // lands on). Ties with no generation information keep the target, + // so crash replay of an already-merged claim stays a no-op (after + // adopting a generation-less candidate the main entry is + // generation-less too); the replay window can re-adopt over an + // interleaved same-recency local write, but it is bounded by the + // claim's lifetime (crash between merge and unlink) and self-heals + // like any stale status. + const candidateNewer = + typeof candidateGeneration === "number" && + typeof targetGeneration === "number" && + candidateGeneration !== targetGeneration + ? candidateGeneration > targetGeneration + : typeof targetRecency !== "number" + ? true + : targetRecency !== candidateRecency + ? candidateRecency > targetRecency + : typeof targetGeneration === "number" && + typeof candidateGeneration !== "number" && + targetGeneration <= strandedMtimeMs; + if (!candidateNewer) { + continue; + } + // Cross-process crash leftover rule (same as sidecar-only entries): a + // truthy streaming flag from a stranded file must not pin the + // workspace "streaming" forever; a genuinely streaming workspace + // re-asserts the flag with its next write. + main.workspaces[workspaceId] = + (entry as { streaming?: unknown }).streaming === true + ? { ...entry, streaming: false } + : entry; + modified = true; + } + if (modified) { + await this.save(main); + } + // The claim path is process-unique: nothing else consumes it, so a + // plain unlink suffices (no token verification needed — see the claim + // comment above for why replay is idempotent anyway). + try { + await unlink(claimPath); + } catch (unlinkError) { + if (!ExtensionMetadataService.isErrnoCode(unlinkError, "ENOENT")) { + throw unlinkError; + } + } + } + + /** + * Finish a quarantine whose main file was already moved to the sidecar: + * restore the sidecar to the main path when its bytes turn out healthy, + * otherwise leave the corrupt bytes quarantined and reset the main path to + * a valid empty file. Factored out of quarantineCorruptFile so a recovery + * interrupted by a crash between the rename and this completion can be + * RESUMED on the next strict read (see resumeQuarantineRecovery) instead + * of leaving strict reads failing until an unrelated writer saves. + * Must run inside withSerializedMutation. + */ + private async completeQuarantineRecovery(quarantinePath: string): Promise { + // The mutation queue is process-local: another backend's atomic save + // can land a healthy file between the validation above and the rename, + // and the rename would move THAT file aside. Re-validate the bytes that + // actually got moved and undo the move when they turn out healthy. + // Only deterministic parse corruption is the expected quarantine + // outcome here; a transient failure reading the sidecar means the + // moved bytes cannot be verified, so it propagates (retryable) rather + // than reporting a successful reset over possibly-healthy data. + // Generation identity captured before the read: consumption below is + // scoped to exactly these bytes (see consumeQuarantineSidecar). + const sidecarToken = await ExtensionMetadataService.statQuarantineToken(quarantinePath); + let moved: unknown; + let movedParses = true; + try { + moved = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; + // Bind the token to the bytes just read (same contract as + // reconcileRecreatedMainWithSidecar): another recovery can consume + // the captured generation and install a newer one between the stat + // and the read. Restoring the new bytes under the OLD token would + // double-apply them — the consume claims the new generation, sees + // the mismatch, and replays it, re-filling fields another backend + // explicitly cleared to null in between. On mismatch leave the file + // for its own recovery pass (the retried read resumes with a fresh + // token). + const postReadToken = await ExtensionMetadataService.statQuarantineToken(quarantinePath); + if ( + sidecarToken == null || + postReadToken == null || + !ExtensionMetadataService.tokensMatch(sidecarToken, postReadToken) + ) { + return false; + } + } catch (readError) { + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + movedParses = false; + } + if ( + movedParses && + (ExtensionMetadataService.isValidMetadataFileShape(moved) || + // A newer build's schema stranded in the sidecar (crash-interrupted + // quarantine on a downgraded install, or a newer backend saving the + // main file between the in-queue corruption check and the rename) is + // preserved data, not corruption: restore it instead of falling + // through to the empty-reset below, which would hand the newer build + // an empty canonical file. The subsequent re-read then fails with + // the non-destructive unsupported-version signal. + ExtensionMetadataService.isUnsupportedVersion(moved)) + ) { + // Restore without ever overwriting a newer file yet another writer + // may have re-created at the main path: link and COPYFILE_EXCL both + // fail with EEXIST in that case. EEXIST is NOT a successful recovery + // though — the re-created file may be a PARTIAL snapshot an older + // backend self-healed from the missing-main window (older builds read + // ENOENT as empty and save their one mutated entry), so the sidecar's + // other entries must be reconciled into it rather than abandoned. + try { + await link(quarantinePath, this.filePath); + } catch (linkError) { + if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + return this.reconcileRecreatedMainWithSidecar(quarantinePath); + } + try { + // Filesystems without hard-link support (or EPERM): copy-based + // restore with the same EEXIST no-overwrite guarantee. + await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + return this.reconcileRecreatedMainWithSidecar(quarantinePath); + } + // Restore failed with the main path missing: rethrow so the + // strict reader propagates a retryable failure instead of + // reading ENOENT as an authoritative empty state while the + // healthy bytes sit in the sidecar. + throw copyError; + } + } + await this.consumeQuarantineSidecar(quarantinePath, sidecarToken); + return false; + } + // Replace the quarantined main file with a valid EMPTY file instead of + // leaving the path missing: between the rename above and here, strict + // readers (this or another process) would otherwise observe ENOENT. + // A missing-main window is dangerous because load() treats plain + // ENOENT as a healthy empty state — combined with a concurrent writer + // repairing the file right before the rename, a reader in the gap + // could return an authoritative {} that a later restore cannot + // retract. With a real empty file the corrupt→empty transition is + // atomic (link/COPYFILE_EXCL below never overwrite a file a + // concurrent writer re-created first), and load()'s sidecar check + // turns any remaining missing-main window into a retryable failure + // instead of an empty read. + // Process-unique temp path: with a fixed shared name, a concurrent + // recovery's writeFile could truncate the inode right after this one + // links it into the canonical path (the shared temp path then aliases + // the canonical file, so the truncate empties BOTH and strict readers + // observe empty/partial JSON), or its cleanup unlink could remove the + // temp between this writeFile and link, failing a recovery whose + // sidecar is still recoverable. Crash leftovers are inert: the suffix + // matches no probe or scan prefix, each crashed recovery leaves at most + // one tiny file, and no sweeper may reclaim them (unlinking another + // process's in-flight temp is exactly the race this name prevents). + const emptyTmpPath = `${this.filePath}.empty-${process.pid}-${randomUUID()}.tmp`; + await writeFile( + emptyTmpPath, + JSON.stringify({ version: 1, workspaces: {} } satisfies ExtensionMetadataFile, null, 2), + "utf-8" + ); + try { + try { + await link(emptyTmpPath, this.filePath); + } catch (linkError) { + if (!ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + try { + await copyFile(emptyTmpPath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (!ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + // Main path still missing: rethrow (retryable) so the caller's + // re-read hits the sidecar guard instead of reading ENOENT. + throw copyError; + } + } + } + } + } finally { + // Caller-owned path only; best-effort on every exit so throw paths do + // not strand the temp. + await unlink(emptyTmpPath).catch(() => undefined); + } + log.error( + `Extension metadata file was corrupt; moved it to ${quarantinePath} and reset to empty` + ); + return true; + } + + /** + * Resume a quarantine that a crash interrupted between quarantineCorruptFile's + * rename and its completion: the main file is missing while the sidecar still + * holds the moved bytes. Without this, every strict read rethrows the ENOENT + * (load()'s sidecar guard classifies it retryable) until an unrelated writer + * happens to save — activity hydration would retry forever on an idle + * process. No-op when the main file reappeared or no sidecar exists. + */ + private async resumeQuarantineRecovery(): Promise { + await this.withSerializedMutation(async () => { + // Re-check inside the queue: a concurrent writer save or a sibling + // strict read may already have recovered the main path. Probe with + // ENOENT-only absence semantics: a transiently unprobeable sidecar + // (EACCES/EIO) must propagate — reporting it absent would let the + // caller accept a recreated partial main and (for the once-per-process + // getAllSnapshots check) never look at the sidecar again. + const quarantinePath = `${this.filePath}.corrupt`; + if (!(await this.probeQuarantineSidecar())) { + return; + } + const mainExists = await access(this.filePath).then( + () => true, + () => false + ); + if (mainExists) { + // Main was recreated during the crash window — possibly a PARTIAL + // file another backend self-healed from the missing-main state. + // Merge the sidecar's other entries back instead of abandoning them. + await this.reconcileRecreatedMainWithSidecar(quarantinePath); + return; + } + await this.completeQuarantineRecovery(quarantinePath); + }); + } + + /** + * A restore found the main path already re-created (EEXIST), or a resumed + * recovery found both files present. The re-created file may be a PARTIAL + * snapshot an older backend self-healed from the missing-main window; + * treating it as authoritative would permanently hide every other + * workspace's recency/goal/status while the full data sits in the sidecar. + * Merge sidecar entries into the main file (per FIELD: main wins fields it + * carries non-null values for — its writes are newer — while null/absent + * fields fill from the sidecar's complete pre-crash entry; ids this + * process write-tombstoned stay out) and consume the sidecar. Only + * same-schema (version 1) sidecars can be merged: a + * corrupt sidecar is left as the bounded fixed-name leftover, while a + * newer-schema sidecar is restored to the canonical path (superseding the + * recreated file, preserved as its own leftover). Must run inside + * withSerializedMutation. Returns false (no empty reset happened). + */ + private async reconcileRecreatedMainWithSidecar(quarantinePath: string): Promise { + // Generation identity captured before the read: consumption below is + // scoped to exactly these bytes (see consumeQuarantineSidecar). + const sidecarToken = await ExtensionMetadataService.statQuarantineToken(quarantinePath); + let sidecarParsed: unknown; + try { + sidecarParsed = JSON.parse(await readFile(quarantinePath, "utf-8")) as unknown; + // Bind the token to the bytes just read: another recovery can consume + // the captured generation and install a NEWER one at the fixed path + // between the stat and the read. Proceeding would merge the new bytes + // under the OLD token — the consume then claims the new generation, + // sees the token mismatch, and replays the same bytes a second time, + // re-filling fields another backend explicitly cleared to null in + // between (the null-fill merge is not idempotent across clears). + // On mismatch leave the file for its own recovery pass (next read). + const postReadToken = await ExtensionMetadataService.statQuarantineToken(quarantinePath); + if ( + sidecarToken == null || + postReadToken == null || + !ExtensionMetadataService.tokensMatch(sidecarToken, postReadToken) + ) { + return false; + } + } catch (readError) { + // Sidecar gone: a concurrent recovery consumed it and the recreated + // main is all there is — nothing left to reconcile. + if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { + return false; + } + // Transient I/O failure (EACCES/EIO/...): the sidecar cannot be + // verified, and reporting success would make the caller accept the + // possibly-partial recreated main while the healthy sidecar is never + // inspected again (loads stop probing it once the main path exists). + // Propagate so the strict read stays retryable; only deterministic + // parse corruption below stays as the bounded fixed-name leftover. + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + return false; + } + if (ExtensionMetadataService.isUnsupportedVersion(sidecarParsed)) { + // A newer build's schema stranded in the sidecar while an older-schema + // backend re-created the main path (usually a partial ENOENT self-heal + // holding one freshly mutated entry). Accepting the recreated file + // would lose the newer data permanently: nothing re-inspects the + // sidecar once the main path exists, so even re-upgrading reads the + // partial file — and a later quarantine's rename would destroy the + // sidecar bytes. Restore the newer bytes to the canonical path + // (upgrade↔downgrade preservation; this build's readers then see the + // same non-destructive retryable unsupported-version signal as any + // downgrade overlap) and preserve the recreated file as a bounded + // fixed-name leftover. A crash between any of the steps leaves the + // resumable missing-main + sidecar state, which restores the + // unsupported sidecar via completeQuarantineRecovery. + // + // Inspect the canonical bytes FIRST: during multi-version overlap the + // canonical path may itself hold an unsupported file whose version is + // same-or-newer than the sidecar's (e.g. a v3 writer recreated it + // while a v2 sidecar remained). This build cannot order or merge + // foreign schemas beyond their version numbers, so swapping blindly + // would park the possibly-newer canonical copy at the unscanned + // fixed leftover and restore older data over it. Keep the canonical + // file in place and RETAIN the sidecar for a build that understands + // both schemas (same retention precedent as a deterministically + // corrupt sidecar: one no-op reconcile per read while the overlap + // lasts). A transiently unreadable canonical propagates (retryable); + // a missing or corrupt canonical proceeds with the swap exactly as + // before. + let canonicalParsed: unknown = null; + try { + canonicalParsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; + } catch (canonicalReadError) { + if ( + !ExtensionMetadataService.isErrnoCode(canonicalReadError, "ENOENT") && + !ExtensionMetadataService.isDeterministicCorruption(canonicalReadError) + ) { + throw canonicalReadError; + } + } + if ( + ExtensionMetadataService.isUnsupportedVersion(canonicalParsed) && + (canonicalParsed as { version: number }).version >= + (sidecarParsed as { version: number }).version + ) { + return false; + } + // The moved bytes are superseded BY DECISION (the newer schema wins), + // so they finalize to the fixed leftover name immediately — no + // re-validation pass ever re-reads them. + const inflightLeftoverPath = await this.moveMainAsideAsRecreatedLeftover(); + await this.finalizeRecreatedLeftover(inflightLeftoverPath); + try { + await link(quarantinePath, this.filePath); + } catch (linkError) { + if (ExtensionMetadataService.isErrnoCode(linkError, "EEXIST")) { + // Yet another writer re-created the main path mid-swap: re-enter + // with the new file (the leftover keeps the latest superseded one). + return this.reconcileRecreatedMainWithSidecar(quarantinePath); + } + try { + await copyFile(quarantinePath, this.filePath, constants.COPYFILE_EXCL); + } catch (copyError) { + if (ExtensionMetadataService.isErrnoCode(copyError, "EEXIST")) { + return this.reconcileRecreatedMainWithSidecar(quarantinePath); + } + // Main path missing with the newer bytes still in the sidecar: + // rethrow (retryable) — the resumed recovery restores them. + throw copyError; + } + } + await this.consumeQuarantineSidecar(quarantinePath, sidecarToken); + return false; + } + if (!ExtensionMetadataService.isValidMetadataFileShape(sidecarParsed)) { + return false; + } + // Resolve tombstone revalidations BEFORE reading the main file: the + // probes await config, and holding a pre-probe main snapshot across + // those awaits would let the save below write stale data over a + // concurrent backend's newer write. The tombstone may be stale (a + // downgraded backend can re-register a deterministic legacy id after + // this process pruned it), and the sidecar may hold that workspace's + // ONLY copy — consumed below, so a wrong drop is permanent, unlike + // write suppression, which self-heals. Without a probe the local + // removal knowledge stands; a FAILING probe aborts the reconcile + // (retryable) rather than consuming the sidecar on unknowable evidence. + // Same generation contract as recheckTombstonedRegistration: a + // tombstone republished mid-probe survives the stale positive. + for (const workspaceId of Object.keys(sidecarParsed.workspaces)) { + if (!this.deletedWorkspaceIds.has(workspaceId) || this.registrationProbe == null) { + continue; + } + const generationBefore = this.deletedWorkspaceIds.get(workspaceId); + const registered = await this.registrationProbe(workspaceId); + if (registered && this.deletedWorkspaceIds.get(workspaceId) === generationBefore) { + this.liftTombstone(workspaceId); + } + } + let main: ExtensionMetadataFile; + try { + const parsed = JSON.parse(await readFile(this.filePath, "utf-8")) as unknown; + if (!ExtensionMetadataService.isValidMetadataFileShape(parsed)) { + // Corrupt/newer-schema main: leave both files for the normal read + // classification paths rather than merging across schemas. + return false; + } + main = parsed; + } catch (readError) { + // Main vanished again mid-reconcile: back to the resumable + // missing-main state the normal read paths already handle. + if (ExtensionMetadataService.isErrnoCode(readError, "ENOENT")) { + return false; + } + // Same transient-I/O contract as the sidecar read above: an + // unverifiable main must stay retryable, not report success. + if (!ExtensionMetadataService.isDeterministicCorruption(readError)) { + throw readError; + } + return false; + } + let modified = false; + for (const [workspaceId, entry] of Object.entries(sidecarParsed.workspaces)) { + if (this.deletedWorkspaceIds.has(workspaceId)) { + // Still tombstoned after the (pre-main-read) revalidation pass: + // the local removal knowledge stands. + continue; + } + if (!(workspaceId in main.workspaces)) { + // Sidecar-only entry: by definition no writer touched it since the + // quarantine (a live workspace's writes land in the recreated main), + // so a truthy streaming flag is a crash leftover. initialize()'s + // clearStaleStreaming already ran against the main file before this + // reconcile, so merging the flag verbatim would leave the workspace + // "streaming" forever; a genuinely streaming workspace re-asserts + // the flag with its next write. + main.workspaces[workspaceId] = + entry !== null && + typeof entry === "object" && + !Array.isArray(entry) && + (entry as { streaming?: unknown }).streaming === true + ? { ...entry, streaming: false } + : entry; + modified = true; + continue; + } + // Same id on both sides: the recreated main entry is commonly a + // PARTIAL self-heal (e.g. a recency write initializes every other + // field to its default), so treating it as wholly authoritative would + // discard the sidecar entry's goal/status/model fields. Field-level + // merge instead: main wins every field it carries a non-null value + // for (its writes are newer); null/absent fields fill from the + // sidecar. An explicit pre-crash clear (null) can be re-filled with a + // stale value — the lesser loss than dropping every unaffected field + // (upgrade↔downgrade data preservation). `streaming` is never filled + // from the sidecar for the crash-leftover reason above. + const target: unknown = main.workspaces[workspaceId]; + const sidecarEntry: unknown = entry; + if ( + sidecarEntry !== null && + typeof sidecarEntry === "object" && + !Array.isArray(sidecarEntry) && + (target === null || typeof target !== "object" || Array.isArray(target)) + ) { + // Uncoercible main entry (null/primitive/array) shadowing a healthy + // sidecar entry: no field merge is possible, and the sidecar is + // consumed below — leaving the malformed value would permanently + // lose the only valid copy. Restore the sidecar entry (streaming + // cleared, same crash-leftover rule as sidecar-only entries). + main.workspaces[workspaceId] = + (sidecarEntry as { streaming?: unknown }).streaming === true + ? { ...entry, streaming: false } + : entry; + modified = true; + continue; + } + if ( + target !== null && + typeof target === "object" && + !Array.isArray(target) && + sidecarEntry !== null && + typeof sidecarEntry === "object" && + !Array.isArray(sidecarEntry) + ) { + const targetRecord = target as Record; + for (const [field, value] of Object.entries(sidecarEntry)) { + if (field === "streaming" || value == null) { + continue; + } + if (targetRecord[field] == null) { + targetRecord[field] = value; + modified = true; + } + } + } + } + if (modified) { + await this.save(main); + } + // Consumed either way: every surviving sidecar entry is now represented + // at the main path. + await this.consumeQuarantineSidecar(quarantinePath, sidecarToken); + return false; + } + + /** + * load() with self-healing for deterministic corruption. Strict reads must + * propagate transient failures (renderer keeps last-known state and + * retries), but deterministic corruption would fail every retry forever on + * an idle process — no subsequent list read or metadata writer is + * guaranteed to repair the file. Quarantine the corrupt bytes and re-read: + * the post-quarantine state is authoritative. Shared by getAllSnapshots + * and getSnapshot so per-workspace emit reads (workflow-run/bash-monitor + * handlers, which drop their emissions on error) self-heal the same way + * instead of leaving a workflow-only workspace's stale activity pinned in + * the renderer indefinitely. + */ + private async loadWithCorruptionRecovery(options?: { + throwOnError?: boolean; + }): Promise { + try { + return await this.load(options); + } catch (error) { + if (ExtensionMetadataService.isDeterministicCorruption(error)) { + try { + await this.quarantineCorruptFile(); + } catch (quarantineError) { + // ENOENT means the file vanished between validation and rename + // (another process moved it) — the re-read below decides the + // outcome. Anything else (rename denied, sidecar unverifiable, + // restore failed) must stay a retryable failure: an ENOENT re-read + // would masquerade as authoritative empty while the moved bytes may + // hold healthy data. + if (!ExtensionMetadataService.isErrnoCode(quarantineError, "ENOENT")) { + throw quarantineError; + } + } + } else if (ExtensionMetadataService.isErrnoCode(error, "ENOENT")) { + // load() only propagates ENOENT while the quarantine sidecar exists: + // a crash between quarantine's rename and its completion left the + // recovery half-done, and nothing else finishes it (lenient writer + // reads self-heal in memory without saving). Resume it here so + // strict reads stop failing on every retry across restarts. + await this.resumeQuarantineRecovery(); + } else { + throw error; + } + return this.load(options); + } + } + + async getAllSnapshots(options?: { + throwOnError?: boolean; + }): Promise> { + let data: ExtensionMetadataFile = await this.loadWithCorruptionRecovery(options); + // A crash between quarantineCorruptFile's rename and its completion can + // strand the full snapshot in the sidecar while another backend + // recreates a VALID (typically partial, single-entry) main file from + // the missing-main window — leaving no ENOENT or corruption for the + // other recovery triggers. Quarantines are cross-process, so this can + // happen at ANY point in this process's lifetime, not just before + // startup: probe the fixed sidecar path on every authoritative read (a + // process-lifetime latch would hide a sidecar stranded after its first + // read until restart). The probe is one access() syscall next to the + // full-file read this method already does; the reconcile only runs when + // a sidecar actually exists. Failures propagate so the read stays + // retryable rather than presenting a partial file as authoritative. A + // deterministically corrupt leftover (kept for inspection by design) + // costs one no-op reconcile per read while it exists — corruption + // incidents are rare and the leftover is consumed by the next + // quarantine or manual cleanup. + if (await this.reconcileLeftoverSidecarIfPresent()) { + data = await this.load(options); + } const map = new Map(); for (const [workspaceId, entry] of Object.entries(data.workspaces)) { const snapshot = this.toSnapshot(entry); diff --git a/src/node/services/coreServices.ts b/src/node/services/coreServices.ts index 5e8e019188..270d57e0cc 100644 --- a/src/node/services/coreServices.ts +++ b/src/node/services/coreServices.ts @@ -98,6 +98,45 @@ export function createCoreServices(opts: CoreServicesOptions): CoreServices { providerService.getConfig() ); const extensionMetadata = new ExtensionMetadataService(extensionMetadataPath); + // Write tombstones are process-local removal knowledge; the shared config + // is the authority (with XUM_ALLOW_MULTIPLE_INSTANCES a downgraded backend + // can legitimately re-register a deterministic legacy id this process + // pruned). Without this probe, a tombstoned id that becomes active again + // would have every metadata write and broadcast suppressed until an + // activity bootstrap happens to run. Raw view first (cheap; complete when + // every persisted entry carries an inline id); only id-less legacy entries + // require the authoritative enumeration. Throws propagate: unknowable + // registration keeps the tombstone. + extensionMetadata.setRegistrationProbe(async (workspaceId) => { + const evidence = config.readPersistedWorkspaceIdEvidence(); + if (evidence.ids.has(workspaceId)) { + return true; + } + if (!evidence.hasWorkspaceEntriesWithoutIds) { + return false; + } + // Targeted lenient positive first: a POSITIVE identity match needs no + // completeness, so a re-registered workspace whose own compatibility + // metadata is healthy must not stay write-suppressed because an + // UNRELATED legacy entry's metadata is malformed (the strict + // enumeration below throws on the first such entry, and the tombstone + // would then pin every one of the target's writes as transient + // indefinitely). A lenient scan only skips unreadable entries — it + // never fabricates a match. + if (config.findWorkspace(workspaceId) != null) { + return true; + } + // Negatives keep requiring the complete strict view: a lenient miss is + // indistinguishable from an identity hidden by a read failure. Alias + // ids: a second resolvable compatibility file's identity stays + // registered for findWorkspace even though it is not any entry's + // primary id — refusing its writes/deletions requires knowing it here. + const legacyAliasIds = new Set(); + const registered = ( + await config.getAllWorkspaceMetadata({ throwOnError: true, legacyAliasIds }) + ).some((metadata) => metadata.id === workspaceId); + return registered || legacyAliasIds.has(workspaceId); + }); const workspaceGoalService = new WorkspaceGoalService( config, historyService, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index b4d4131a4d..ec60d1d277 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -592,6 +592,7 @@ function createWorkspaceServiceMocks( emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; create: ReturnType; + discardExtensionMetadataEntry: ReturnType; } { const sendMessage = overrides?.sendMessage ?? mock((): Promise> => Promise.resolve(Ok(undefined))); @@ -678,10 +679,12 @@ function createWorkspaceServiceMocks( (): Promise> => Promise.resolve(Err("workspaceService.create not mocked")) ); + const discardExtensionMetadataEntry = mock((): Promise => Promise.resolve()); return { workspaceService: { create, + discardExtensionMetadataEntry, // No-op by default: task-create tests exercise launch flow, not the // registration-time plugin-override sanitizer (workspaceService.test.ts // covers it). Returning undefined means "clean". @@ -731,6 +734,7 @@ function createWorkspaceServiceMocks( countQueuedAgentPeerMessages, } as unknown as WorkspaceService, create, + discardExtensionMetadataEntry, sendMessage, resumeStream, clearQueue, @@ -23801,7 +23805,9 @@ describe("TaskService", () => { ); const { aiService } = createAIServiceMocks(config); const failingSendMessage = mock(() => Promise.resolve(Err("send failed"))); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage: failingSendMessage }); + const { workspaceService, discardExtensionMetadataEntry } = createWorkspaceServiceMocks({ + sendMessage: failingSendMessage, + }); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); const created = await createAgentTask(taskService, parentId, "do the thing"); @@ -23814,6 +23820,11 @@ describe("TaskService", () => { .some((w) => w.id === "aaaaaaaaaa"); expect(stillExists).toBe(false); + // Rollback must also drop the extension-metadata entry: the failed send + // may already have scheduled metadata writes that would otherwise leak a + // stale key after deregistration (#3959). + expect(discardExtensionMetadataEntry).toHaveBeenCalledWith("aaaaaaaaaa"); + const workspaceName = "agent_explore_aaaaaaaaaa"; const workspacePath = runtime.getWorkspacePath(projectPath, workspaceName); let workspacePathExists = true; @@ -23825,6 +23836,63 @@ describe("TaskService", () => { expect(workspacePathExists).toBe(false); }, 20_000); + test("failed config deregistration during rollback does not tombstone the task's metadata", async () => { + const config = await createTestConfig(rootDir); + stubStableIds(config, ["bbbbbbbbbb"], "bbbbbbbbbb"); + + const projectPath = await createTestProject(rootDir); + + const runtimeConfig = { type: "worktree" as const, srcBaseDir: config.srcDir }; + const runtime = createRuntime(runtimeConfig, { projectPath }); + const initLogger = createNullInitLogger(); + + const parentName = "parent-b"; + const parentCreate = await runtime.createWorkspace({ + projectPath, + branchName: parentName, + trunkBranch: "main", + directoryName: parentName, + initLogger, + }); + expect(parentCreate.success).toBe(true); + + const parentId = "2222222222"; + const parentPath = runtime.getWorkspacePath(projectPath, parentName); + + await saveWorkspaces( + config, + projectPath, + [ + { + path: parentPath, + id: parentId, + name: parentName, + createdAt: new Date().toISOString(), + runtimeConfig, + }, + ], + testTaskSettings() + ); + const { aiService } = createAIServiceMocks(config); + const failingSendMessage = mock(() => Promise.resolve(Err("send failed"))); + const { workspaceService, discardExtensionMetadataEntry } = createWorkspaceServiceMocks({ + sendMessage: failingSendMessage, + }); + const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + // Deregistration fails: the rollback must NOT discard (and thereby + // write-tombstone) metadata for a workspace that is still registered. + const removeSpy = spyOn(config, "removeWorkspace").mockImplementation(() => + Promise.reject(new Error("config locked")) + ); + try { + const created = await createAgentTask(taskService, parentId, "do the thing"); + expect(created.success).toBe(false); + expect(discardExtensionMetadataEntry).not.toHaveBeenCalled(); + } finally { + removeSpy.mockRestore(); + } + }, 20_000); + test("agent_report posts report to parent, finalizes pending task tool output, and triggers cleanup", async () => { const config = await createTestConfig(rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 7e43d21fde..16ff389490 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7312,8 +7312,10 @@ export class TaskService { preservePhysicalWorkspace?: boolean; } ): Promise { + let removedFromConfig = false; try { await this.config.removeWorkspace(taskId); + removedFromConfig = true; } catch (error: unknown) { log.error("Task.create rollback: failed to remove workspace from config", { taskId, @@ -7321,6 +7323,17 @@ export class TaskService { }); } + // A create that failed after sendMessage may already have scheduled + // extension-metadata writes (e.g. the recency update), which would + // recreate the entry after the deregistration above and leak a stale key + // until the next process start's lazy prune. Only after deregistration + // actually succeeded: discarding also write-tombstones the id for this + // process, which must not silence metadata for a workspace that is still + // registered because removeWorkspace failed. + if (removedFromConfig) { + await this.workspaceService.discardExtensionMetadataEntry(taskId); + } + this.workspaceService.emit("metadata", { workspaceId: taskId, metadata: null }); if (options?.preservePhysicalWorkspace) { diff --git a/src/node/services/workspaceGoalService.test.ts b/src/node/services/workspaceGoalService.test.ts index 22f828b9ad..d4267a25d0 100644 --- a/src/node/services/workspaceGoalService.test.ts +++ b/src/node/services/workspaceGoalService.test.ts @@ -2298,6 +2298,32 @@ describe("WorkspaceGoalService", () => { }); }); + test("previewStreamAccounting skips the durable fallback when the strict baseline read is unavailable", async () => { + // "unavailable" (failed sidecar reconcile) must stay distinct from the + // authoritative "no baseline": the durable pushSnapshot fallback writes + // through the lenient load — accepting the suspect partial main the + // strict read refused — and emits it, clearing renderer goal/status + // state. The preview must resolve without delivering or writing. + await setGoalOk(service, { workspaceId, objective: "Preview goal" }); + const metadataFilePath = path.join(config.rootDir, "extensionMetadata.json"); + const before = await fs.readFile(metadataFilePath, "utf-8"); + // A directory at the sidecar path yields a deterministic errno (EISDIR) + // standing in for EACCES/EIO-class reconcile failures. + await fs.mkdir(`${metadataFilePath}.corrupt`); + try { + const activityUpdates = captureGoalActivity(service); + + const preview = await service.previewStreamAccounting({ workspaceId, costUsd: 1 }); + + expect(preview).toMatchObject({ objective: "Preview goal" }); + // No emit (renderer keeps last-known state) and no durable write. + expect(activityUpdates).toHaveLength(0); + expect(await fs.readFile(metadataFilePath, "utf-8")).toBe(before); + } finally { + await fs.rm(`${metadataFilePath}.corrupt`, { recursive: true }); + } + }); + test("successful no-op queued drains clear the pending snapshot", async () => { const created = await setGoalOk(service, { workspaceId, objective: "Existing goal" }); await extensionMetadata.setStreaming(workspaceId, true); @@ -5470,7 +5496,14 @@ describe("WorkspaceGoalService", () => { objective: "Preview without metadata", budgetCents: 1_000, }); - await extensionMetadata.deleteWorkspace(workspaceId); + // Clear the snapshot by rewriting the file directly: deleteWorkspace now + // write-tombstones removed workspaces for the rest of the process, which + // would (correctly) block the preview persistence below. This test + // simulates a LIVE workspace that merely has no activity snapshot yet. + await fs.writeFile( + path.join(config.rootDir, "extensionMetadata.json"), + JSON.stringify({ version: 1, workspaces: {} }) + ); const activityUpdates = captureGoalActivity(service); const preview = await service.previewStreamAccounting({ diff --git a/src/node/services/workspaceGoalService.ts b/src/node/services/workspaceGoalService.ts index 5b220db50f..db8539a29b 100644 --- a/src/node/services/workspaceGoalService.ts +++ b/src/node/services/workspaceGoalService.ts @@ -1280,24 +1280,41 @@ export class WorkspaceGoalService { private async pushTransientGoalSnapshot( workspaceId: string, snapshot: GoalSnapshot - ): Promise { - const activity = await this.extensionMetadata.getSnapshot(workspaceId); + ): Promise<"delivered" | "no_baseline" | "unavailable"> { + let activity: WorkspaceActivitySnapshot | null; + try { + activity = await this.extensionMetadata.getSnapshot(workspaceId, { throwOnError: true }); + } catch (error) { + // A suspect baseline (failed sidecar reconcile / unreadable main) + // must not feed an emitted overlay: partial-main fields would clear + // status in the renderer. "unavailable" is deliberately DISTINCT from + // the authoritative "no_baseline": the durable pushSnapshot fallback + // writes through the lenient load (accepting the same suspect partial + // main this strict read refused) and emits the result — converting + // this failure into that fallback would clear exactly the renderer + // state the strict read preserves. + log.debug("Skipping transient goal emit after failed snapshot read", { + workspaceId, + error, + }); + return "unavailable"; + } if (!activity) { // No baseline activity snapshot to overlay the transient goal on // (extensionMetadata has no entry for this workspace yet). Callers // that must guarantee delivery — e.g. live cost previews — should - // observe this `false` return and fall back to `pushSnapshot`, which + // observe "no_baseline" and fall back to `pushSnapshot`, which // creates the entry and emits via the durable path. Pending-goal // publication does not retry here because it only fires after a // `setGoal` that already created the entry. - return false; + return "no_baseline"; } this.onActivityChange?.(workspaceId, { ...activity, goal: snapshot, transientGoalOnly: true, }); - return true; + return "delivered"; } private async pushLiveGoalPreviewOverlay( @@ -4114,13 +4131,17 @@ export class WorkspaceGoalService { }); const snapshot = toGoalSnapshot(preview); this.liveGoalPreviewSnapshots.set(input.workspaceId, snapshot); - const didEmitTransient = await this.pushTransientGoalSnapshot(input.workspaceId, snapshot); - if (!didEmitTransient) { + const transientResult = await this.pushTransientGoalSnapshot(input.workspaceId, snapshot); + if (transientResult === "no_baseline") { // If the baseline activity snapshot does not exist yet (for // example, extensionMetadata was reset or stream-start's // fire-and-forget metadata write has not finished), fall back to // the durable path so this preview is still delivered to Goals UI - // subscribers instead of being dropped. + // subscribers instead of being dropped. "unavailable" must NOT take + // this path: the durable write's lenient load would accept the + // suspect partial main the strict read refused and emit it, + // clearing renderer goal/status state — return the computed preview + // without delivery instead (renderer keeps last-known state). return this.pushSnapshot(input.workspaceId, preview); } return snapshot; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 77081b1d03..885d25ad4a 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -141,6 +141,10 @@ const mockInitStateManager: Partial = { clearInMemoryState: mock(() => undefined), }; const mockExtensionMetadataService: Partial = { + isWorkspaceDeleted: mock(() => false), + clearTombstonesForRegisteredIds: mock(() => undefined), + getTombstonedIds: mock((): ReadonlyMap => new Map()), + setTombstoneClearedListener: mock(() => undefined), setStreaming: mock(() => Promise.resolve({ recency: Date.now(), @@ -3417,6 +3421,16 @@ describe("WorkspaceService bash monitor wakes", () => { const { config, cleanup } = await createTestHistoryService(); try { const workspaceId = "bash-monitor-tombstone-failed-clear"; + // getActivityList only emits entries for config-known workspaces; the + // tombstone contract below is scoped to known ids. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); let activeMonitorCount = 1; const backgroundProcessManager = Object.assign(new EventEmitter(), { cleanup: mock(() => Promise.resolve()), @@ -3470,6 +3484,16 @@ describe("WorkspaceService bash monitor wakes", () => { const { config, cleanup } = await createTestHistoryService(); try { const workspaceId = "bash-monitor-tombstone"; + // getActivityList only emits entries for config-known workspaces; the + // tombstone contract below is scoped to known ids. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); let activeMonitorCount = 1; const backgroundProcessManager = Object.assign(new EventEmitter(), { cleanup: mock(() => Promise.resolve()), @@ -5114,6 +5138,15 @@ describe("WorkspaceService workflow activity", () => { try { const workspaceId = "workflow-activity-race"; + // getActivityList only emits entries for config-known workspaces. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); const workspaceService = createWorkspaceServiceForTest({ config, historyService, @@ -5165,6 +5198,17 @@ describe("WorkspaceService workflow activity", () => { try { const workspaceId = "workflow-activity-overlap"; + // getActivityList only emits entries for config-known workspaces; keep + // this workspace known so the zero-count assertion below exercises the + // tombstone path rather than trivially missing the entry. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); const workspaceService = createWorkspaceServiceForTest({ config, historyService, @@ -5224,6 +5268,2785 @@ describe("WorkspaceService workflow activity", () => { }); }); +describe("WorkspaceService activity list scoping", () => { + test("drops stale extension metadata entries and lazily prunes them once", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "activity-scoping-known"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + // Simulates the leaked entry of a removed workspace/sub-agent. + await extensionMetadata.updateRecency("removed-workspace", 200); + const pruneSpy = spyOn(extensionMetadata, "pruneMissingWorkspaces"); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.recency).toBe(100); + expect(activityList?.["removed-workspace"]).toBeUndefined(); + + // The one-time lazy cleanup dropped the stale entry from disk while + // keeping the still-existing workspace's entry. + const snapshots = await extensionMetadata.getAllSnapshots(); + expect(snapshots.has("removed-workspace")).toBe(false); + expect(snapshots.get(workspaceId)?.recency).toBe(100); + + // One-time: a second bootstrap must not re-run the cleanup scan. + await workspaceService.getActivityList(); + expect(pruneSpy).toHaveBeenCalledTimes(1); + } finally { + await cleanup(); + } + }); + + test("repeat lists keep omitting idle workspaces after the first list installs caches", async () => { + // The first list's workflow probe installs an empty run cache for every + // scoped id. Cache initialization must not read as activity: treating it + // as the zero-count tombstone signal would emit a fabricated recency:0 + // entry for every idle config-known workspace from the second list on, + // re-bloating exactly the payload this scoping trims. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "activity-scoping-idle"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const firstList = await workspaceService.getActivityList(); + expect(firstList).not.toBeNull(); + expect(firstList?.[workspaceId]).toBeUndefined(); + const secondList = await workspaceService.getActivityList(); + expect(secondList).not.toBeNull(); + expect(secondList?.[workspaceId]).toBeUndefined(); + } finally { + await cleanup(); + } + }); + + test("a run-status event racing cache eviction does not strand the seen marker", async () => { + // An eviction (removal, or a tombstone lifted for revival) can land in + // the microtask gap after the run cache resolves. The status event must + // retry against the freshly installed cache instead of mutating the + // detached set and marking the seen set — a stale marker would fabricate + // zero-count entries for the idle revived workspace on every later list. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "evict-race"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const internals = workspaceService as unknown as { + getActiveWorkflowRunIds(workspaceId: string): Promise>; + evictWorkspaceActivityCaches(workspaceId: string): void; + }; + const realGetActiveWorkflowRunIds = internals.getActiveWorkflowRunIds.bind(workspaceService); + let evicted = false; + internals.getActiveWorkflowRunIds = async (targetWorkspaceId: string) => { + const result = await realGetActiveWorkflowRunIds(targetWorkspaceId); + if (!evicted && targetWorkspaceId === workspaceId) { + evicted = true; + // Lands after the cache read resolved, before the caller's + // continuation — the exact revival-eviction window. + internals.evictWorkspaceActivityCaches(targetWorkspaceId); + } + return result; + }; + + await workspaceService.emitWorkflowRunActivity({ + workspaceId, + runId: "wfr_race", + status: "running", + }); + const activity = (await workspaceService.getActivityList())?.[workspaceId]; + // The retried update must land the run in the INSTALLED cache (not a + // detached pre-eviction set that leaves only the stale seen marker). + expect(activity?.activeWorkflowRunCount).toBe(1); + } finally { + await cleanup(); + } + }); + + test("getActivityList re-establishes the config baseline after a transient initial read failure", async () => { + // The pre-await baseline read can fail transiently while the strict + // scoping enumeration succeeds. Without a replacement baseline both + // cross-process removal guards stay disabled on an authoritative + // response: a workspace another backend deregisters during the workflow + // probes (its metadata entry still present in the normal cleanup gap) + // would ride back into the renderer with no event to correct it. + const { config, historyService, cleanup } = await createTestHistoryService(); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const workspaceId = "baseline-retry"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 321); + const realSuperset = config.readPersistedWorkspaceIdSuperset.bind(config); + let failedOnce = false; + const supersetSpy = spyOn(config, "readPersistedWorkspaceIdSuperset").mockImplementation( + () => { + if (!failedOnce) { + failedOnce = true; + throw new Error("transient config read failure"); + } + return realSuperset(); + } + ); + let removedFromConfig = false; + listStatusSnapshotsSpy.mockImplementation(async () => { + if (!removedFromConfig) { + removedFromConfig = true; + // Another backend deregisters the workspace while the per-id + // probe awaits; its metadata entry intentionally stays behind. + const configPath = path.join(config.rootDir, "config.json"); + const parsed = JSON.parse(await fsPromises.readFile(configPath, "utf-8")) as { + projects?: Array<[string, { workspaces?: Array<{ id?: string }> }]>; + }; + for (const [, projectConfig] of parsed.projects ?? []) { + projectConfig.workspaces = (projectConfig.workspaces ?? []).filter( + (workspace) => workspace.id !== workspaceId + ); + } + await fsPromises.writeFile(configPath, JSON.stringify(parsed)); + } + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + supersetSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + + test("prune spares both legacy identities when compatibility files disagree", async () => { + // An id-less legacy entry can have BOTH supported session layouts with + // different stable ids (stale basename-side file + live generated-legacy + // metadata). findWorkspace resolves either id, so the one-time prune must + // spare extension-metadata entries under both — classifying the second + // identity as stale would delete activity findWorkspace still vouches + // for. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "old-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]] }) + ); + const basenameSessionDir = config.getSessionDir("old-ws"); + await fsPromises.mkdir(basenameSessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(basenameSessionDir, "metadata.json"), + JSON.stringify({ id: "basename-stable-id", name: "old-ws" }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: "generated-live-id", name: "old-ws" }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("basename-stable-id", 100); + await extensionMetadata.updateRecency("generated-live-id", 200); + await extensionMetadata.updateRecency("truly-stale-id", 300); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + + const snapshots = await extensionMetadata.getAllSnapshots(); + expect(snapshots.get("basename-stable-id")?.recency).toBe(100); + expect(snapshots.get("generated-live-id")?.recency).toBe(200); + expect(snapshots.has("truly-stale-id")).toBe(false); + } finally { + await cleanup(); + } + }); + + test("getActivityList fails closed when no raw baseline can be established", async () => { + // If every raw baseline read fails transiently while the strict scoping + // enumeration succeeds, both cross-process removal guards would stay + // disabled on a response the renderer applies as authoritative — a + // workspace another backend deregisters during the probes would ride + // back with no event to correct it. The list must fail (null → renderer + // keeps last-known state and retries) instead of serving guardless + // authoritative data; only the fail-open scope (config unreadable) may + // do that, and there the enumeration fails too. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "baseline-unavailable"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 111); + const supersetSpy = spyOn(config, "readPersistedWorkspaceIdSuperset").mockImplementation( + () => { + throw new Error("persistent raw read failure"); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + expect(await workspaceService.getActivityList()).toBeNull(); + } finally { + supersetSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("first prune also removes stale entries stranded in a sidecar", async () => { + // Crash strands the full snapshot in .corrupt while a valid partial main + // was recreated. The one-time prune must reconcile FIRST: sidecar-only + // stale entries would otherwise dodge the deletion set and merge back on + // the very next read — with the prune latched, they would keep inflating + // every read and rewrite until restart. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "sidecar-live"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + await fsPromises.writeFile( + metadataPath, + JSON.stringify({ + version: 1, + workspaces: { [workspaceId]: { recency: 100, streaming: false } }, + }) + ); + await fsPromises.writeFile( + `${metadataPath}.corrupt`, + JSON.stringify({ + version: 1, + workspaces: { + [workspaceId]: { recency: 90, streaming: false }, + "sidecar-stale": { recency: 80, streaming: false }, + }, + }) + ); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.["sidecar-stale"]).toBeUndefined(); + + const snapshots = await extensionMetadata.getAllSnapshots({ throwOnError: true }); + expect(snapshots.get(workspaceId)?.recency).toBe(100); + expect(snapshots.has("sidecar-stale")).toBe(false); + } finally { + await cleanup(); + } + }); + + test("late-admitted raw-registered ids keep their initial snapshot when re-reads fail", async () => { + // A raw-registered entry outside the normalized scope (invalid project + // path) is admitted through the raw config view. When both mid-list + // snapshot re-reads fail transiently, the already-loaded initial + // snapshot must still supply its recency/goal/status — an authoritative + // response omitting the entry would clear that renderer state with no + // repair event. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "raw-only-live"; + const projectPath = path.join(config.rootDir, "project"); + // Migration flags pre-seeded: without them the first load schedules an + // async settings-migration persist that rewrites config.json through + // the parsed view mid-test whenever it happens to land before the + // second list's raw reads (observed flake). + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [ + [ + projectPath, + { workspaces: [{ id: workspaceId, path: path.join(projectPath, "ws") }] }, + ], + ], + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }) + ); + // Raw-visible but enumeration-invisible: the strict normalized + // enumeration resolves no ids while the raw persisted view carries the + // inline id, keeping it out of the per-id scope so it takes the + // late-candidate path. (Strict loads now reject the previously used + // malformed-project-key vehicle, so divergence is modeled directly.) + const enumerateSpy = spyOn(config, "getAllWorkspaceMetadata").mockResolvedValue([]); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 42); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + // List #1 (prune latch): initial + fresh reads stay real. List #2: + // the initial read (call 3) succeeds; the fresh and final re-reads + // fail transiently. + if (snapshotCalls > 3) { + throw new Error("transient snapshot re-read failure"); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const firstList = await workspaceService.getActivityList(); + expect(firstList?.[workspaceId]?.recency).toBe(42); + + const secondList = await workspaceService.getActivityList(); + expect(secondList).not.toBeNull(); + expect(secondList?.[workspaceId]?.recency).toBe(42); + } finally { + snapshotsSpy.mockRestore(); + enumerateSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("mid-list enumeration proves removal when the raw refresh fails", async () => { + // An inline-id workspace is removed by another backend while the + // mid-list authoritative enumeration awaits, and the post-enumeration + // raw refresh fails transiently. The raw comparison is disabled (fresh + // view null) and the id sits in the initial baseline, so without the + // enumeration fallback every removal guard passes and the stale entry + // rides the authoritative response with no event to repair the + // renderer. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "inline-removed-mid-enum"; + const projectPath = path.join(config.rootDir, "project"); + const configPath = path.join(config.rootDir, "config.json"); + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [ + [ + projectPath, + { + workspaces: [ + { id: workspaceId, path: path.join(projectPath, "ws") }, + // Id-less legacy entry whose stable id lives in session + // metadata.json: raw-INVISIBLE at the initial baseline, so + // its retained entry forces the mid-list authoritative + // enumeration this test exercises (the read-time migration + // may persist the id later, but the baseline predates it). + { path: path.join(projectPath, "legacy-ws") }, + ], + }, + ], + ], + // Migration flags pre-seeded: without them the first load schedules + // an async settings-migration persist that rewrites config.json + // through the parsed view — attaching the resolved legacy id inline + // — which would make this entry raw-VISIBLE mid-test and skip the + // mid-list enumeration whenever the persist lands first. + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }) + ); + const legacyStableId = "legacy-stable-mid-enum"; + const legacySessionDir = config.getSessionDir("legacy-ws"); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: legacyStableId, name: "legacy-ws" }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 77); + await extensionMetadata.updateRecency(legacyStableId, 55); + const realEnumerate = config.getAllWorkspaceMetadata.bind(config); + let enumerationCalls = 0; + let failEvidenceReads = false; + const enumerationSpy = spyOn(config, "getAllWorkspaceMetadata").mockImplementation( + async (options?: Parameters[0]) => { + enumerationCalls += 1; + if (enumerationCalls === 2) { + // Mid-list enumeration: another backend deregisters the inline-id + // workspace just before the config read, and every later raw + // view read fails transiently. + const parsed = JSON.parse(await fsPromises.readFile(configPath, "utf-8")) as { + projects: Array<[string, { workspaces: Array<{ id?: string }> }]>; + }; + for (const [, projectConfig] of parsed.projects) { + projectConfig.workspaces = projectConfig.workspaces.filter( + (workspace) => workspace.id !== workspaceId + ); + } + await fsPromises.writeFile(configPath, JSON.stringify(parsed)); + const result = await realEnumerate(options); + failEvidenceReads = true; + return result; + } + return realEnumerate(options); + } + ); + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + if (failEvidenceReads) { + throw new Error("transient raw config read failure"); + } + return realEvidence(); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + // The still-registered raw-invisible entry survives the fallback... + expect(activityList?.[legacyStableId]?.recency).toBe(55); + // ...while the enumeration-proven removal is dropped. + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + enumerationSpy.mockRestore(); + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("final enumeration proves removal when the post-probe raw read fails", async () => { + // An inline-id workspace is deregistered by another backend while the + // late-candidate workflow probes await, and the post-probe raw reads + // fail transiently. Without the enumeration fallback every raw + // deregistration guard is disabled (finalConfigIds null) and the stale + // retained entry rides the authoritative response with no cross-process + // event to repair it. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const removedId = "inline-removed-final"; + const survivorId = "inline-survivor-final"; + const lateId = "late-registered-final"; + const projectPath = path.join(config.rootDir, "project"); + const configPath = path.join(config.rootDir, "config.json"); + const configFor = (ids: string[]): string => + JSON.stringify({ + projects: [ + [ + projectPath, + { workspaces: ids.map((id) => ({ id, path: path.join(projectPath, id) })) }, + ], + ], + // Migration flags pre-seeded so the first load never schedules the + // async settings-migration persist mid-test. + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }); + await fsPromises.writeFile(configPath, configFor([removedId, survivorId])); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(removedId, 77); + await extensionMetadata.updateRecency(survivorId, 55); + // Fresh snapshot re-read (call 2) doubles as the moment "another + // backend" registers a new workspace: its id enters the fresh raw + // view outside the initial scope, forcing the late-candidate probes + // and with them the final post-probe views this test exercises. + // The final-phase snapshot re-read (call 3) marks the start of the + // post-probe views: the concurrent deregistration lands there and + // every later raw evidence read fails transiently, so only the + // fallback enumeration can prove the removal. + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + let failRawEvidenceReads = false; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile(configPath, configFor([removedId, survivorId, lateId])); + } + if (snapshotCalls === 3) { + await fsPromises.writeFile(configPath, configFor([survivorId, lateId])); + failRawEvidenceReads = true; + } + return realGetAllSnapshots(options); + } + ); + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + if (failRawEvidenceReads) { + throw new Error("transient raw config read failure"); + } + return realEvidence(); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[survivorId]?.recency).toBe(55); + // The enumeration-proven removal is dropped despite the raw view + // being unreadable. + expect(activityList?.[removedId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("foreign removals observed mid-list evict process-local activity caches", async () => { + // A cross-process removal publishes no local tombstone, so the + // tombstone-cleared eviction listener never fires. Without eviction at + // the removal guards, the removed incarnation's workflow caches survive + // — and a downgraded backend re-registering the same deterministic + // legacy id would then be served ghost runs instead of a fresh probe. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "foreign-removed-evict"; + const projectPath = path.join(config.rootDir, "project"); + const configPath = path.join(config.rootDir, "config.json"); + const configFor = (ids: string[]): string => + JSON.stringify({ + projects: [ + [ + projectPath, + { workspaces: ids.map((id) => ({ id, path: path.join(projectPath, id) })) }, + ], + ], + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }); + await fsPromises.writeFile(configPath, configFor([workspaceId])); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 42); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const first = await workspaceService.getActivityList(); + expect(first?.[workspaceId]?.recency).toBe(42); + const internals = workspaceService as unknown as { + activeWorkflowRunIdsByWorkspace: Map>; + }; + expect(internals.activeWorkflowRunIdsByWorkspace.has(workspaceId)).toBe(true); + // Another backend removes the workspace between the second list's + // initial and fresh raw reads (its metadata cleanup may lag). + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile(configPath, configFor([])); + } + return realGetAllSnapshots(options); + } + ); + try { + const second = await workspaceService.getActivityList(); + expect(second?.[workspaceId]).toBeUndefined(); + expect(internals.activeWorkflowRunIdsByWorkspace.has(workspaceId)).toBe(false); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("first bootstrap reuses the prune's config enumeration for scoping", async () => { + // getAllWorkspaceMetadata walks every workspace with per-workspace disk + // probes; the latency-sensitive first bootstrap must not pay it twice. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "activity-scoping-reuse"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + await extensionMetadata.updateRecency("removed-workspace", 200); + const metadataSpy = spyOn(config, "getAllWorkspaceMetadata"); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.recency).toBe(100); + expect(activityList?.["removed-workspace"]).toBeUndefined(); + // The single walk belongs to the prune's initial enumeration: the + // list's SCOPING reuses the prune's ids, and the prune's mid-pass + // re-registration recheck uses the raw config view (complete evidence + // here — every persisted workspace id is inline) instead of repeating + // the per-workspace walk while the metadata queue blocks live writes. + expect(metadataSpy).toHaveBeenCalledTimes(1); + // The stale entry really was reclaimed on disk, not merely filtered. + expect((await extensionMetadata.getAllSnapshots()).has("removed-workspace")).toBe(false); + } finally { + await cleanup(); + } + }); + + test("first bootstrap scope keeps raw-registered ids the normalized view cannot see", async () => { + // A duplicate project path key (e.g. a trailing-slash variant) shadows + // the earlier pair in the normalized view — its workspace is registered + // and raw-visible (spared by the prune) yet absent from every strict + // enumeration. The first-bootstrap scope must come from the prune's + // FULL raw-plus-normalized union, not the enumeration alone: when the + // later raw refreshes fail transiently, an enumeration-only scope would + // serve an authoritative response omitting the live workspace, clearing + // its renderer activity state with no event to correct it. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const shadowedId = "raw-only-shadowed-ws"; + const winnerId = "normalized-winner-ws"; + const projectPath = path.join(config.rootDir, "project"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [ + // Map construction keeps the LAST duplicate key: the first pair + // (trailing-slash variant of the same path) is dropped from the + // normalized view with its workspace, while the raw id scan + // still collects it. + [ + `${projectPath}/`, + { workspaces: [{ id: shadowedId, path: path.join(projectPath, "shadowed") }] }, + ], + [ + projectPath, + { workspaces: [{ id: winnerId, path: path.join(projectPath, "winner") }] }, + ], + ], + taskSettings: { preserveSubagentsUntilArchive: true }, + migrations: { persistentSubagentsDefaulted: true, defaultModelFallbacksSeeded: true }, + }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(shadowedId, 42); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Every raw config read AFTER the prune's successful one fails + // transiently (the finding's window: the id was already loaded, and + // only the discarded return kept it out of scope). + const internals = workspaceService as unknown as { + pruneStaleExtensionMetadataOnce(): Promise; + }; + const realPrune = internals.pruneStaleExtensionMetadataOnce.bind(workspaceService); + let failRawReads = false; + internals.pruneStaleExtensionMetadataOnce = async () => { + const prefetched = await realPrune(); + failRawReads = true; + return prefetched; + }; + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + if (failRawReads) { + throw new Error("transient config read failure"); + } + return realEvidence(); + } + ); + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[shadowedId]?.recency).toBe(42); + // Its snapshot was spared by the prune too, not merely re-admitted. + expect((await extensionMetadata.getAllSnapshots()).has(shadowedId)).toBe(true); + } finally { + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("a mid-list corruption reset does not read as workspace removal", async () => { + // getAllSnapshots self-heals a deterministically corrupt metadata file + // into a valid (possibly EMPTY) one, so a quarantine landing between the + // initial and fresh reads makes every earlier snapshot key vanish from a + // SUCCESSFUL re-read while the config still registers the workspaces. + // Treating that disappearance as foreign-removal evidence would evict + // the workflow caches and omit live workspaces from an authoritative + // response — with no cross-process event to repair the renderer. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "corruption-reset-survivor"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 123); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const realGetAll = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotReads = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + (options) => { + snapshotReads += 1; + if (snapshotReads === 1) { + return realGetAll(options); + } + // Every re-read after the initial one models the post-quarantine + // self-healed EMPTY file: a successful, authoritative-looking + // read with every previous key gone. + return Promise.resolve(new Map()); + } + ); + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.recency).toBe(123); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("a raw-removed id affirmed by the fresh enumeration is retained, not tombstoned", async () => { + // A downgraded backend can remove an inline-id workspace entry and + // re-register the SAME deterministic id as an id-less legacy entry + // while this list awaits. The id then vanishes from every fresh raw + // view (its identity lives in session metadata.json) while the fresh + // authoritative enumeration — the very evidence that clears the id's + // tombstone — still resolves it. Treating the raw disappearance alone + // as removal would drop the revived workspace's activity and republish + // the tombstone that evidence just cleared, suppressing it again. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "raw-invisible-revival"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 42); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const internals = workspaceService as unknown as { + pruneStaleExtensionMetadataOnce(): Promise; + enumerateAuthoritativeWorkspaceIds(): Promise>; + }; + const realPrune = internals.pruneStaleExtensionMetadataOnce.bind(workspaceService); + let revivedIdless = false; + internals.pruneStaleExtensionMetadataOnce = async () => { + const prefetched = await realPrune(); + // The removal + id-less re-registration lands after the initial + // baseline and the prune, before the fresh evidence read. + revivedIdless = true; + return prefetched; + }; + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + const evidence = realEvidence(); + if (!revivedIdless) { + return evidence; + } + // The downgraded backend rewrote the entry without an inline id: + // the id disappears from the raw view, and the id-less entry + // marks that view incomplete. + const ids = new Set(evidence.ids); + ids.delete(workspaceId); + return { ids, hasWorkspaceEntriesWithoutIds: true }; + } + ); + const realEnumerate = internals.enumerateAuthoritativeWorkspaceIds.bind(workspaceService); + internals.enumerateAuthoritativeWorkspaceIds = async () => { + // The enumeration resolves the id-less entry's stable identity. + const ids = await realEnumerate(); + ids.add(workspaceId); + return ids; + }; + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.recency).toBe(42); + // No republished tombstone suppressing the revived workspace. + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + } finally { + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("a revival landing between the enumeration and the raw refresh is re-checked, not dropped", async () => { + // The staleness window the post-refresh re-enumeration closes: the id + // is removed BEFORE the mid-list enumeration runs (so that enumeration + // denies it) and re-registered id-less right after it. The raw refresh + // then reports id-less entries — proof the earlier denial may be + // stale — so the removal arms must consult a fresh enumeration (which + // resolves the revived identity) instead of dropping the workspace on + // the stale denial and tombstoning it. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "revived-between-reads"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 42); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const internals = workspaceService as unknown as { + pruneStaleExtensionMetadataOnce(): Promise; + enumerateAuthoritativeWorkspaceIds(): Promise>; + }; + const realPrune = internals.pruneStaleExtensionMetadataOnce.bind(workspaceService); + let removed = false; + internals.pruneStaleExtensionMetadataOnce = async () => { + const prefetched = await realPrune(); + // The cross-process removal lands after the initial baseline and + // the prune. + removed = true; + return prefetched; + }; + const realEvidence = config.readPersistedWorkspaceIdEvidence.bind(config); + const evidenceSpy = spyOn(config, "readPersistedWorkspaceIdEvidence").mockImplementation( + () => { + const evidence = realEvidence(); + if (!removed) { + return evidence; + } + // Post-removal raw views: the id is gone, and an unrelated + // id-less legacy entry keeps the view incomplete throughout. + const ids = new Set(evidence.ids); + ids.delete(workspaceId); + return { ids, hasWorkspaceEntriesWithoutIds: true }; + } + ); + const realEnumerate = internals.enumerateAuthoritativeWorkspaceIds.bind(workspaceService); + let postRemovalEnumerations = 0; + internals.enumerateAuthoritativeWorkspaceIds = async () => { + const ids = await realEnumerate(); + if (!removed) { + return ids; + } + postRemovalEnumerations += 1; + if (postRemovalEnumerations === 1) { + // First post-removal enumeration: the removal is visible, the + // id-less re-registration has not landed yet — a stale denial. + ids.delete(workspaceId); + } + // Later enumerations resolve the revived id-less identity. + return ids; + }; + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.recency).toBe(42); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + } finally { + evidenceSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("first bootstrap admits snapshotless ids registered after the prune enumerated config", async () => { + // A workspace another backend registers after the prune captured its id + // set may have workflow- or bash-monitor-only activity and therefore no + // extensionMetadata snapshot. Admission must come from the refreshed raw + // config view — filtering through snapshot keys would skip the per-id + // workflow probe entirely and return an authoritative list without it. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const lateWorkspaceId = "late-registered-workspace"; + // Registered for real (the mid-list registration lands in config.json + // in the modeled race); the spies below hide it from the baseline and + // prune reads so only the refresh discovers it — the authoritative + // removal recheck must then still find it registered. + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: lateWorkspaceId, + name: lateWorkspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realSuperset = config.readPersistedWorkspaceIdSuperset.bind(config); + let supersetCalls = 0; + const supersetSpy = spyOn(config, "readPersistedWorkspaceIdSuperset").mockImplementation( + () => { + supersetCalls += 1; + const ids = realSuperset(); + // Calls 1 (pre-await baseline) and 2 (prune enumeration) see the + // pre-registration config; the refresh and the post-await + // revalidation see the concurrently registered workspace. + if (supersetCalls <= 2) { + ids.delete(lateWorkspaceId); + } + return ids; + } + ); + const realMetadata = config.getAllWorkspaceMetadata.bind(config); + let metadataCalls = 0; + const metadataSpy = spyOn(config, "getAllWorkspaceMetadata").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + // Only the prune's enumeration (first call) predates the + // registration in the modeled race; the revalidation's fresh + // authoritative enumeration sees the registered workspace. + metadataCalls += 1; + const all = await realMetadata(options); + if (metadataCalls === 1) { + return all.filter((metadata) => metadata.id !== lateWorkspaceId); + } + return all; + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Workflow-only activity: discoverable by the per-id probe, never a + // persisted snapshot. + await workspaceService.emitWorkflowRunActivity({ + workspaceId: lateWorkspaceId, + runId: "late-run", + status: "running", + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[lateWorkspaceId]?.activeWorkflowRunCount).toBe(1); + } finally { + supersetSpy.mockRestore(); + metadataSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("suppresses activity emissions for removed workspaces", async () => { + // A late in-flight producer completing after removal must not broadcast: + // the renderer would re-insert the removed id into its activity map after + // already processing the metadata-removal event. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "activity-emit-after-removal"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const events: Array<{ workspaceId: string }> = []; + workspaceService.on("activity", (event) => events.push(event)); + + await workspaceService.updateAgentStatus(workspaceId, { emoji: "🛠️", message: "Working" }); + expect(events.length).toBe(1); + + // Discard verifies deregistration against persisted config, so remove + // the workspace first (mirroring the real removal flow). + await config.removeWorkspace(workspaceId); + await workspaceService.discardExtensionMetadataEntry(workspaceId); + // Simulates the producer that was already in flight when removal ran. + await workspaceService.updateAgentStatus(workspaceId, { emoji: "🛠️", message: "Late" }); + expect(events.length).toBe(1); + // Clearing (null) emissions stay allowed for removed workspaces. + workspaceService.emitWorkspaceActivity(workspaceId, null); + expect(events.length).toBe(2); + // A late workflow-run producer can also fire after removal: its cache + // entry turns a null snapshot into a non-null merged payload, which + // must be suppressed exactly like a non-null snapshot emission — the + // tombstone check runs on the merged payload, not the raw snapshot. + await workspaceService.emitWorkflowRunActivity({ + workspaceId, + runId: "late-run", + status: "running", + }); + expect(events.length).toBe(2); + workspaceService.emitWorkspaceActivity(workspaceId, null); + expect(events.length).toBe(2); + } finally { + await cleanup(); + } + }); + + test("a re-registered id sheds its tombstone on the next activity list", async () => { + // Tombstones are process-local removal knowledge; the shared config is + // the authority. A downgraded concurrent backend can legitimately + // re-register a deterministic legacy id this process pruned — the next + // activity list observes the id in fresh config evidence and must lift + // the write suppression instead of muting the revived workspace until + // restart. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "revived-legacy-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const workspaceEntry = { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" as const }, + }; + await config.addWorkspace(projectPath, workspaceEntry); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Removal flow: deregister, then discard (delete + tombstone). + await config.removeWorkspace(workspaceId); + await workspaceService.discardExtensionMetadataEntry(workspaceId); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(true); + // Writes are suppressed while tombstoned. + await extensionMetadata.updateRecency(workspaceId, 200); + expect((await extensionMetadata.getAllSnapshots()).has(workspaceId)).toBe(false); + + // The "other backend" re-registers the same id in the shared config. + await config.addWorkspace(projectPath, workspaceEntry); + + await workspaceService.getActivityList(); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + // Writes persist again after the revival. + await extensionMetadata.updateRecency(workspaceId, 300); + expect((await extensionMetadata.getAllSnapshots()).get(workspaceId)?.recency).toBe(300); + } finally { + await cleanup(); + } + }); + + test("discardExtensionMetadataEntry keeps the entry when the workspace is still persisted", async () => { + // saveConfig swallows write failures, so config.removeWorkspace can + // resolve while the workspace is still persisted in config.json. + // Discarding then would write-tombstone a live id and suppress all of + // its future activity writes for the rest of the process. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "discard-still-persisted"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + await workspaceService.discardExtensionMetadataEntry(workspaceId); + + expect((await extensionMetadata.getAllSnapshots()).has(workspaceId)).toBe(true); + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(false); + } finally { + await cleanup(); + } + }); + + test("discardExtensionMetadataEntry keeps entries of id-less legacy workspaces", async () => { + // An id-less legacy config entry resolves its stable id from + // sessions//metadata.json. The raw config scan + // cannot see that id, so the discard's registration check must resolve + // it through the same authoritative path getAllWorkspaceMetadata uses; + // otherwise the still-registered workspace would be reported absent and + // its activity writes permanently tombstoned for this process. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "legacy-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(stableId, 100); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + await workspaceService.discardExtensionMetadataEntry(stableId); + + expect((await extensionMetadata.getAllSnapshots()).has(stableId)).toBe(true); + expect(extensionMetadata.isWorkspaceDeleted(stableId)).toBe(false); + } finally { + await cleanup(); + } + }); + + test("discardExtensionMetadataEntry keeps entries when legacy metadata parses without an id", async () => { + // Same identity-unknowable contract as the unparseable case: a legacy + // metadata.json that parses as `{}` carries no id, so the strict + // findWorkspace lookup must fail closed rather than fall through to + // "not registered" — the entry under the real (unknowable) stable id + // would otherwise be deleted and write-tombstoned while its workspace + // remains registered. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "legacy-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile(path.join(legacySessionDir, "metadata.json"), "{}"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(stableId, 100); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + await workspaceService.discardExtensionMetadataEntry(stableId); + + expect((await extensionMetadata.getAllSnapshots()).has(stableId)).toBe(true); + expect(extensionMetadata.isWorkspaceDeleted(stableId)).toBe(false); + } finally { + await cleanup(); + } + }); + + test("getActivityList merges snapshots another process persisted mid-list", async () => { + // Another backend can register a workspace and persist its first + // activity after this process's initial snapshot read. The refreshed + // config admits the id, but the per-id computation saw a null snapshot + // and no local caches, so the entry would be omitted — and the activity + // subscription is process-local, so no delta ever heals it. The fresh + // revalidation re-read must merge the addition. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "late-snapshot-workspace"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + // Simulate the cross-process write landing between the initial read + // (call 1) and the revalidation re-read (call 2). + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 1) { + return realGetAllSnapshots(options); + } + await extensionMetadata.updateRecency(workspaceId, 777); + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[workspaceId]?.recency).toBe(777); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList merges workspaces registered and written entirely mid-list", async () => { + // Harder variant of the mid-list merge: the workspace is registered AND + // written after every scope read (baseline, prune, refresh), so it is in + // neither the per-id scope nor the initial snapshots — only the fresh + // revalidation views (snapshot re-read + raw config re-read) know it. + // The merge must admit ids those fresh views agree on. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "brand-new-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + // The other backend registers the workspace and persists its + // first activity between the initial read and the revalidation + // re-read (before the fresh raw config re-read). + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(workspaceId, 888); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[workspaceId]?.recency).toBe(888); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList bootstraps workflow runs for workspaces merged mid-list", async () => { + // A workspace admitted only by the fresh revalidation re-reads never went + // through the per-id loop, so its on-disk active workflow runs are not in + // the process-local cache. The merge must probe disk for them — a + // cached-only merge would omit activeWorkflowRunCount for exactly the + // cross-process registrations it exists to bootstrap, and the + // process-local activity subscription can never deliver that delta. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "brand-new-workflow-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + // The other backend registers the workspace, persists its first + // activity, AND starts a workflow run before the revalidation + // re-read. + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(workspaceId, 888); + const runStore = new WorkflowRunStore({ + sessionDir: config.getSessionDir(workspaceId), + }); + await runStore.createRun({ + id: "wfr_midlist", + workspaceId, + workflow: { + name: "demo", + description: "Demo workflow", + scope: "global" as const, + executable: true, + }, + source: "export default function workflow() { return {}; }", + args: {}, + now: "2026-06-17T00:00:00.000Z", + }); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[workspaceId]?.recency).toBe(888); + expect(activityList?.[workspaceId]?.activeWorkflowRunCount).toBe(1); + expect(activityList?.[workspaceId]?.activeWorkflowRunIds).toEqual(["wfr_midlist"]); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList admits raw-invisible legacy workspaces registered mid-list", async () => { + // A downgraded backend can register a legacy (id-less config entry) + // workspace mid-list: its stable id lives only in session metadata.json, + // so the fresh raw config re-read can never vouch for it. The merge must + // resolve such fresh-snapshot ids through the authoritative identity + // path instead of excluding them until reconnect. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "late-legacy-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + await extensionMetadata.updateRecency(stableId, 777); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[stableId]?.recency).toBe(777); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList bootstraps workflow-only workspaces registered mid-list", async () => { + // A backend can register a workspace after the scope reads and start a + // workflow WITHOUT writing extension metadata: the fresh snapshot re-read + // never contains the id, so admission must come from the fresh raw + // config view alone — otherwise the workflow-only activity is missing + // from the authoritative response until reconnect. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "workflow-only-late-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + // Registered + workflow started, but NO metadata write. + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const runStore = new WorkflowRunStore({ + sessionDir: config.getSessionDir(workspaceId), + }); + await runStore.createRun({ + id: "wfr_workflow_only", + workspaceId, + workflow: { + name: "demo", + description: "Demo workflow", + scope: "global" as const, + executable: true, + }, + source: "export default function workflow() { return {}; }", + args: {}, + now: "2026-06-17T00:00:00.000Z", + }); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[workspaceId]?.activeWorkflowRunCount).toBe(1); + expect(activityList?.[workspaceId]?.activeWorkflowRunIds).toEqual(["wfr_workflow_only"]); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("a workflow bootstrap evicted mid-flight is retried instead of served detached", async () => { + // Cache eviction (removal / tombstone-lift revival) can race an + // in-flight bootstrap: waiters that captured the pre-eviction Set would + // return the removed incarnation's runs — ghost counts with no terminal + // event to clear them. The read must detect the eviction and re-probe. + const { config, historyService, cleanup } = await createTestHistoryService(); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const workspaceId = "evicted-mid-bootstrap"; + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + }); + const internals = workspaceService as unknown as { + getActiveWorkflowRunIds(id: string): Promise>; + evictWorkspaceActivityCaches(id: string): void; + }; + const releaseFirstScan = createDeferred(); + let scanCalls = 0; + listStatusSnapshotsSpy.mockImplementation(async () => { + scanCalls += 1; + if (scanCalls === 1) { + // Old-incarnation bootstrap: parked, then reports a ghost run. + await releaseFirstScan.promise; + return [ + { + id: "wfr_ghost", + workspaceId, + status: "running" as const, + createdAt: "2026-06-17T00:00:00.000Z", + updatedAt: "2026-06-17T00:00:00.000Z", + }, + ]; + } + // Post-revival probe: the new incarnation has no runs. + return []; + }); + + const read = internals.getActiveWorkflowRunIds(workspaceId); + // Removal + re-registration land while the bootstrap is parked. + internals.evictWorkspaceActivityCaches(workspaceId); + releaseFirstScan.resolve(); + const runIds = await read; + expect(runIds.size).toBe(0); + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + + test("a re-registered id does not inherit workflow caches from its removed incarnation", async () => { + // Workspace removal deletes session state without producing terminal + // workflow events, and the process-local run cache was never evicted: + // a deterministic legacy id re-registered by a downgraded backend would + // show the removed incarnation's ghost activeWorkflowRunCount forever + // (the per-id bootstrap returns the cached set without re-probing disk). + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "revived-workspace"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Cache a live workflow run for the (unregistered) old incarnation. + await workspaceService.emitWorkflowRunActivity({ + workspaceId, + runId: "wfr_ghost", + status: "running", + }); + // Removal cleanup: deregistered (never in config here), so the entry + // is tombstoned and the process-local caches must be evicted. + await workspaceService.discardExtensionMetadataEntry(workspaceId); + // The downgraded backend re-registers the same id; its session dir has + // no workflow runs. + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + // No ghost count from the removed incarnation's cache: the revived id + // re-probes disk (empty) and stays absent from the list. + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + await cleanup(); + } + }); + + test("getActivityList bootstraps workflow-only late ids even when the metadata reread fails", async () => { + // The fresh raw config re-read can discover a workflow-only late + // registration while the metadata re-read transiently fails. The list + // still returns an authoritative (non-null) response, and the + // process-local subscription cannot supply the foreign workflow event — + // so the config-proven id must be probed regardless of the failed + // snapshot view. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "workflow-only-late-reread-fails"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 1) { + return realGetAllSnapshots(options); + } + if (snapshotCalls === 2) { + // Registration + workflow start land before the (failing) + // revalidation re-read. + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const runStore = new WorkflowRunStore({ + sessionDir: config.getSessionDir(workspaceId), + }); + await runStore.createRun({ + id: "wfr_reread_fail", + workspaceId, + workflow: { + name: "demo", + description: "Demo workflow", + scope: "global" as const, + executable: true, + }, + source: "export default function workflow() { return {}; }", + args: {}, + now: "2026-06-17T00:00:00.000Z", + }); + } + // Every re-read after the initial one fails transiently. + throw new Error("transient metadata read failure"); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]?.activeWorkflowRunCount).toBe(1); + expect(activityList?.[workspaceId]?.activeWorkflowRunIds).toEqual(["wfr_reread_fail"]); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList bootstraps workflow-only legacy workspaces registered mid-list", async () => { + // Combined raw-invisible + snapshotless case: a downgraded backend + // registers an id-less legacy workspace mid-list and starts a workflow + // WITHOUT writing extension metadata. The stable id appears in neither + // the fresh raw view nor the fresh snapshots, so discovery must come + // from the authoritative enumeration triggered by the raw evidence's + // id-less-entry signal. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "late-legacy-workflow-only"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + const runStore = new WorkflowRunStore({ + sessionDir: config.getSessionDir(stableId), + }); + await runStore.createRun({ + id: "wfr_legacy_only", + workspaceId: stableId, + workflow: { + name: "demo", + description: "Demo workflow", + scope: "global" as const, + executable: true, + }, + source: "export default function workflow() { return {}; }", + args: {}, + now: "2026-06-17T00:00:00.000Z", + }); + } + return realGetAllSnapshots(options); + } + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.[stableId]?.activeWorkflowRunCount).toBe(1); + expect(activityList?.[stableId]?.activeWorkflowRunIds).toEqual(["wfr_legacy_only"]); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("getActivityList drops legacy additions deregistered during the workflow probe", async () => { + // A raw-invisible legacy workspace admitted mid-list and deregistered + // while the workflow probe awaits: every raw view is blind to it and its + // metadata snapshot survives the deregistration gap, so only the + // post-probe authoritative re-enumeration can prove the removal. + const { config, historyService, cleanup } = await createTestHistoryService(); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const stableId = "late-legacy-then-removed"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + await extensionMetadata.updateRecency(stableId, 999); + } + return realGetAllSnapshots(options); + } + ); + listStatusSnapshotsSpy.mockImplementation(async () => { + // Another backend deregisters the legacy workspace mid-probe; its + // metadata snapshot intentionally survives (cleanup gap). + await fsPromises.writeFile(configPath, JSON.stringify({ projects: [] })); + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[stableId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + + test("getActivityList drops late additions deregistered during the workflow probe", async () => { + // A workspace registered AFTER the initial raw baseline and removed + // while the workflow probe awaits sits in the normal gap between config + // deregistration and extension-metadata cleanup: its snapshot still + // exists, so only the post-probe raw config re-read (compared against + // the fresh view that admitted it — the initial baseline never saw it) + // can prove the removal. + const { config, historyService, cleanup } = await createTestHistoryService(); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const workspaceId = "late-then-deregistered"; + const projectPath = path.join(config.rootDir, "project"); + const configPath = path.join(config.rootDir, "config.json"); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(workspaceId, 888); + } + return realGetAllSnapshots(options); + } + ); + listStatusSnapshotsSpy.mockImplementation(async () => { + // Another backend deregisters the workspace mid-probe; its metadata + // entry intentionally survives (cleanup has not run yet). + await fsPromises.writeFile(configPath, JSON.stringify({ projects: [] })); + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + + test("getActivityList drops mid-list additions removed during the workflow probe", async () => { + // The workflow-run bootstrap for late merge candidates awaits disk; a + // cross-process removal landing during that probe is invisible to every + // guard view captured before it. The final post-probe snapshot re-read + // must drop the entry instead of riding the deleted id back into the + // renderer (the process-local subscription cannot correct it). + const { config, historyService, cleanup } = await createTestHistoryService(); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const workspaceId = "late-then-removed"; + const projectPath = path.join(config.rootDir, "project"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(workspaceId, 888); + } + return realGetAllSnapshots(options); + } + ); + listStatusSnapshotsSpy.mockImplementation(async () => { + // Another backend removes the workspace while the probe is awaited: + // its persisted metadata entry disappears, unseen by this process's + // tombstones. + await new ExtensionMetadataService(metadataPath).deleteWorkspace(workspaceId); + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + + test("getActivityList drops retained entries removed during late workflow probes", async () => { + // The retained-entry filter runs before the late-candidate workflow + // probes await disk. A cross-process removal of an ALREADY-RETAINED + // workspace landing during those probes is invisible to every view the + // filter used — without the post-probe re-filter the removed id rides + // the response back into the renderer with no event to correct it. + const { config, historyService, cleanup } = await createTestHistoryService(); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + const listStatusSnapshotsSpy = spyOn(WorkflowRunStore.prototype, "listRunStatusSnapshots"); + try { + const retainedId = "retained-then-removed"; + const lateId = "late-registered"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: retainedId, + name: retainedId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + await extensionMetadata.updateRecency(retainedId, 555); + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + // Another backend registers a NEW workspace mid-list so the + // merge has a late candidate whose probe awaits disk. + await config.addWorkspace(projectPath, { + id: lateId, + name: lateId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + await extensionMetadata.updateRecency(lateId, 777); + } + return realGetAllSnapshots(options); + } + ); + let probeCalls = 0; + listStatusSnapshotsSpy.mockImplementation(async () => { + probeCalls += 1; + if (probeCalls === 2) { + // The late candidate's probe is awaited: another backend removes + // the RETAINED workspace — config deregistration first (the real + // removeUnlocked order), then the metadata entry deletion unseen + // by this process's tombstones. + await config.removeWorkspace(retainedId); + await new ExtensionMetadataService(metadataPath).deleteWorkspace(retainedId); + } + return []; + }); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[lateId]).toBeDefined(); + expect(activityList?.[retainedId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + listStatusSnapshotsSpy.mockRestore(); + await cleanup(); + } + }); + + test("getActivityList drops snapshotless legacy entries removed mid-list", async () => { + // A legacy id-less config entry's stable id is resolved authoritatively + // during enumeration and can never appear in the raw config-id baseline, + // so the raw-superset removal comparison is blind to it. If another + // backend removes the workspace while the per-id reads run, a + // snapshotless (workflow-only) entry has no metadata-file revalidation + // to catch it either — the authoritative findWorkspace recheck must + // drop it instead of reinserting the removed workspace. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "legacy-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Workflow-only activity: no persisted snapshot. + await workspaceService.emitWorkflowRunActivity({ + workspaceId: stableId, + runId: "legacy-run", + status: "running", + }); + // Simulate the cross-process removal between the entry computation and + // the revalidation phase: the fresh metadata re-read is the first + // revalidation step, so rewriting config.json there lands mid-list. + const realGetAllSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + let snapshotCalls = 0; + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation( + async (options?: { throwOnError?: boolean }) => { + snapshotCalls += 1; + if (snapshotCalls === 2) { + await fsPromises.writeFile(configPath, JSON.stringify({ projects: [] })); + } + return realGetAllSnapshots(options); + } + ); + try { + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[stableId]).toBeUndefined(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("drops retained legacy entries removed during the mid-list identity scan", async () => { + // An id-less legacy workspace is retained on the strength of the + // mid-list authoritative enumeration — which can observe the stable id + // right before another backend deregisters it and deletes its metadata + // later in the same await. Raw config scans can never see the stable + // id and the fresh snapshot re-read predates the removal, so with zero + // late candidates nothing else re-reads: the final revalidation must + // run for retained raw-invisible ids too, or the deleted workspace + // rides every authoritative response until reconnect. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const stableId = "legacy-retained-stable-id"; + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + const configPath = path.join(config.rootDir, "config.json"); + await fsPromises.writeFile( + configPath, + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + JSON.stringify({ id: stableId, name: "legacy-ws" }) + ); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + // Persisted snapshot: the entry is RETAINED by the per-id loop, so + // the late-candidate merge has nothing to probe. + await extensionMetadata.updateRecency(stableId, 321); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + const internals = workspaceService as unknown as { + enumerateAuthoritativeWorkspaceIds(): Promise>; + }; + const realEnumerate = internals.enumerateAuthoritativeWorkspaceIds.bind(workspaceService); + let enumerateCalls = 0; + internals.enumerateAuthoritativeWorkspaceIds = async () => { + enumerateCalls += 1; + const ids = await realEnumerate(); + if (enumerateCalls === 2) { + // The removal lands INSIDE the mid-list enumeration await, after + // the enumeration observed the id: config deregistration first + // (the real removal write order), then the metadata deletion by + // another backend (no local tombstone). + await fsPromises.writeFile(configPath, JSON.stringify({ projects: [] })); + await new ExtensionMetadataService(metadataPath).deleteWorkspace(stableId); + } + return ids; + }; + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[stableId]).toBeUndefined(); + } finally { + await cleanup(); + } + }); + + test("getActivityList quarantines a deterministically corrupt metadata file", async () => { + // Parse/structure corruption fails identically on every retry, so a + // strict read that only rethrows would leave activity hydration broken + // across restarts until some unrelated writer replaced the file. The + // strict path quarantines the bytes (preserved for inspection, never + // silently deleted) and the resulting empty state is authoritative. + // Note: a valid file with version !== 1 is deliberately NOT here — that + // is a newer build's schema, treated as unsupported (propagated, never + // quarantined/reset) so a downgrade round-trip cannot destroy it. + const corruptFiles = [ + "{not json", + JSON.stringify({ version: 1, workspaces: [] }), + JSON.stringify({ version: 1, workspaces: "bogus" }), + JSON.stringify({ version: 1, workspaces: null }), + ]; + for (const corruptFile of corruptFiles) { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + await fsPromises.writeFile(metadataPath, corruptFile, "utf-8"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + // Lenient reads (writer paths) self-heal without quarantining. + expect((await extensionMetadata.getAllSnapshots()).size).toBe(0); + expect(await fsPromises.readFile(metadataPath, "utf-8")).toBe(corruptFile); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).toEqual({}); + // The corrupt bytes were moved aside, not destroyed. + expect(await fsPromises.readFile(`${metadataPath}.corrupt`, "utf-8")).toBe(corruptFile); + // Quarantine must leave a valid EMPTY main file behind (never a + // missing path): readers of a missing-main-plus-sidecar state treat + // it as a retryable mid-quarantine window, not authoritative empty. + expect(JSON.parse(await fsPromises.readFile(metadataPath, "utf-8"))).toEqual({ + version: 1, + workspaces: {}, + }); + } finally { + await cleanup(); + } + } + }); + + test("getActivityList returns null when the metadata path exists but cannot be read", async () => { + // Only a genuinely missing file (ENOENT) is a healthy empty state. Any + // other read failure (here EISDIR; EACCES/ENOTDIR/EIO in the field) must + // surface as the null read-failure signal instead of masquerading as an + // authoritative empty list. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + await fsPromises.mkdir(metadataPath, { recursive: true }); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + // Lenient reads (writer paths) still self-heal. + expect((await extensionMetadata.getAllSnapshots()).size).toBe(0); + + expect(await workspaceService.getActivityList()).toBeNull(); + } finally { + await cleanup(); + } + }); + + test("getActivityList drops workspaces removed while the list was computing", async () => { + // A removal that lands between the snapshot read and the response must + // not ride the delayed list past emitWorkspaceActivity's tombstone + // suppression: a renderer that already processed the removal event would + // re-insert the deleted id until the next reconnect. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "removed-mid-list"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + const readSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + spyOn(extensionMetadata, "getAllSnapshots").mockImplementationOnce(async () => { + const snapshots = await readSnapshots(); + // Simulates a concurrent removal completing after this request read + // its snapshot view but before the response was assembled — in the + // real removeUnlocked order: config deregistration first, then the + // metadata deletion. + await config.removeWorkspace(workspaceId); + await extensionMetadata.deleteWorkspace(workspaceId); + return snapshots; + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + } finally { + await cleanup(); + } + }); + + test("getActivityList drops entries whose metadata another process removed mid-list", async () => { + // XUM_ALLOW_MULTIPLE_INSTANCES: a removal in another backend never + // reaches this process's in-memory tombstones, so the final response + // revalidates against a fresh read of the shared file instead. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "removed-by-other-process"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const metadataPath = path.join(config.rootDir, "extensionMetadata.json"); + const extensionMetadata = new ExtensionMetadataService(metadataPath); + await extensionMetadata.updateRecency(workspaceId, 100); + const readSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + spyOn(extensionMetadata, "getAllSnapshots").mockImplementationOnce(async (options) => { + const snapshots = await readSnapshots(options); + // Simulates another backend's removal landing after this request read + // its snapshot view: rewrite the shared file without the entry, with + // no in-process deleteWorkspace tombstone. Faithful to the removal + // protocol's write order (removeUnlocked deregisters config BEFORE + // deleting metadata): a vanished snapshot with config still + // registering the id is a corruption-reset lookalike and must be + // retained, so removal simulations must deregister first. + await config.removeWorkspace(workspaceId); + await fsPromises.writeFile( + metadataPath, + JSON.stringify({ version: 1, workspaces: {} }), + "utf-8" + ); + return snapshots; + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + // The removal was detected from foreign evidence — and retained as a + // local tombstone: cache eviction alone cannot stop a LATE local + // producer (workflow-run/bash-monitor completion) from re-emitting + // the removed incarnation's activity right after this authoritative + // response dropped it, because emitWorkspaceActivity's + // isWorkspaceDeleted check only knows local removals. + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(true); + // A late producer's write stays unpersisted (transient) instead of + // recreating the removed entry on disk. + await extensionMetadata.updateRecency(workspaceId, 200); + expect((await extensionMetadata.getAllSnapshots()).has(workspaceId)).toBe(false); + } finally { + await cleanup(); + } + }); + + test("getActivityList drops entries another process deregistered from config mid-list", async () => { + // Covers entries without a persisted snapshot too: the metadata-file + // revalidation cannot see workflow/bash-monitor-only entries, so final + // membership is also re-checked against the shared config state. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const workspaceId = "deregistered-by-other-process"; + const projectPath = path.join(config.rootDir, "project"); + await config.addWorkspace(projectPath, { + id: workspaceId, + name: workspaceId, + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency(workspaceId, 100); + const readSnapshots = extensionMetadata.getAllSnapshots.bind(extensionMetadata); + spyOn(extensionMetadata, "getAllSnapshots").mockImplementationOnce(async (options) => { + const snapshots = await readSnapshots(options); + // Simulates another backend deregistering the workspace after this + // request read its snapshot view. The metadata entry stays behind, so + // only the fresh config membership check can catch it. + await config.removeWorkspace(workspaceId); + return snapshots; + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList).not.toBeNull(); + expect(activityList?.[workspaceId]).toBeUndefined(); + // Foreign removals proven by the list guards publish a local + // tombstone (late-producer suppression — see the metadata-removal + // test above). + expect(extensionMetadata.isWorkspaceDeleted(workspaceId)).toBe(true); + } finally { + await cleanup(); + } + }); + + test("getActivityList returns null on metadata read failure instead of {}", async () => { + // With scoping, {} is a valid authoritative answer that clears renderer + // state; failures must be distinguishable (null) so the renderer keeps + // its last-known snapshots and retries. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + const snapshotsSpy = spyOn(extensionMetadata, "getAllSnapshots").mockImplementation(() => + Promise.reject(new Error("metadata unreadable")) + ); + try { + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + expect(await workspaceService.getActivityList()).toBeNull(); + } finally { + snapshotsSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + + test("falls back to the unscoped union when config workspaces cannot be listed", async () => { + // Real on-disk corruption shapes. loadConfigOrDefault SWALLOWS the first + // (parse failure) and lenient-normalizes the rest (parseable but + // structurally invalid) into an empty/partial workspace view unless + // callers opt into the strict read. Without throwOnError + strict + // structural validation, each of these states would silently wipe every + // metadata entry (prune sees an "empty" config) and drop every live + // entry from the list instead of reaching the fail-open fallback. + const corruptConfigs = [ + "{not json", + JSON.stringify({ projects: {} }), + JSON.stringify({ projects: [["/tmp/project", { workspaces: "bogus" }]] }), + // Arrays pass typeof "object": lenient normalization turns an + // array-valued project config into a project with no workspaces. + JSON.stringify({ projects: [["/tmp/project", []]] }), + ]; + for (const corruptConfig of corruptConfigs) { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("possibly-live", 100); + await fsPromises.writeFile(path.join(config.rootDir, "config.json"), corruptConfig); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + // Fail open: without a trustworthy config view, stale ids cannot be + // told apart from live ones, so nothing may be dropped from the list + // or pruned from disk. + const activityList = await workspaceService.getActivityList(); + expect(activityList?.["possibly-live"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); + } finally { + await cleanup(); + } + } + }); + + test("falls back to the unscoped union when config.json exists but cannot be read", async () => { + // EISDIR here; EACCES/ENOTDIR/EIO in the field. existsSync-style probes + // report all of these as "missing", which would masquerade as an empty + // config and let the prune delete every metadata entry instead of + // failing open. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("possibly-live", 100); + const configPath = path.join(config.rootDir, "config.json"); + await fsPromises.rm(configPath, { force: true }); + await fsPromises.mkdir(configPath); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.["possibly-live"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); + } finally { + await cleanup(); + } + }); + + test("fails open when a legacy workspace's identity lookup fails", async () => { + // A legacy config entry without an id resolves its authoritative stable + // id from its session metadata.json. If that file is unreadable or + // unparseable, the lenient path substitutes the generated path id — the + // strict enumeration must instead propagate the failure so the prune + // cannot classify the real stable id's entries as stale. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("legacy-stable-id", 100); + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + // Corrupt the metadata.json holding that entry's stable id. + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile(path.join(legacySessionDir, "metadata.json"), "{not json"); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + // Fail open: the identity of the legacy workspace is unknowable, so + // nothing may be dropped from the list or pruned from disk. + expect(activityList?.["legacy-stable-id"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("legacy-stable-id")).toBe(true); + } finally { + await cleanup(); + } + }); + + test("fails open when a legacy metadata.json parses without a usable id", async () => { + // Successful JSON parsing does not establish identity: `{}` (or an + // array) passes the parse but resolves an id-less entry, and the raw + // config has no id to contribute. Strict enumeration must fail closed + // exactly like the unparseable case above, or the prune classifies the + // real stable id's entries as stale and deletes them. + const idlessMetadataFiles = ["{}", "[]"]; + for (const idlessMetadataFile of idlessMetadataFiles) { + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("legacy-stable-id", 100); + const projectPath = path.join(config.rootDir, "project"); + const workspacePath = path.join(projectPath, "legacy-ws"); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[projectPath, { workspaces: [{ path: workspacePath }] }]], + }) + ); + const legacySessionDir = config.getSessionDir( + config.generateLegacyId(projectPath, workspacePath) + ); + await fsPromises.mkdir(legacySessionDir, { recursive: true }); + await fsPromises.writeFile( + path.join(legacySessionDir, "metadata.json"), + idlessMetadataFile + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + + const activityList = await workspaceService.getActivityList(); + expect(activityList?.["legacy-stable-id"]?.recency).toBe(100); + expect((await extensionMetadata.getAllSnapshots()).has("legacy-stable-id")).toBe(true); + } finally { + await cleanup(); + } + } + }); + + test("never prunes entries whose config entry is discarded by normalization", async () => { + // A parseable config entry that lenient normalization filters out (null + // project path): the workspace vanishes from the normalized view — and + // thus from the activity list, matching every other renderer surface — + // but its metadata entry must survive the prune. Two guards enforce it: + // strict loads reject the malformed project key outright (aborting the + // prune, fail closed), and the raw-superset union spares the inline id + // even if enumeration were to succeed. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const extensionMetadata = new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ); + await extensionMetadata.updateRecency("possibly-live", 100); + await fsPromises.writeFile( + path.join(config.rootDir, "config.json"), + JSON.stringify({ + projects: [[null, { workspaces: [{ id: "possibly-live", path: "/tmp/x" }] }]], + }) + ); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata, + }); + await workspaceService.getActivityList(); + expect((await extensionMetadata.getAllSnapshots()).has("possibly-live")).toBe(true); + } finally { + await cleanup(); + } + }); + + test("prunes the extension metadata entry after a workspace is removed", async () => { + const { historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "remove-prunes-metadata"; + const tempRoot = await fsPromises.mkdtemp(path.join(tmpdir(), "mux-remove-metadata-")); + try { + const sessionRoot = path.join(tempRoot, "sessions"); + await fsPromises.mkdir(path.join(sessionRoot, workspaceId), { recursive: true }); + + const deleteWorkspace = mock(() => Promise.resolve()); + const extensionMetadata = { + ...mockExtensionMetadataService, + deleteWorkspace, + } as unknown as ExtensionMetadataService; + const mockConfig: Partial = { + rootDir: path.join(tempRoot, "root"), + srcDir: "/tmp/src", + getSessionDir: mock((id: string) => path.join(sessionRoot, id)), + removeWorkspace: mock(() => Promise.resolve()), + findWorkspace: mock(() => null), + loadConfigOrDefault: mock(() => ({ projects: new Map() })), + // The discard verifies deregistration against the persisted superset + // (and the findWorkspace mock above) before deleting. + readPersistedWorkspaceIdSuperset: mock(() => new Set()), + getAllWorkspaceMetadata: mock(() => Promise.resolve([])), + }; + const workspaceService = createWorkspaceServiceForTest({ + config: mockConfig, + historyService, + extensionMetadata, + aiService: createMockAIService({ + isStreaming: mock(() => false), + stopStream: mock(() => Promise.resolve(Ok(undefined))), + getWorkspaceMetadata: mock(() => + Promise.resolve( + Ok(createFrontendWorkspaceMetadata({ id: workspaceId, name: workspaceId })) + ) + ), + }), + }); + + const removeResult = await workspaceService.remove(workspaceId, true); + expect(removeResult.success).toBe(true); + expect(deleteWorkspace).toHaveBeenCalledWith(workspaceId); + } finally { + await fsPromises.rm(tempRoot, { recursive: true, force: true }); + await cleanup(); + } + }); + + test("discardExtensionMetadataEntry swallows deletion failures", async () => { + // Rollback paths (e.g. TaskService's failed task-create rollback) call + // this best-effort; a metadata disk failure must not abort the rollback. + const { config, historyService, cleanup } = await createTestHistoryService(); + try { + const deleteWorkspace = mock(() => Promise.reject(new Error("disk full"))); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + extensionMetadata: { + ...mockExtensionMetadataService, + deleteWorkspace, + } as unknown as ExtensionMetadataService, + }); + + await workspaceService.discardExtensionMetadataEntry("rollback-ws"); + expect(deleteWorkspace).toHaveBeenCalledWith("rollback-ws"); + } finally { + await cleanup(); + } + }); +}); + describe("WorkspaceService workflow invocation events", () => { test("emits workflow slash invocation rows through the active session chat stream", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 717c81b798..f350eed887 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2135,7 +2135,10 @@ export class WorkspaceService extends EventEmitter { // the clear. Cost: an occasional duplicate emit, which renderers apply idempotently. this.lastEmittedBashMonitorCounts.delete(workspaceId); void this.extensionMetadata - .getSnapshot(workspaceId) + // Strict: emitting a suspect (partial-main) snapshot after a failed + // sidecar reconcile would clear goal/status in the renderer; the + // catch below already retains "unknown" and re-emits on next change. + .getSnapshot(workspaceId, { throwOnError: true }) .then((snapshot) => { this.emitWorkspaceActivity(workspaceId, snapshot); // Record only after a successful emit. Re-read the count because the emit merges @@ -2160,6 +2163,13 @@ export class WorkspaceService extends EventEmitter { Promise> >(); private readonly activeWorkflowRunIdsByWorkspace = new Map>(); + // Workspaces where NONZERO workflow-run activity was actually observed + // (bootstrap probe, run-status event, or list read). This — not cache + // presence — is the zero-count tombstone signal: the activity list's own + // probe installs an (empty) cache for every scoped id, so cache existence + // would fabricate recency:0 entries for every idle config-known workspace + // from the second list on, re-bloating the payload this scoping trims. + private readonly workflowRunSeenWorkspaces = new Set(); // Debounce post-compaction metadata refreshes (file_edit_* can fire rapidly) private readonly postCompactionRefreshTimers = new Map>(); @@ -2401,6 +2411,19 @@ export class WorkspaceService extends EventEmitter { }); this.setupMetadataListeners(); this.setupInitMetadataListeners(); + // A cleared tombstone means a removed id was re-registered (downgraded + // concurrent backend re-creating a deterministic legacy id): evict the + // process-local activity caches bootstrapped for the REMOVED incarnation + // so the revived workspace re-probes disk instead of showing ghost + // workflow-run counts from session state that removal deleted. + // Guarded like the backgroundProcessManager subscriptions above: tests + // may construct WorkspaceService with a partial ExtensionMetadataService + // stub. + if (typeof this.extensionMetadata.setTombstoneClearedListener === "function") { + this.extensionMetadata.setTombstoneClearedListener((workspaceId) => { + this.evictWorkspaceActivityCaches(workspaceId); + }); + } // r63 startup self-heal: reclaim removal tombstones left behind by a // removal whose config deregistration AND tombstone rollback both failed // — otherwise that workspace stays registered but refused every mutation @@ -4059,29 +4082,66 @@ export class WorkspaceService extends EventEmitter { } private async getActiveWorkflowRunIds(workspaceId: string): Promise> { + const activeRunIds = await this.resolveActiveWorkflowRunIds(workspaceId); + if ( + activeRunIds.size > 0 && + // Installation re-check in THIS continuation: an eviction (removal, or + // a tombstone lifted for revival) can land in the microtask gap after + // resolve's own final check. A detached set must not repopulate the + // seen marker — the evicted id's caches are empty, so a stale marker + // would fabricate zero-count entries for the idle revived workspace. + this.activeWorkflowRunIdsByWorkspace.get(workspaceId) === activeRunIds + ) { + // A list- or event-delivered nonzero count is what makes a later zero + // meaningful as a tombstone (see workflowRunSeenWorkspaces). + this.workflowRunSeenWorkspaces.add(workspaceId); + } + return activeRunIds; + } + + private async resolveActiveWorkflowRunIds(workspaceId: string): Promise> { assert(workspaceId.length > 0, "getActiveWorkflowRunIds requires workspaceId"); - const cached = this.activeWorkflowRunIdsByWorkspace.get(workspaceId); - if (cached != null) { - const bootstrap = this.activeWorkflowRunIdBootstrapsByWorkspace.get(workspaceId); - if (bootstrap != null) { - await bootstrap; + // Bounded retry: evictWorkspaceActivityCaches (removal, or a tombstone + // lifted for re-registration) can race an in-flight bootstrap. A waiter + // that captured the pre-eviction Set would otherwise return the removed + // incarnation's runs — ghost counts with no future terminal event to + // clear them — so after every await the Set is re-verified as still the + // installed cache and the read restarts when it was evicted. + for (let attempt = 0; attempt < 3; attempt++) { + const cached = this.activeWorkflowRunIdsByWorkspace.get(workspaceId); + if (cached != null) { + const bootstrap = this.activeWorkflowRunIdBootstrapsByWorkspace.get(workspaceId); + if (bootstrap != null) { + await bootstrap; + } + if (this.activeWorkflowRunIdsByWorkspace.get(workspaceId) !== cached) { + continue; + } + return cached; } - return cached; - } - // Install the shared Set before awaiting disk so parallel workflow status events - // mutate the same cache instead of racing to replace each other after bootstrap. - const activeRunIds = new Set(); - this.activeWorkflowRunIdsByWorkspace.set(workspaceId, activeRunIds); - const bootstrap = this.populateActiveWorkflowRunIds(workspaceId, activeRunIds); - this.activeWorkflowRunIdBootstrapsByWorkspace.set(workspaceId, bootstrap); - try { - return await bootstrap; - } finally { - if (this.activeWorkflowRunIdBootstrapsByWorkspace.get(workspaceId) === bootstrap) { - this.activeWorkflowRunIdBootstrapsByWorkspace.delete(workspaceId); + // Install the shared Set before awaiting disk so parallel workflow status events + // mutate the same cache instead of racing to replace each other after bootstrap. + const activeRunIds = new Set(); + this.activeWorkflowRunIdsByWorkspace.set(workspaceId, activeRunIds); + const bootstrap = this.populateActiveWorkflowRunIds(workspaceId, activeRunIds); + this.activeWorkflowRunIdBootstrapsByWorkspace.set(workspaceId, bootstrap); + try { + await bootstrap; + } finally { + if (this.activeWorkflowRunIdBootstrapsByWorkspace.get(workspaceId) === bootstrap) { + this.activeWorkflowRunIdBootstrapsByWorkspace.delete(workspaceId); + } } + if (this.activeWorkflowRunIdsByWorkspace.get(workspaceId) !== activeRunIds) { + continue; + } + return activeRunIds; } + // Eviction churn exhausted the retries (pathological): probe disk once + // more DETACHED so the caller still gets current durable state without + // installing a cache that may itself be mid-eviction. + return this.populateActiveWorkflowRunIds(workspaceId, new Set()); } private async updateActiveWorkflowRunCount(event: { @@ -4089,13 +4149,32 @@ export class WorkspaceService extends EventEmitter { runId: string; status: WorkflowRunStatus; }): Promise { - const activeRunIds = await this.getActiveWorkflowRunIds(event.workspaceId); - if (isActiveWorkflowRunStatus(event.status)) { - activeRunIds.add(event.runId); - } else { - activeRunIds.delete(event.runId); + // Bounded retry (same reason as resolveActiveWorkflowRunIds): an + // eviction can land in the microtask gap after the cache read resolves, + // so the mutation below would hit a detached incarnation — and must not + // mark the seen set for an id whose caches were just evicted, or the + // idle revived workspace emits fabricated zero-count entries forever. + let detachedSize = 0; + for (let attempt = 0; attempt < 3; attempt++) { + const activeRunIds = await this.getActiveWorkflowRunIds(event.workspaceId); + if (isActiveWorkflowRunStatus(event.status)) { + activeRunIds.add(event.runId); + } else { + activeRunIds.delete(event.runId); + } + detachedSize = activeRunIds.size; + if (this.activeWorkflowRunIdsByWorkspace.get(event.workspaceId) !== activeRunIds) { + continue; + } + if (isActiveWorkflowRunStatus(event.status)) { + this.workflowRunSeenWorkspaces.add(event.workspaceId); + } + return activeRunIds.size; } - return activeRunIds.size; + // Eviction churn exhausted the retries (pathological): report the last + // detached mutation's size without seen-marking, mirroring the detached + // disk-probe fallback in resolveActiveWorkflowRunIds. + return detachedSize; } private mergeCachedActiveWorkflowRuns( @@ -4183,10 +4262,23 @@ export class WorkspaceService extends EventEmitter { assert(event.workspaceId.length > 0, "emitWorkflowRunActivity requires workspaceId"); assert(event.runId.length > 0, "emitWorkflowRunActivity requires runId"); await this.updateActiveWorkflowRunCount(event); - this.emitWorkspaceActivity( - event.workspaceId, - await this.extensionMetadata.getSnapshot(event.workspaceId) - ); + let snapshot: WorkspaceActivitySnapshot | null; + try { + snapshot = await this.extensionMetadata.getSnapshot(event.workspaceId, { + throwOnError: true, + }); + } catch (error) { + // Emitting a suspect (partial-main) snapshot after a failed sidecar + // reconcile would clear goal/status in the renderer with no repair + // event. Retention is recoverable: the run count is already cached, + // so the next emit or list read delivers it. + log.debug("Skipping workflow-run activity emit after failed snapshot read", { + workspaceId: event.workspaceId, + error, + }); + return; + } + this.emitWorkspaceActivity(event.workspaceId, snapshot); } /** @@ -4198,16 +4290,27 @@ export class WorkspaceService extends EventEmitter { workspaceId: string, snapshot: WorkspaceActivitySnapshot | null ): void { - this.emit("activity", { + const activity = this.mergeCurrentActiveBashMonitorCount( workspaceId, - activity: this.mergeCurrentActiveBashMonitorCount( + this.mergeCachedActiveWorkflowRuns( workspaceId, - this.mergeCachedActiveWorkflowRuns( - workspaceId, - this.overlayPendingGoal(workspaceId, snapshot) - ) - ), - }); + this.overlayPendingGoal(workspaceId, snapshot) + ) + ); + // A late in-flight producer (e.g. a stream-abort stop-status handler mid + // todo read) can complete after removal deleted this workspace's + // metadata entry. Its disk write is already blocked by the write + // tombstone; suppress the broadcast too, or the renderer re-inserts the + // removed id into its activity map after having processed the + // metadata-removal event. Null (clearing) emissions stay allowed — the + // check runs on the MERGED payload (not the raw snapshot) because the + // workflow/bash-monitor cache overlays above can turn a null snapshot + // into a non-null activity from still-populated caches, which would + // re-insert the deleted id just the same. + if (activity !== null && this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { + return; + } + this.emit("activity", { workspaceId, activity }); } /** @@ -4978,6 +5081,13 @@ export class WorkspaceService extends EventEmitter { const config = this.config.loadConfigOrDefault({ throwOnError: true }); const knownIds = new Set(allMetadata.map((metadata) => metadata.id)); + // Same rationale as the extension-metadata prune: normalization is lossy, + // and a live workspace whose config entry gets filtered (e.g. invalid + // project path) must not have its session data reaped as an orphan. + // Throws on unreadable/unparseable config, aborting this cleanup. + for (const persistedId of this.config.readPersistedWorkspaceIdSuperset()) { + knownIds.add(persistedId); + } // The config load-time migration (removeLegacyXumChatEntries) drops the // removed Chat with Xum workspace from config, which would make its // session dir look orphaned here. Preserve the transcript: a downgraded @@ -6666,6 +6776,11 @@ export class WorkspaceService extends EventEmitter { removedFromConfig = true; this.autoTitlingWorkspaces.delete(workspaceId); + // Deregistration succeeded: drop the workspace's activity/status entry + // so extensionMetadata.json stays bounded (stale entries were + // historically never pruned and grew monotonically, issue #3959). + await this.discardExtensionMetadataEntry(workspaceId); + if (removedMetadata || persistedWorkspace) { await this.syncCodeWorkspaceFiles( removedMetadata ?? { @@ -13158,30 +13273,322 @@ export class WorkspaceService extends EventEmitter { } } - async getActivityList(): Promise | null> { + /** + * Best-effort removal of a deregistered workspace's activity/status entry + * from extensionMetadata.json. Used by remove() and by rollback paths that + * deregister via config.removeWorkspace directly (e.g. TaskService's failed + * task-create rollback, where a send that failed mid-create may already + * have scheduled metadata writes that would recreate the entry after + * deregistration). A missed delete is reclaimed by the one-time + * pruneStaleExtensionMetadataOnce pass on a later process start. + */ + async discardExtensionMetadataEntry(workspaceId: string): Promise { try { - const snapshots = await this.extensionMetadata.getAllSnapshots(); - const workspaceIds = new Set(snapshots.keys()); - for (const workspaceId of this.activeWorkflowRunIdsByWorkspace.keys()) { - workspaceIds.add(workspaceId); + // Deleting also write-tombstones the id for the rest of this process, + // so verify deregistration actually landed before publishing it: + // saveConfig swallows write failures, meaning config.removeWorkspace + // can resolve while the workspace is still persisted in config.json — + // tombstoning a still-live id would suppress all of its future + // activity writes. A failed verification (unreadable config) skips the + // delete too; like a missed delete, the entry is reclaimed by a later + // process start's prune, which re-checks against config. + // + // Deliberately NOT getAllWorkspaceMetadata here: that walks every + // configured workspace with per-workspace fs probes — an O(n) + // traversal just to check one id. The raw persisted superset (which + // throws on an unreadable config, also making findWorkspace's lenient + // internal load safe below) covers entries normalization would drop, + // and the targeted findWorkspace lookup covers normalized/legacy ids + // (metadata.json / generated legacy ids) the raw scan cannot see. + const knownIds = this.config.readPersistedWorkspaceIdSuperset(); + // throwOnError: a lenient findWorkspace swallows unreadable legacy + // metadata.json files, and "identity unknowable" must fail closed here + // (skip the delete) rather than read as "not registered". + if ( + knownIds.has(workspaceId) || + this.config.findWorkspace(workspaceId, { throwOnError: true }) != null + ) { + log.debug("Skipping extension metadata discard: workspace still persisted in config", { + workspaceId, + }); + return; } - for (const workspaceId of this.bashMonitorSeenWorkspaces) { - workspaceIds.add(workspaceId); + await this.extensionMetadata.deleteWorkspace(workspaceId); + // Removal-side counterpart of the tombstone-cleared eviction: drop the + // process-local workflow/bash-monitor caches for the removed id so a + // later re-registration never inherits activity bootstrapped from + // session state that removal deleted. + this.evictWorkspaceActivityCaches(workspaceId); + } catch (error) { + log.debug("Failed to prune extension metadata after workspace deregistration", { + workspaceId, + error: getErrorMessage(error), + }); + } + } + + /** + * Strict authoritative enumeration for destructive/identity decisions: + * every primary workspace id PLUS legacy alias identities (a second + * resolvable compatibility file findWorkspace still vouches for — see + * Config.getAllWorkspaceMetadata's legacyAliasIds). Known-id sets built + * without the aliases would prune or drop live alias-keyed activity. + */ + private async enumerateAuthoritativeWorkspaceIds(): Promise> { + const legacyAliasIds = new Set(); + const ids = new Set( + (await this.config.getAllWorkspaceMetadata({ throwOnError: true, legacyAliasIds })).map( + (metadata) => metadata.id + ) + ); + for (const aliasId of legacyAliasIds) { + ids.add(aliasId); + } + return ids; + } + + /** + * Evict process-local activity caches for a removed (or removed-then- + * revived) workspace id. The caches re-bootstrap from disk on next access; + * an in-flight bootstrap keeps populating its orphaned Set harmlessly. + */ + private evictWorkspaceActivityCaches(workspaceId: string): void { + this.activeWorkflowRunIdsByWorkspace.delete(workspaceId); + this.activeWorkflowRunIdBootstrapsByWorkspace.delete(workspaceId); + this.bashMonitorSeenWorkspaces.delete(workspaceId); + this.workflowRunSeenWorkspaces.delete(workspaceId); + } + + /** + * One-time lazy cleanup for pre-existing deployments: drop + * extensionMetadata.json entries whose workspace no longer exists in + * config. remove() prunes going forward, but entries that leaked before + * that hook existed (issue #3959) would otherwise bloat the file — and + * every serialized rewrite of it — forever. Runs at most once per process + * from the activity bootstrap path; never a per-read scan. + */ + private prunedStaleExtensionMetadata = false; + /** + * Returns the config-known workspace ids captured during the prune so the + * first activity bootstrap can reuse them for scoping — + * getAllWorkspaceMetadata walks every workspace with per-workspace disk + * probes, which large deployments should not pay twice in the + * latency-sensitive bootstrap. `knownIds` is the FULL raw-plus-normalized + * union the prune spared from deletion: scoping to anything narrower (the + * normalized view alone) would drop raw-registered ids the normalized + * view cannot see (e.g. a project pair shadowed by a duplicate path key), + * and if the bootstrap's later raw refreshes then failed transiently, the + * authoritative response would omit a live workspace whose id this READ + * already loaded successfully — clearing its renderer state with no event + * to correct it. `enumeratedIds` is what the strict enumeration itself + * vouched for (see scopeEnumerationIds: only those ids may treat + * enumeration absence as removal evidence). Null when the prune was + * skipped or failed. + */ + private async pruneStaleExtensionMetadataOnce(): Promise<{ + knownIds: Set; + enumeratedIds: ReadonlySet; + } | null> { + if (this.prunedStaleExtensionMetadata) { + return null; + } + // Latch before awaiting so concurrent bootstraps don't queue redundant + // prunes; a failed attempt is retried on the next process start rather + // than on every read. + this.prunedStaleExtensionMetadata = true; + try { + let prunedScope: { knownIds: Set; enumeratedIds: ReadonlySet } | null = null; + const prunedCount = await this.extensionMetadata.pruneMissingWorkspaces( + async () => { + // Invoked inside the file's serialized mutation AFTER the file load, + // reading config fresh from disk (see pruneMissingWorkspaces), so a + // concurrently created workspace — even in another backend process — + // cannot lose its just-written entry. + // + // Union of two views, both of which throw (aborting the prune, caught + // below) rather than resolving with a silently lossy id set: + // - the raw persisted superset covers entries loadConfigOrDefault's + // validation/normalization would filter or discard (see + // readPersistedWorkspaceIdSuperset), so a live workspace with a + // malformed config entry is never treated as removed; + // - the strict normalized view covers ids produced by in-memory + // config migrations that are not yet persisted verbatim. + // A missing config file resolves as a healthy empty set in both. + const knownIds = this.config.readPersistedWorkspaceIdSuperset(); + const enumeratedIds = await this.enumerateAuthoritativeWorkspaceIds(); + for (const workspaceId of enumeratedIds) { + knownIds.add(workspaceId); + } + prunedScope = { knownIds, enumeratedIds }; + return knownIds; + }, + async () => { + // Mid-prune re-registration recheck. The first callback's strict + // enumeration walks a session lookup per configured workspace; + // rerunning it doubles that cost on exactly the stale-heavy + // deployments this prune exists to fix, while the serialized + // metadata queue blocks live recency/status writes. The raw id view + // is complete registration evidence whenever every persisted + // workspace entry carries its id inline — only id-less legacy + // entries (whose stable id lives in session metadata.json) can be + // registered raw-invisibly, so the enumeration is repeated only + // when such entries exist. Both reads throw on failure, aborting + // the prune rather than deleting on lossy evidence. + const evidence = this.config.readPersistedWorkspaceIdEvidence(); + if (!evidence.hasWorkspaceEntriesWithoutIds) { + return evidence.ids; + } + for (const workspaceId of await this.enumerateAuthoritativeWorkspaceIds()) { + evidence.ids.add(workspaceId); + } + return evidence.ids; + } + ); + if (prunedCount > 0) { + log.info(`Pruned ${prunedCount} stale extension metadata entries`); } + return prunedScope; + } catch (error) { + log.debug("Failed to prune stale extension metadata entries", { error }); + return null; + } + } + + async getActivityList(): Promise | null> { + try { + // On the first bootstrap the prune already enumerated the config; reuse + // that id set instead of paying the per-workspace disk walk twice. + // Baseline for the post-await cross-process removal revalidation at + // the end of this method: captured before ANY await (including the + // first-bootstrap prune, whose enumeration another backend's removal + // could otherwise outdate before this baseline is read) so ids + // deregistered from the shared config while this list computes can be + // told apart from ids the raw scan can never see. + let initialConfigIds: ReadonlySet | null = null; try { - for (const metadata of await this.config.getAllWorkspaceMetadata()) { - workspaceIds.add(metadata.id); + initialConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + } catch { + initialConfigIds = null; + } + const prefetchedScope = await this.pruneStaleExtensionMetadataOnce(); + // throwOnError: the default load self-heals an unreadable/malformed + // metadata file into an empty one, which this list would then present + // as an authoritative "no activity anywhere" answer — the renderer + // applies that by wiping every cached streaming/status/goal snapshot + // with no retry (the subscription stays connected). Throwing to the + // null-returning catch below instead keeps last-known renderer state + // and lets the bootstrap retry read the real list later. + const snapshots = await this.extensionMetadata.getAllSnapshots({ throwOnError: true }); + // Scope the list to config-known workspaces. extensionMetadata.json was + // historically never pruned, so long-lived deployments accumulate stale + // entries for removed workspaces/sub-agents by the thousands (issue + // #3959: 13,901 entries / 2.89 MB / ~59 s per bootstrap while the + // sidebar needed ~246). Stale ids must neither inflate the payload nor + // trigger the per-id workflow-run bootstrap disk probe below. Known ids + // WITHOUT a snapshot still flow through the tombstone logic below — + // scoping only drops ids that are not in config at all. + let workspaceIds: Set; + // Whether scoping ended up on the fail-open legacy union (config + // unreadable): only that availability path may serve a response with + // the cross-process removal guards disabled. + let scopedFailOpen = false; + // Ids the SCOPE's strict enumeration itself vouched for, captured + // before raw-view additions: the authoritative-removal fallback below + // may treat mid-list enumeration absence as removal evidence only for + // these ids — raw-registered ids the normalized view cannot see + // (invalid project path) are legitimately absent from every + // enumeration and must not read that absence as removal. + let scopeEnumerationIds: ReadonlySet | null = null; + if (prefetchedScope != null) { + scopeEnumerationIds = prefetchedScope.enumeratedIds; + // Scope to the FULL known-id union the prune spared (raw + + // normalized), not the enumeration alone: a raw-registered id the + // normalized view cannot see was already loaded successfully by the + // prune's raw read, and relying solely on the refresh below to + // re-admit it would let a transient refresh failure turn into an + // authoritative response that omits the live workspace. + workspaceIds = prefetchedScope.knownIds; + // The prune enumerated config BEFORE the snapshot read above, so a + // workspace registered in between would be missing here — and an + // authoritative list omitting it would clear its live-arrived + // renderer state with no retry. Admit every id a fresh raw config + // view now knows, NOT just ids with a persisted snapshot: a + // concurrently registered workspace with workflow- or bash-monitor- + // only activity has no extensionMetadata entry, and its on-disk + // workflow runs are only discovered by the per-id probe below. + // (Cheap sync read; on failure the prefetched view stands and the + // miss is a transient one.) + try { + const refreshedConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + for (const workspaceId of refreshedConfigIds) { + workspaceIds.add(workspaceId); + } + } catch (error) { + log.debug("Failed to refresh config ids for first-bootstrap scoping", { error }); + } + } else { + try { + // throwOnError so a corrupted config.json actually reaches the + // fail-open fallback below instead of silently resolving as the + // empty default and dropping every live entry from the list. + workspaceIds = await this.enumerateAuthoritativeWorkspaceIds(); + scopeEnumerationIds = workspaceIds; + } catch (error) { + // Fail open: without the config view, stale ids cannot be told apart + // from live ones, and dropping live entries would strand renderer + // activity state. Fall back to the legacy unscoped union. + log.debug("Failed to scope activity list to known workspaces", { error }); + scopedFailOpen = true; + workspaceIds = new Set(snapshots.keys()); + for (const workspaceId of this.activeWorkflowRunIdsByWorkspace.keys()) { + workspaceIds.add(workspaceId); + } + for (const workspaceId of this.bashMonitorSeenWorkspaces) { + workspaceIds.add(workspaceId); + } + for (const workspaceId of this.workflowRunSeenWorkspaces) { + workspaceIds.add(workspaceId); + } } - } catch (error) { - log.debug("Failed to include all workspaces while listing activity", { error }); } + // Re-establish the raw baseline when the pre-await read failed: with a + // null baseline BOTH cross-process removal guards stay disabled while + // the response is still authoritative — a workspace another backend + // removes during the probes below would ride back into the renderer + // with no event to correct it. This retry still precedes every per-id + // probe await, so it remains a valid "registered at list start" + // baseline; ids it cannot see flow through the authoritative-identity + // path instead. If it fails again the config is genuinely unreadable + // and the fail-open scoping above already chose availability. + if (initialConfigIds == null) { + try { + initialConfigIds = this.config.readPersistedWorkspaceIdSuperset(); + } catch (error) { + // Authoritative scope but NO raw baseline even on retry: both + // cross-process removal guards would silently stay disabled on a + // response the renderer applies as authoritative — a workspace + // another backend deregisters during the probes below would ride + // back with no event to correct it. Fail the list instead (null → + // renderer keeps last-known state and retries). The fail-open + // scope keeps its availability contract: config is unreadable + // there by definition, and no baseline exists by design. + if (!scopedFailOpen) { + throw error; + } + initialConfigIds = null; + } + } const entries = await Promise.all( Array.from( workspaceIds, async (workspaceId): Promise => { const snapshot = snapshots.get(workspaceId) ?? null; - const hadWorkflowActivityCache = this.activeWorkflowRunIdsByWorkspace.has(workspaceId); + // Nonzero-observation signal, NOT cache presence: the probe below + // installs an empty cache for every scoped id, which would turn + // every idle config-known workspace into a fabricated recency:0 + // entry on the next list. + const hadWorkflowActivity = this.workflowRunSeenWorkspaces.has(workspaceId); // Bash-monitor counterpart of the workflow tombstone: a monitor that stopped // while the renderer was disconnected (or whose stop emit failed) must still // surface a zero-count entry here, otherwise the renderer's last-known @@ -13201,7 +13608,7 @@ export class WorkspaceService extends EventEmitter { if ( snapshot == null && activeWorkflowRunCount === 0 && - !hadWorkflowActivityCache && + !hadWorkflowActivity && activeBashMonitorCount === 0 && !hadBashMonitorActivityCache ) { @@ -13227,15 +13634,651 @@ export class WorkspaceService extends EventEmitter { } ) ); - return Object.fromEntries( - entries.filter( - (entry): entry is readonly [string, WorkspaceActivitySnapshot] => entry != null - ) + // Cross-process counterparts of the in-process tombstone check below: + // with XUM_ALLOW_MULTIPLE_INSTANCES another backend can remove a + // workspace while this list computes, invisible to this process's + // deletedWorkspaceIds. Re-read the (post-prune bounded) metadata file + // and the raw persisted config id superset once. Best-effort: an + // unreadable re-read skips its revalidation instead of failing an + // otherwise complete response. + let freshSnapshots: ReadonlyMap | null = null; + try { + freshSnapshots = await this.extensionMetadata.getAllSnapshots({ throwOnError: true }); + } catch { + freshSnapshots = null; + } + const freshPersistedIds: ReadonlySet | null = + freshSnapshots != null ? new Set(freshSnapshots.keys()) : null; + // Tombstones the registration evidence below may legitimately clear: + // only ones that already exist HERE, before the evidence is captured + // (the raw superset read is synchronous with this snapshot, and the + // authoritative enumeration runs later still). A same-process removal + // landing during the enumeration await publishes its tombstone after + // the evidence reads began — stale evidence still showing the id + // registered must not clear that fresh tombstone, or the pre-removal + // snapshot rides back into the renderer and late producers persist the + // entry again. Such a tombstone stays clearable by the NEXT list's + // fresh evidence if the id really is re-registered. + const clearableTombstoneIds = this.extensionMetadata.getTombstonedIds(); + let freshConfigIds: ReadonlySet | null = null; + // Whether the fresh raw view is COMPLETE registration evidence: id-less + // legacy entries register raw-invisibly (their stable id lives in + // session metadata.json), so their presence — or an unreadable raw + // view — forces the authoritative enumeration below for admission and + // removal decisions the raw comparison cannot make. + let freshConfigHasRawInvisibleEntries = false; + try { + const evidence = this.config.readPersistedWorkspaceIdEvidence(); + freshConfigIds = evidence.ids; + freshConfigHasRawInvisibleEntries = evidence.hasWorkspaceEntriesWithoutIds; + } catch { + freshConfigIds = null; + freshConfigHasRawInvisibleEntries = true; + } + // Like-for-like raw-superset comparison only: an id that WAS persisted + // in config before the awaits and is gone from the fresh raw view was + // verifiably deregistered. Ids the raw scan cannot see (legacy entries + // whose stable id lives in a session metadata.json, in-memory migration + // ids) never appear in either view — dropping them on a cheap fresh + // view would misclassify identity-lookup gaps as removals, so they are + // revalidated through the authoritative identity path below instead. + // Skipped when either superset read fails. + const isRemovedFromConfig = (workspaceId: string): boolean => + initialConfigIds != null && + freshConfigIds != null && + initialConfigIds.has(workspaceId) && + !freshConfigIds.has(workspaceId); + // Raw-invisible ids (legacy stable ids resolved from session + // metadata.json during enumeration) need their own removal + // revalidation: the raw baseline can never contain them, so the + // comparison above is blind to their cross-process removal — a + // snapshotless (workflow/bash-monitor-only) legacy entry removed + // mid-list would otherwise ride the delayed authoritative response + // back into the renderer. Revalidate against ONE fresh authoritative + // enumeration (per-id findWorkspace lookups would re-read and scan the + // whole config per entry — O(n²) on legacy-heavy first bootstraps, + // recreating the very stall this scoping removes): a verified "not + // registered" drops the entry, while an unknowable identity anywhere + // (unreadable/id-less metadata.json throws in strict mode) skips the + // recheck and conservatively retains every raw-invisible id. Computed + // only when a retained entry is actually missing from the raw + // baseline (or a raw-invisible late snapshot needs admission below), + // so modern deployments (every workspace id persisted in config) + // never pay the extra walk. + // Second trigger: a fresh-snapshot id outside the per-id scope that + // the raw view cannot vouch for. It is either a raw-invisible legacy + // id a downgraded backend registered mid-list — which the merge below + // must ADMIT, and only the authoritative enumeration can prove + // registered — or noise the merge must keep excluding; either way the + // enumeration is the only view that can tell. + const hasRawInvisibleLateSnapshotId = + freshSnapshots != null && + Array.from(freshSnapshots.keys()).some( + (workspaceId) => + !workspaceIds.has(workspaceId) && !(freshConfigIds?.has(workspaceId) ?? false) + ); + let authoritativeIds: ReadonlySet | null = null; + if ( + initialConfigIds != null && + (entries.some((entry) => entry != null && !initialConfigIds.has(entry[0])) || + hasRawInvisibleLateSnapshotId || + // Third trigger: id-less legacy entries exist, so a downgraded + // backend may have registered a raw-invisible workspace with + // workflow-only activity (no snapshot) mid-list — only the + // enumeration can discover it for the merge admission below. + // Modern deployments (every id inline) never pay this walk. + freshConfigHasRawInvisibleEntries) + ) { + try { + authoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); + } catch (error) { + log.debug("Failed to enumerate authoritative ids for removal revalidation", { error }); + authoritativeIds = null; + } + // The enumeration awaited disk: refresh the raw view so the + // like-for-like removal comparison below never compares two + // pre-removal reads. An inline-id workspace removed by another + // backend DURING the enumeration is invisible to the pre-await + // freshConfigIds (and deliberately exempt from the authoritative + // check, which skips ids present in the initial baseline), so its + // stale entry would otherwise pass the retained-entry filter with no + // cross-process event to correct it. Fresher raw evidence is + // strictly better for every downstream consumer; the tombstone-clear + // eligibility snapshot was captured before ALL evidence reads, so + // ordering stays sound. + try { + const refreshedEvidence = this.config.readPersistedWorkspaceIdEvidence(); + freshConfigIds = refreshedEvidence.ids; + freshConfigHasRawInvisibleEntries = refreshedEvidence.hasWorkspaceEntriesWithoutIds; + } catch { + freshConfigIds = null; + freshConfigHasRawInvisibleEntries = true; + } + if (freshConfigHasRawInvisibleEntries) { + // The refresh postdates the enumeration and reveals id-less + // entries, so the enumeration's DENIALS may already be stale: an + // id removed before the enumeration can have been re-registered + // id-less (with fresh cross-process activity) in the gap, and + // trusting the stale denial would drop the revived workspace and + // tombstone it. Re-enumerate ONCE so the enumeration-backed + // removal arms use the freshest capable view; a revival landing + // after this last read is the contract's out-of-scope window + // (the next list's evidence clears any republished tombstone). + // The raw view deliberately stays at its pre-re-enumeration + // read: staleness there errs toward retention (an id present in + // the older view is kept), never toward a wrong drop. On failure + // fall back to retention, not the stale denial set. + try { + authoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); + } catch (error) { + log.debug("Failed to re-enumerate authoritative ids after raw refresh", { error }); + authoritativeIds = null; + } + } + } + const isRemovedPerAuthoritativeIdentity = (workspaceId: string): boolean => + (initialConfigIds != null && + !initialConfigIds.has(workspaceId) && + // An id visible in the FRESH raw view is verifiably registered + // regardless of what the (possibly earlier) authoritative + // enumeration saw — e.g. a workspace registered after that + // enumeration ran must not read as removed. + !(freshConfigIds?.has(workspaceId) ?? false) && + authoritativeIds != null && + !authoritativeIds.has(workspaceId)) || + // Raw view unavailable (initial evidence read or post-enumeration + // refresh failed): the mid-list authoritative enumeration is the + // only usable post-removal view — without this arm an inline-id + // workspace removed during that enumeration would ride the + // authoritative response with every raw guard disabled and no + // cross-process event to repair the renderer. Confined to ids the + // scope enumeration itself vouched for (see scopeEnumerationIds). + (freshConfigIds == null && + authoritativeIds != null && + (scopeEnumerationIds?.has(workspaceId) ?? false) && + !authoritativeIds.has(workspaceId)); + // Raw-visible→raw-invisible is verifiable removal only while the + // fresh raw view is COMPLETE registration evidence. With id-less + // legacy entries present, the id may have been removed and + // RE-REGISTERED by a downgraded backend as an id-less entry during + // the awaits (raw-invisible from then on) — the raw comparison alone + // would drop the revived workspace and republish the very tombstone + // the authoritative evidence just cleared, suppressing its activity + // again. In that case the authoritative enumeration (which resolves + // id-less identities, and is attempted whenever id-less entries + // exist — see the third trigger above) must DENY the id before the + // transition counts as removal; an affirmation or a failed + // enumeration retains the entry (keeping a stale entry briefly is + // recoverable, wrongly suppressing a live workspace is not). The + // enumeration consulted here postdates the raw refresh whenever that + // refresh reports id-less entries (see the re-enumeration above), so + // a stale pre-refresh denial can never veto a revived id; a revival + // landing after that last enumeration is the contract's out-of-scope + // window — the next list's initial baseline no longer contains the + // id, so the raw arm cannot re-fire and its fresh evidence clears + // the republished tombstone. + const isVerifiablyRemovedFromRawConfig = (workspaceId: string): boolean => + isRemovedFromConfig(workspaceId) && + (!freshConfigHasRawInvisibleEntries || + (authoritativeIds != null && !authoritativeIds.has(workspaceId))); + // Tombstones are process-local removal knowledge; the shared config is + // the authority. A downgraded concurrent backend can legitimately + // re-register a deterministic legacy id this process pruned earlier — + // observing the id in a FRESH config-derived view (raw superset or the + // strict authoritative enumeration above; never snapshot/cache keys, + // which do not prove registration) makes the tombstone stale, so its + // write suppression and list filtering must end. Cleared before the + // revalidation filter below so a re-registered id's entry survives. + if (freshConfigIds != null || authoritativeIds != null) { + const registeredIds = new Set(); + for (const workspaceId of freshConfigIds ?? []) { + registeredIds.add(workspaceId); + } + for (const workspaceId of authoritativeIds ?? []) { + registeredIds.add(workspaceId); + } + this.extensionMetadata.clearTombstonesForRegisteredIds( + registeredIds, + clearableTombstoneIds + ); + } + const activityById = Object.fromEntries( + entries.filter((entry): entry is readonly [string, WorkspaceActivitySnapshot] => { + if (entry == null) { + return false; + } + const workspaceId = entry[0]; + // Revalidate after the per-workspace awaits above: a workspace + // removed while this list was computing would otherwise ride the + // delayed response past emitWorkspaceActivity's tombstone + // suppression — a renderer that already processed the removal + // event would re-insert the deleted id until the next reconnect. + if (this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { + return false; + } + const foreignRemoved = + // Persisted snapshot vanished from the shared file mid-list. + // Metadata keys normally only disappear through removal, but + // NOT always: a deterministically corrupt file self-heals into + // a valid (possibly EMPTY) one on the strict re-read, so a + // mid-list quarantine makes every earlier key vanish while the + // workspaces stay registered. Disappearance therefore counts as + // removal evidence only when a fresh config-derived view + // CAPABLE of seeing the id corroborates the deregistration: + // raw-visible ids need the fresh raw view to deny them, + // raw-invisible (legacy) ids the authoritative enumeration. + // With neither view available the entry is retained — keeping a + // stale entry briefly is recoverable, wiping live renderer + // state on a corruption reset is not. Entries that never had a + // persisted snapshot are covered by the config check. + (freshPersistedIds != null && + snapshots.has(workspaceId) && + !freshPersistedIds.has(workspaceId) && + !(authoritativeIds?.has(workspaceId) ?? false) && + ((initialConfigIds?.has(workspaceId) ?? false) || + (freshConfigIds?.has(workspaceId) ?? false) + ? freshConfigIds != null && !freshConfigIds.has(workspaceId) + : authoritativeIds != null)) || + // Deregistered from the shared config mid-list — also covers + // workflow/bash-monitor-only entries with no persisted snapshot. + isVerifiablyRemovedFromRawConfig(workspaceId) || + // Raw-invisible (legacy stable) ids: authoritative-lookup + // counterpart of the raw-superset comparison above. + isRemovedPerAuthoritativeIdentity(workspaceId); + if (foreignRemoved) { + // A cross-process removal publishes no local tombstone, so the + // tombstone-cleared eviction listener never fires for it. Stale + // workflow-run/monitor caches would then survive the removal — + // and if a downgraded backend re-registers the same + // deterministic legacy id later, getActiveWorkflowRunIds would + // serve the REMOVED incarnation's cached runs as ghost activity + // instead of probing the recreated session. + this.evictWorkspaceActivityCaches(workspaceId); + // Eviction alone cannot stop a LATE local producer (workflow-run + // or bash-monitor completion) from repopulating the caches and + // re-emitting the removed incarnation's activity right after + // this authoritative response dropped it — publish a local + // tombstone so emits and writes stay suppressed until fresh + // config evidence proves a revival. + this.extensionMetadata.suppressForeignRemoval(workspaceId); + return false; + } + return true; + }) ); + // Addition-side counterpart of the fresh re-read: another backend can + // register a workspace AND persist its first activity after this + // process's initial snapshot read. The refreshed config admits the id + // into scope, but its per-id computation saw a null snapshot (and no + // local workflow/monitor caches, which are process-local), so the + // entry was omitted — and the activity subscription cannot heal that + // (it is backed by this process's EventEmitter, so cross-process + // writes produce no delta). Merge in-scope additions from the fresh + // re-read, subject to the same removal guards as retained entries. + // NOT gated on the snapshot re-read succeeding: config-proven late ids + // (workflow-only registrations) must be probed even when the metadata + // re-read transiently failed — the method still returns an + // authoritative (non-null) response in that case, and the + // process-local subscription can never supply the foreign workflow + // event, so skipping the probe would hide that activity until + // reconnect. Snapshot-derived candidates and snapshot guards simply + // degrade to the views that ARE available. + { + // Merge scope: the (possibly stale) per-id scope PLUS fresh-snapshot + // ids a fresh config-derived view proves registered — a workspace + // registered and written between the scope reads and the fresh + // re-read is in both fresh views but in neither stale one. Raw- + // invisible ids (legacy stable ids resolved from session + // metadata.json) can never appear in the raw view, so they are + // admitted through the authoritative enumeration instead — without + // that, a legacy workspace registered by a downgraded backend + // mid-list would stay absent until reconnect (the process-local + // subscription cannot deliver the cross-process snapshot). + const mergeCandidateIds = new Set(workspaceIds); + for (const workspaceId of freshSnapshots?.keys() ?? []) { + if ( + (freshConfigIds?.has(workspaceId) ?? false) || + (authoritativeIds?.has(workspaceId) ?? false) + ) { + mergeCandidateIds.add(workspaceId); + } + } + // Workflow-only late registrations: a workspace registered after the + // scope reads can have active workflow runs but no metadata snapshot + // at all, so admission cannot come from fresh-snapshot keys alone — + // every fresh config-derived id outside the stale per-id scope is a + // candidate. The authoritative enumeration contributes the ids the + // raw view can never carry (id-less legacy registrations by a + // downgraded backend). Ids with neither a snapshot nor live activity + // cost one workflow probe and are dropped by the emptiness check + // below. + for (const lateIdSource of [freshConfigIds, authoritativeIds]) { + for (const workspaceId of lateIdSource ?? []) { + if (!workspaceIds.has(workspaceId)) { + mergeCandidateIds.add(workspaceId); + } + } + } + // Workflow-run bootstrap for candidates the per-id loop never saw: + // their on-disk active runs are not in the process-local cache, and + // a cached-only merge would omit activeWorkflowRunCount for exactly + // the cross-process registrations this merge exists to bootstrap. + // Probed BEFORE the final guard views below so no await separates + // guard evaluation from insertion (ids the per-id loop already + // probed resolve synchronously from the shared cached Set). + const probedWorkflowRunIds = new Map>(); + for (const workspaceId of mergeCandidateIds) { + if ( + workspaceId in activityById || + // In-scope ids without a late snapshot were fully decided by the + // per-id loop (probe + zero-tombstone logic); re-probing them + // would only trigger the final re-reads below on every list. + (workspaceIds.has(workspaceId) && !(freshSnapshots?.has(workspaceId) ?? false)) + ) { + continue; + } + probedWorkflowRunIds.set(workspaceId, await this.getActiveWorkflowRunIds(workspaceId)); + } + const isRawInvisible = (workspaceId: string): boolean => + !(initialConfigIds?.has(workspaceId) ?? false) && + !(freshConfigIds?.has(workspaceId) ?? false); + // The final revalidation must ALSO run when a retained entry is + // raw-invisible even with zero late candidates: the mid-list + // authoritative enumeration can observe a legacy stable id right + // before another backend deregisters it and deletes its metadata + // later in the same await. Every view the retained filter used + // (pre-enumeration freshSnapshots, raw scans blind to stable ids, + // the stale enumeration itself) then still shows the workspace, and + // with no candidates to probe nothing else would re-read — the + // deleted workspace would ride every authoritative response + // indefinitely because cross-process removals emit no local event. + // Modern deployments (all ids raw-visible) never pay this path. + if (probedWorkflowRunIds.size > 0 || Object.keys(activityById).some(isRawInvisible)) { + // Final post-probe views: the probes awaited disk, so a removal + // landing during them is invisible to every view captured above — + // inserting on those alone would ride the deleted id back into the + // renderer. Re-read the (post-prune bounded) metadata file and the + // raw config superset once more, then evaluate every guard with no + // awaits before insertion. A removed workspace's metadata key is + // deleted on removal, so the snapshot re-read also covers legacy + // ids the raw view cannot see; same-process removals are covered + // by the tombstone check. Best-effort like the other fresh + // re-reads: an unreadable view falls back to the pre-probe one. + let finalSnapshots: ReadonlyMap | null = null; + try { + finalSnapshots = await this.extensionMetadata.getAllSnapshots({ throwOnError: true }); + } catch { + finalSnapshots = null; + } + // Post-probe authoritative recheck, only when a probed candidate is + // raw-invisible (in no raw view): the raw deregistration guard + // below is blind to a legacy workspace removed during the probes, + // and in the normal gap between config deregistration and metadata + // cleanup its snapshot still exists — the pre-probe authoritative + // set is the view that ADMITTED it, so only a fresh enumeration + // can prove the removal. Modern deployments never pay this walk. + let finalAuthoritativeIds: ReadonlySet | null = null; + let finalConfigIds: ReadonlySet | null = null; + // Completeness of the final raw view, mirroring the mid-list + // fresh read: id-less legacy entries make raw denial insufficient + // removal evidence (see isVerifiablyRemovedFromRawConfig). + let finalConfigHasRawInvisibleEntries = false; + try { + const finalEvidence = this.config.readPersistedWorkspaceIdEvidence(); + finalConfigIds = finalEvidence.ids; + finalConfigHasRawInvisibleEntries = finalEvidence.hasWorkspaceEntriesWithoutIds; + } catch { + finalConfigIds = null; + finalConfigHasRawInvisibleEntries = true; + } + if ( + Array.from(probedWorkflowRunIds.keys()).some(isRawInvisible) || + // Retained entries are re-filtered with the final views below + // (they were admitted before the probes awaited), and a + // raw-invisible retained id's removal is provable only through + // the same fresh enumeration. + Object.keys(activityById).some(isRawInvisible) || + // A transiently unreadable post-probe raw view would otherwise + // disable the raw deregistration guards below entirely: an + // inline-id workspace deregistered during the probes (its + // metadata key not yet cleaned up) would ride the response with + // no cross-process event to repair it. The enumeration + // substitutes as removal evidence for ids the scope enumeration + // vouched for. + finalConfigIds == null || + // Id-less legacy entries in the final raw view: a raw-visible + // id deregistered during the probes is indistinguishable from + // one removed-and-revived as an id-less entry by a downgraded + // backend, so the raw deregistration guards below need the + // enumeration to tell them apart (same rule as the mid-list + // isVerifiablyRemovedFromRawConfig). Modern deployments (every + // id inline) never pay this walk. + finalConfigHasRawInvisibleEntries + ) { + try { + finalAuthoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); + } catch (error) { + log.debug("Failed to re-enumerate authoritative ids after workflow probes", { + error, + }); + finalAuthoritativeIds = null; + } + // The enumeration is itself an await: a raw-invisible legacy + // workspace can be admitted by it and then removed (metadata + // key deleted) before it finishes — invisible to the snapshot + // view captured before it and to every raw view. Re-read the + // snapshot evidence after the enumeration so the vanish guard + // sees the removal; on failure keep the pre-enumeration view + // (still a post-probe view). + try { + finalSnapshots = await this.extensionMetadata.getAllSnapshots({ + throwOnError: true, + }); + } catch { + // Keep the pre-enumeration re-read (possibly null). + } + // Re-read the raw view after the awaits above so the raw + // deregistration guards see the freshest possible view; on a + // repeat failure keep the earlier successful read (still a + // valid post-probe view) rather than degrading to null. + try { + const refreshedFinalEvidence = this.config.readPersistedWorkspaceIdEvidence(); + finalConfigIds = refreshedFinalEvidence.ids; + finalConfigHasRawInvisibleEntries = + refreshedFinalEvidence.hasWorkspaceEntriesWithoutIds; + } catch { + // Keep the pre-enumeration read (possibly null). + } + if (finalConfigHasRawInvisibleEntries) { + // Same staleness rule as the mid-list re-enumeration: the + // refresh revealed id-less entries, so the enumeration's + // denials may predate an id-less re-registration — the + // enumeration-backed drops below must use the freshest + // capable view (a revival after this last read is the + // contract's out-of-scope window). The raw and snapshot + // views deliberately stay at their earlier reads: their + // staleness errs toward retention, never a wrong drop. On + // failure retain rather than trust the stale denial set. + try { + finalAuthoritativeIds = await this.enumerateAuthoritativeWorkspaceIds(); + } catch (error) { + log.debug("Failed to re-enumerate authoritative ids after final raw refresh", { + error, + }); + finalAuthoritativeIds = null; + } + } + } + // The retained-entry filter above ran BEFORE the workflow probes, + // so a removal landing during those awaits is invisible to every + // view it used — the removed workspace would ride the response + // back into the renderer with no cross-process event to correct + // it. Re-apply the removal guards to retained entries with the + // post-probe views (final filtering must follow the last await). + // Removal guards only: zero-count tombstone entries legitimately + // have no snapshot and no live counts, so the probed candidates' + // emptiness check must not run here. + for (const workspaceId of Object.keys(activityById)) { + const foreignRemoved = + // Persisted snapshot vanished during the probes — also covers + // legacy ids the raw views cannot see. Corroborated like the + // mid-list filter's vanish arm: a strict re-read self-heals + // deterministic corruption into a valid (possibly EMPTY) + // file, so disappearance alone is not removal evidence — a + // post-probe config-derived view capable of seeing the id + // must also deny it. + (finalSnapshots != null && + (snapshots.has(workspaceId) || (freshSnapshots?.has(workspaceId) ?? false)) && + !finalSnapshots.has(workspaceId) && + !(finalConfigIds?.has(workspaceId) ?? false) && + !(finalAuthoritativeIds?.has(workspaceId) ?? false) && + (isRawInvisible(workspaceId) + ? finalAuthoritativeIds != null + : finalConfigIds != null)) || + // Verifiably deregistered from the raw config during the + // probes: visible in an earlier raw view, gone from the + // post-probe one. With id-less entries in the final raw view + // the id may instead have been removed-and-revived id-less, + // so the final enumeration must deny it (same completeness + // rule as isVerifiablyRemovedFromRawConfig). + (finalConfigIds != null && + !finalConfigIds.has(workspaceId) && + ((initialConfigIds?.has(workspaceId) ?? false) || + (freshConfigIds?.has(workspaceId) ?? false)) && + (!finalConfigHasRawInvisibleEntries || + (finalAuthoritativeIds != null && !finalAuthoritativeIds.has(workspaceId)))) || + // Raw-invisible retained ids: post-probe authoritative + // counterpart of the raw guard above. + (isRawInvisible(workspaceId) && + !(finalConfigIds?.has(workspaceId) ?? false) && + finalAuthoritativeIds != null && + !finalAuthoritativeIds.has(workspaceId)) || + // Raw view unreadable post-probe: the fallback enumeration + // substitutes as removal evidence, but only for ids the scope + // enumeration itself vouched for — raw-only ids are + // enumeration-invisible by design and stay retained on + // transient read failures. + (finalConfigIds == null && + finalAuthoritativeIds != null && + (scopeEnumerationIds?.has(workspaceId) ?? false) && + !finalAuthoritativeIds.has(workspaceId)); + if (foreignRemoved) { + // Cross-process removals publish no local tombstone, so the + // tombstone-cleared eviction listener never fires — see the + // mid-list filter above (including the late-producer + // suppression rationale). + this.evictWorkspaceActivityCaches(workspaceId); + this.extensionMetadata.suppressForeignRemoval(workspaceId); + delete activityById[workspaceId]; + } else if (this.extensionMetadata.isWorkspaceDeleted(workspaceId)) { + delete activityById[workspaceId]; + } + } + for (const [workspaceId, activeWorkflowRunIds] of probedWorkflowRunIds) { + // Freshest available view, degrading to the INITIAL read: a raw- + // registered id outside the normalized scope (e.g. invalid + // project path) is admitted here, and when both re-reads failed + // transiently its already-loaded initial snapshot must still + // supply goal/status/recency — omitting it from an authoritative + // response would clear that renderer state with no repair event. + const lateSnapshot = + finalSnapshots != null + ? (finalSnapshots.get(workspaceId) ?? null) + : freshSnapshots != null + ? (freshSnapshots.get(workspaceId) ?? null) + : (snapshots.get(workspaceId) ?? null); + const lateForeignRemoved = + isVerifiablyRemovedFromRawConfig(workspaceId) || + isRemovedPerAuthoritativeIdentity(workspaceId) || + // Persisted snapshot vanished during the probes — also covers + // legacy ids the raw views cannot see. Same corruption-reset + // corroboration as the retained-entry vanish arms: a + // self-healed (possibly empty) re-read is not removal + // evidence on its own. + (finalSnapshots != null && + (freshSnapshots?.has(workspaceId) ?? false) && + !finalSnapshots.has(workspaceId) && + !(finalConfigIds?.has(workspaceId) ?? false) && + !(finalAuthoritativeIds?.has(workspaceId) ?? false) && + (isRawInvisible(workspaceId) + ? finalAuthoritativeIds != null + : finalConfigIds != null)) || + // Verifiably deregistered from the raw config during the + // probes: the id was visible in an EARLIER raw view — the + // initial baseline or the fresh re-read that admitted it (a + // late registration is absent from the initial baseline by + // definition) — and is gone from the post-probe view. During + // the normal gap between config deregistration and metadata + // cleanup the snapshot still exists, so the vanish check + // above cannot catch this. Same raw-view completeness rule as + // the retained-entry arm: with id-less entries present the + // final enumeration must deny a possibly-revived id. + (finalConfigIds != null && + !finalConfigIds.has(workspaceId) && + ((initialConfigIds?.has(workspaceId) ?? false) || + (freshConfigIds?.has(workspaceId) ?? false)) && + (!finalConfigHasRawInvisibleEntries || + (finalAuthoritativeIds != null && !finalAuthoritativeIds.has(workspaceId)))) || + // Raw-invisible candidates: post-probe authoritative + // counterpart of the raw guard above — a legacy workspace + // deregistered during the probes is invisible to every raw + // view, and its snapshot may outlive the deregistration. + (!(initialConfigIds?.has(workspaceId) ?? false) && + !(freshConfigIds?.has(workspaceId) ?? false) && + !(finalConfigIds?.has(workspaceId) ?? false) && + finalAuthoritativeIds != null && + !finalAuthoritativeIds.has(workspaceId)) || + // Raw view unreadable post-probe: same enumeration fallback + // as the retained-entry filter, scoped to ids the scope + // enumeration vouched for (raw-only ids stay admitted on + // transient read failures — see lateSnapshot above). + (finalConfigIds == null && + finalAuthoritativeIds != null && + (scopeEnumerationIds?.has(workspaceId) ?? false) && + !finalAuthoritativeIds.has(workspaceId)); + if (lateForeignRemoved) { + // Same cross-process eviction + late-producer suppression + // rationale as the mid-list filter: no local tombstone means + // no listener-driven eviction, and the probes above may have + // installed caches for the removed incarnation. + this.evictWorkspaceActivityCaches(workspaceId); + this.extensionMetadata.suppressForeignRemoval(workspaceId); + continue; + } + if ( + // Nothing to contribute: no persisted snapshot and no live + // counts (a workflow-only candidate legitimately has no + // snapshot, so its absence alone is not removal evidence) — + // never removal proof, so no cache eviction. + (lateSnapshot == null && + activeWorkflowRunIds.size === 0 && + this.getActiveBashMonitorCount(workspaceId) === 0) || + this.extensionMetadata.isWorkspaceDeleted(workspaceId) + ) { + continue; + } + // Same overlay path emitWorkspaceActivity uses. + const merged = this.mergeCurrentActiveBashMonitorCount( + workspaceId, + mergeActiveWorkflowRuns( + this.overlayPendingGoal(workspaceId, lateSnapshot), + activeWorkflowRunIds + ) + ); + if (merged != null) { + activityById[workspaceId] = merged; + } + } + } + } + return activityById; } catch (error) { log.error("Failed to list activity:", error); // null (not {}) so the renderer can tell a read failure from a legitimately - // idle deployment — {} is a valid successful result when nothing is active. + // idle deployment — with scoping, {} is a valid authoritative answer (no + // known workspace has activity) that the renderer must apply to clear + // stale entries after a disconnected removal. On null the renderer keeps + // last-known state and retries in the background. return null; } } diff --git a/src/node/utils/extensionMetadata.ts b/src/node/utils/extensionMetadata.ts index 50f77655c3..0249c8faeb 100644 --- a/src/node/utils/extensionMetadata.ts +++ b/src/node/utils/extensionMetadata.ts @@ -35,6 +35,15 @@ export interface ExtensionMetadata { // a restart so unchanged chats are not regenerated. Never exposed on // WorkspaceActivitySnapshot (IPC shape). sidebarStatusInputHash?: string | null; + // Backend-only monotonic write counter, advanced by every persisted + // mutation of this entry. Recovery merges cannot order metadata copies by + // `recency` — that is a USER-INTERACTION timestamp which status/goal/ + // streaming writers deliberately preserve — so cross-copy ordering uses + // this generation instead (see recoverStrandedRecreatedLeftover). Never + // exposed on WorkspaceActivitySnapshot (IPC shape). Builds without this + // field drop it on their writes; ordering then degrades to the recency + // tiebreak, never resurrecting against a generation-less newer write. + writeGeneration?: number; goal?: GoalSnapshot | null; } @@ -129,6 +138,9 @@ export function coerceExtensionMetadata(value: unknown): ExtensionMetadata | nul ...(typeof record.sidebarStatusInputHash === "string" ? { sidebarStatusInputHash: record.sidebarStatusInputHash } : {}), + ...(typeof record.writeGeneration === "number" && Number.isFinite(record.writeGeneration) + ? { writeGeneration: record.writeGeneration } + : {}), ...(goal !== undefined ? { goal } : {}), }; } diff --git a/tests/ipc/acp.sessionMethods.test.ts b/tests/ipc/acp.sessionMethods.test.ts index 02395c5e84..6c607da846 100644 --- a/tests/ipc/acp.sessionMethods.test.ts +++ b/tests/ipc/acp.sessionMethods.test.ts @@ -32,6 +32,12 @@ interface HarnessOptions { activeWorkspaces?: WorkspaceInfo[]; archivedWorkspaces?: WorkspaceInfo[]; workspaceActivity?: WorkspaceActivityById; + /** + * Resolve activity.list with null — the backend's read-failure signal for + * an unreadable extensionMetadata.json (getActivityList never rejects; it + * logs and returns null so callers can tell failure from an idle {}). + */ + activityListUnavailable?: boolean; onChatEvents?: WorkspaceChatMessage[]; onChatStream?: AsyncIterable; requireTrustedProjectForCreate?: boolean; @@ -133,7 +139,12 @@ function createMockServer(options?: HarnessOptions): MockServer { return input?.archived ? archivedWorkspaces : activeWorkspaces; }, activity: { - list: async () => workspaceActivity, + list: async () => { + if (options?.activityListUnavailable) { + return null; + } + return workspaceActivity; + }, }, getInfo: async ({ workspaceId }: { workspaceId: string }) => allWorkspacesById.get(workspaceId) ?? null, @@ -393,6 +404,30 @@ describe("ACP session list/resume/fork support", () => { expect(harness.listCalls.slice(0, 2)).toEqual([{ archived: false }, { archived: true }]); }); + it("lists sessions when the activity list is unavailable", async () => { + // activity.list resolves null when extensionMetadata.json is unreadable + // so the renderer can keep cached state; ACP has none, so session + // listing must degrade to no-activity sorting instead of failing + // wholesale. + const workspace = createWorkspaceInfo({ + id: "ws-no-activity", + projectPath: "/repo/a", + namedWorkspacePath: "/repo/a/.mux/ws-no-activity", + createdAt: "2026-02-18T10:00:00.000Z", + }); + const harness = createHarness({ + activeWorkspaces: [workspace], + activityListUnavailable: true, + }); + + await harness.agent.initialize({ protocolVersion: PROTOCOL_VERSION }); + + const response = await harness.agent.listSessions({ cwd: "/repo/a/" }); + expect(response.sessions.map((session) => session.sessionId)).toEqual(["ws-no-activity"]); + // Falls back to createdAt when no activity recency is available. + expect(response.sessions[0]?.updatedAt).toBe("2026-02-18T10:00:00.000Z"); + }); + it("lists and resumes sessions from a sub-project cwd", async () => { const workspace = createWorkspaceInfo({ id: "ws-sub-project",