From 9665489578ce8048a1f79d0d1d062662fb6d32ae Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:18:19 +0000 Subject: [PATCH 01/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20deliver=20terminal?= =?UTF-8?q?=20wakes=20for=20kernel-launched=20background=20workflow=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 14 +++ .../services/agentWorkflowRunReferences.ts | 5 +- src/node/services/taskService.ts | 5 + src/node/services/workspaceService.test.ts | 98 +++++++++++++++++++ src/node/services/workspaceService.ts | 47 ++++++--- 5 files changed, 153 insertions(+), 16 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 88a64361395..44568222c86 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -31,4 +31,18 @@ describe("agent workflow run references", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); + + test("keeps the newest createdAtMs across re-records", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_re_recorded"; + await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs: 2_000 }); + await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs: 1_000 }); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toEqual([{ runId, createdAtMs: 2_000 }]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index b202f4a83db..909b69a9264 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -75,7 +75,10 @@ export async function recordAgentWorkflowRunReference(input: { const previous = byRunId.get(input.runId); byRunId.set(input.runId, { runId: input.runId, - createdAtMs: previous ? Math.min(previous.createdAtMs, createdAtMs) : createdAtMs, + // Latest record wins: workflow_resume re-records the reference, and a resume issued after + // a manual user message must re-establish provenance for supersession-timestamp + // comparisons (isWorkflowInvocationCurrent, listAgentReferencedWorkflowRunIds). + createdAtMs: previous ? Math.max(previous.createdAtMs, createdAtMs) : createdAtMs, }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 7e43d21fde6..15bb3c686f7 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8232,6 +8232,11 @@ export class TaskService { notification.sourceId ); if (workflowPrompt == null) { + // Dropping a notify_on_terminal wake strands the run's owner; keep the drop diagnosable. + log.warn("Dropping superseded workflow terminal attention", { + ownerWorkspaceId, + runId: notification.sourceId, + }); await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); continue; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ec69f044d2d..9bd9871048a 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -54,9 +54,11 @@ import type { SendMessageOptions, WorkspaceChatMessage } from "@/common/orpc/typ import { createMuxMessage } from "@/common/types/message"; import { buildStagedAttachmentNotice } from "@/browser/features/ChatInput/stagedAttachments"; import { + WORKFLOW_RESULT_METADATA_TYPE, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, } from "@/common/utils/workflowRunMessages"; +import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; import * as todoStorageModule from "@/node/services/todos/todoStorage"; import * as runtimeFactory from "@/node/runtime/runtimeFactory"; @@ -5924,6 +5926,102 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("counts a kernel-launched run recorded in the sidecar as the current invocation", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-kernel"; + const runId = "wfr_currentness_kernel"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-kernel", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // mux.workflow_run inside code_execution leaves no workflow_run tool part in history; the + // agent-workflow-runs sidecar reference is the only durable invocation evidence. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-kernel-launch", "assistant", "", { timestamp: 1_100 }, [ + { + type: "dynamic-tool", + toolCallId: "code-exec-1", + toolName: "code_execution", + state: "output-available", + input: { code: "return xum.workflow_run({ script_path: './workflows/demo.js' })" }, + output: { success: true, result: { status: "running", runId } }, + }, + ]) + ); + + // The nested runId in the code_execution output alone is not invocation evidence. + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A newer manual user message supersedes the sidecar reference. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A kernel workflow_resume re-records the reference after the supersession and + // re-establishes provenance (latest record wins). + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_250, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // Once the terminal result was delivered, the sidecar must not resurrect the invocation. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("workflow-result", "user", "The workflow below has finished.", { + timestamp: 1_300, + synthetic: true, + muxMetadata: { + type: WORKFLOW_RESULT_METADATA_TYPE, + rawCommand: "workflow_run ./workflows/demo.js", + runId, + }, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6393454098e..48f0dbc3ada 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3,6 +3,7 @@ import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { EventEmitter } from "events"; import * as path from "path"; import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; +import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; @@ -10935,21 +10936,17 @@ export class WorkspaceService extends EventEmitter { assert(workspaceId.length > 0, "isWorkflowInvocationCurrent requires workspaceId"); assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); - let current = false; - let foundDecision = false; + let outcome: "invocation" | "consumed" | "superseded" | null = null; + let supersededAtMs: number | null = null; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", (messages) => { for (const message of messages) { - if (isManualUserSupersessionMessage(message)) { - current = false; - foundDecision = true; - return false; - } - if (isResetBoundaryMessage(message)) { - current = false; - foundDecision = true; + if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { + outcome = "superseded"; + const timestamp = message.metadata?.timestamp; + supersededAtMs = typeof timestamp === "number" ? timestamp : null; return false; } if ( @@ -10957,13 +10954,11 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - current = false; - foundDecision = true; + outcome = "consumed"; return false; } if (isWorkflowInvocationMessage(message, runId)) { - current = true; - foundDecision = true; + outcome = "invocation"; return false; } } @@ -10979,7 +10974,29 @@ export class WorkspaceService extends EventEmitter { return false; } - return foundDecision && current; + if (outcome === "invocation") { + return true; + } + if (outcome === "consumed") { + return false; + } + + // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave + // no recognizable invocation part in history, so the backward walk above stops at the prior + // real user message and would wrongly treat the run as superseded, silently dropping its + // notify_on_terminal wake. Their durable provenance is the agent-workflow-runs sidecar: a + // reference recorded after the latest supersession boundary counts as the current + // invocation. A boundary without a durable timestamp fails safe to superseded, mirroring + // TaskService.listAgentReferencedWorkflowRunIds. + if (outcome === "superseded" && supersededAtMs === null) { + return false; + } + const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + const reference = references.find((candidate) => candidate.runId === runId); + if (reference == null) { + return false; + } + return supersededAtMs === null || reference.createdAtMs > supersededAtMs; } /** From f41cbfbb1113b47d72b83bc3346c7d72af797ab9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:39:44 +0000 Subject: [PATCH 02/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20persist=20workflow?= =?UTF-8?q?=5Fresume=20terminal=20consumption=20for=20kernel-nested=20call?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/tools/workflow_resume.test.ts | 100 ++++++++++++++++++ src/node/services/tools/workflow_resume.ts | 24 ++++- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index 1ae69b94021..63df2868bc6 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -6,6 +6,7 @@ import { TestTempDir, createTestToolConfig } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; import { WORKFLOW_CHECKPOINT_RETRY_ERROR_MESSAGE } from "@/common/utils/workflowRetryEligibility"; import type { WorkflowRunRecord } from "@/common/types/workflow"; +import type { TaskService } from "@/node/services/taskService"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; const mockToolCallOptions: ToolExecutionOptions = { @@ -229,6 +230,105 @@ describe("workflow_resume tool", () => { }); }); + test("marks terminal attention consumed when returning an already-completed run's result", async () => { + using tempDir = new TestTempDir("test-workflow-resume-consumed"); + const completedRun = buildRun({ + status: "completed", + events: [ + { sequence: 1, type: "status", at: "2026-05-29T00:00:00.000Z", status: "running" }, + { + sequence: 2, + type: "result", + at: "2026-05-29T00:00:01.000Z", + result: { reportMarkdown: "already done" }, + }, + { sequence: 3, type: "status", at: "2026-05-29T00:00:01.000Z", status: "completed" }, + ], + }); + const workflowService = buildWorkflowService({ getRun: mock(async () => completedRun) }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: null }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + }); + + test("does not mark terminal attention consumed for background dispatches", async () => { + using tempDir = new TestTempDir("test-workflow-resume-background-no-consume"); + // The refresh after a background dispatch can still observe the stale pre-dispatch failed + // status; consuming it would tombstone the retried run's future terminal wake. + const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: true, mode: "retry_from_checkpoint" }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).not.toHaveBeenCalled(); + }); + + test("marks terminal attention consumed when a foreground retry finishes terminal", async () => { + using tempDir = new TestTempDir("test-workflow-resume-foreground-consume"); + const failedRun = buildFailedRun(); + const completedRun = buildRun({ + status: "completed", + events: [ + { sequence: 1, type: "status", at: "2026-05-29T00:00:00.000Z", status: "running" }, + { sequence: 2, type: "status", at: "2026-05-29T00:00:02.000Z", status: "completed" }, + ], + }); + let getRunCalls = 0; + const workflowService = buildWorkflowService({ + getRun: mock(async () => { + getRunCalls += 1; + return getRunCalls === 1 ? failedRun : completedRun; + }), + retryRunFromCheckpoint: mock(async () => ({ + runId: "wfr_resume_me", + status: "completed" as const, + result: { reportMarkdown: "retried" }, + })), + }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: "retry_from_checkpoint" }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + }); + test("rejects default resume of a failed run with checkpoint retry guidance", async () => { using tempDir = new TestTempDir("test-workflow-resume-failed"); const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index a94188569a6..649c1ca90bf 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -2,7 +2,7 @@ import { tool } from "ai"; import { getErrorMessage } from "@/common/utils/errors"; import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; -import type { WorkflowRunRecord } from "@/common/types/workflow"; +import { isTerminalWorkflowRunStatus, type WorkflowRunRecord } from "@/common/types/workflow"; import { getWorkflowCheckpointRetryEligibility } from "@/common/utils/workflowRetryEligibility"; import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import { @@ -154,9 +154,25 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const mode: WorkflowResumeMode = args.mode ?? "resume"; const invocationStartedAtMs = Date.now(); + // A kernel-nested resume (mux.workflow_resume inside code_execution) leaves no top-level + // workflow_resume part in history, so the history-walk consumption predicates cannot see + // that this turn already received the terminal result. Persist consumption durably so the + // terminal-attention drain never re-delivers it. + const markTerminalAttentionConsumed = async (terminalRun: WorkflowRunRecord) => { + if (!isTerminalWorkflowRunStatus(terminalRun.status)) { + return; + } + await config.taskService?.markWorkflowRunTerminalAttentionConsumed?.({ + ownerWorkspaceId: workspaceId, + runId: terminalRun.id, + status: terminalRun.status, + }); + }; + // Idempotent success: the work is already done, so hand back the durable result instead // of failing the agent's recovery loop (e.g. resuming after a crash that actually finished). if (run.status === "completed" && mode === "resume") { + await markTerminalAttentionConsumed(run); return parseToolResult( WorkflowResumeToolResultSchema, { @@ -231,6 +247,12 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const refreshedRunIsStale = isBackgroundDispatch && refreshedRun != null && refreshedRun.status === run.status; + // Foreground only: a background dispatch can still observe the stale pre-dispatch terminal + // status, and consuming it would tombstone the retried run's future terminal wake. + if (!isBackgroundDispatch && refreshedRun != null) { + await markTerminalAttentionConsumed(refreshedRun); + } + return parseToolResult( WorkflowResumeToolResultSchema, { From a19c5a304c63c58a04f55c592c564c3fce1f36e9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:46:20 +0000 Subject: [PATCH 03/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20let=20newer=20sidec?= =?UTF-8?q?ar=20records=20outrank=20consumed=20results;=20clamp=20future?= =?UTF-8?q?=20timestamps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 18 ++++++++++++++ .../services/agentWorkflowRunReferences.ts | 9 +++++-- src/node/services/workspaceService.test.ts | 9 +++++++ src/node/services/workspaceService.ts | 24 +++++++++---------- 4 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 44568222c86..fde568bd491 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -32,6 +32,24 @@ describe("agent workflow run references", () => { } }); + test("clamps future-dated createdAtMs to the current time", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_future"; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs: Date.now() + 86_400_000, + }); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(1); + expect(references[0]?.createdAtMs).toBeLessThanOrEqual(Date.now()); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("keeps the newest createdAtMs across re-records", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 909b69a9264..eab9be58d8b 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -30,6 +30,7 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { } const parsed: AgentWorkflowRunReference[] = []; + const now = Date.now(); for (const reference of references) { if (reference == null || typeof reference !== "object") { continue; @@ -41,7 +42,10 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (typeof record.createdAtMs !== "number" || !Number.isFinite(record.createdAtMs)) { continue; } - parsed.push({ runId: record.runId, createdAtMs: record.createdAtMs }); + // Self-heal implausible future timestamps (clock correction, corruption): a future-dated + // reference would otherwise outrank every later user/reset boundary in supersession + // comparisons until wall time catches up. + parsed.push({ runId: record.runId, createdAtMs: Math.min(record.createdAtMs, now) }); } return parsed; } @@ -71,7 +75,8 @@ export async function recordAgentWorkflowRunReference(input: { await referenceFileLocks.withLock(filePath, async () => { const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); const byRunId = new Map(existing.map((reference) => [reference.runId, reference])); - const createdAtMs = input.createdAtMs ?? Date.now(); + // Clamp like parseReferences: never persist a future-dated timestamp. + const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); byRunId.set(input.runId, { runId: input.runId, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 9bd9871048a..08ecf7a1c05 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6016,6 +6016,15 @@ describe("WorkspaceService workflow invocation events", () => { }) ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A kernel background resume issued after the delivered result re-records the reference, + // so the retried run's next terminal wake must count as current again. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_350, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); workspaceService.disposeSession(workspaceId); } finally { await cleanup(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 48f0dbc3ada..b04e43a550f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10937,7 +10937,7 @@ export class WorkspaceService extends EventEmitter { assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); let outcome: "invocation" | "consumed" | "superseded" | null = null; - let supersededAtMs: number | null = null; + let boundaryAtMs: number | null = null; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", @@ -10946,7 +10946,7 @@ export class WorkspaceService extends EventEmitter { if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { outcome = "superseded"; const timestamp = message.metadata?.timestamp; - supersededAtMs = typeof timestamp === "number" ? timestamp : null; + boundaryAtMs = typeof timestamp === "number" ? timestamp : null; return false; } if ( @@ -10955,6 +10955,8 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowToolResultMessage(message, runId) ) { outcome = "consumed"; + const timestamp = message.metadata?.timestamp; + boundaryAtMs = typeof timestamp === "number" ? timestamp : null; return false; } if (isWorkflowInvocationMessage(message, runId)) { @@ -10977,18 +10979,16 @@ export class WorkspaceService extends EventEmitter { if (outcome === "invocation") { return true; } - if (outcome === "consumed") { - return false; - } // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave // no recognizable invocation part in history, so the backward walk above stops at the prior - // real user message and would wrongly treat the run as superseded, silently dropping its - // notify_on_terminal wake. Their durable provenance is the agent-workflow-runs sidecar: a - // reference recorded after the latest supersession boundary counts as the current - // invocation. A boundary without a durable timestamp fails safe to superseded, mirroring - // TaskService.listAgentReferencedWorkflowRunIds. - if (outcome === "superseded" && supersededAtMs === null) { + // real user message (or, after a delivered result, at that consumed terminal message) and + // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the + // agent-workflow-runs sidecar: a reference recorded after that boundary counts as the + // current invocation. For a consumed boundary that means a background resume/retry issued + // after the prior result was delivered. A boundary without a durable timestamp fails safe + // to not-current, mirroring TaskService.listAgentReferencedWorkflowRunIds. + if (outcome !== null && boundaryAtMs === null) { return false; } const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); @@ -10996,7 +10996,7 @@ export class WorkspaceService extends EventEmitter { if (reference == null) { return false; } - return supersededAtMs === null || reference.createdAtMs > supersededAtMs; + return boundaryAtMs === null || reference.createdAtMs > boundaryAtMs; } /** From 30a3437d0b9bd6e890b44c73af0b24ead7a24f8b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:59:12 +0000 Subject: [PATCH 04/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20reject=20persisted?= =?UTF-8?q?=20future-dated=20sidecar=20references=20instead=20of=20clampin?= =?UTF-8?q?g=20per=20read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 37 +++++++++++++++++++ .../services/agentWorkflowRunReferences.ts | 12 ++++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index fde568bd491..b89d1185215 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -32,6 +32,43 @@ describe("agent workflow run references", () => { } }); + test("drops persisted future-dated references and repairs them on the next record", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const futureMs = Date.now() + 86_400_000; + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_corrupt_future", createdAtMs: futureMs }, + { runId: "wfr_sane", createdAtMs: 1_000 }, + ], + }) + ); + + // A per-read clamp would re-evaluate to "now" on every read and outrank every later + // user/reset boundary; the corrupted entry must be dropped instead. + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([ + { runId: "wfr_sane", createdAtMs: 1_000 }, + ]); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_corrupt_future", + createdAtMs: 2_000, + }); + const repaired = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(repaired).toContainEqual({ runId: "wfr_corrupt_future", createdAtMs: 2_000 }); + const raw = await fs.readFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + "utf-8" + ); + expect(raw).not.toContain(String(futureMs)); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("clamps future-dated createdAtMs to the current time", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index eab9be58d8b..90992b5a900 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -42,10 +42,14 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (typeof record.createdAtMs !== "number" || !Number.isFinite(record.createdAtMs)) { continue; } - // Self-heal implausible future timestamps (clock correction, corruption): a future-dated - // reference would otherwise outrank every later user/reset boundary in supersession - // comparisons until wall time catches up. - parsed.push({ runId: record.runId, createdAtMs: Math.min(record.createdAtMs, now) }); + // Reject future-dated references (clock correction, corruption) instead of clamping at + // read time: a per-read clamp re-evaluates to "now" on every read, so the entry would + // outrank every later user/reset boundary until wall time catches up. Rejected entries are + // replaced with a sane timestamp by the next legitimate record. + if (record.createdAtMs > now) { + continue; + } + parsed.push({ runId: record.runId, createdAtMs: record.createdAtMs }); } return parsed; } From 89263dd6b0c85445c1ab4a9aa57af4b9860302ae Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:13:03 +0000 Subject: [PATCH 05/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20fail=20safe=20after?= =?UTF-8?q?=20full=20history=20clear;=20dedupe=20sidecar=20references=20at?= =?UTF-8?q?=20parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 26 +++++++++++ .../services/agentWorkflowRunReferences.ts | 12 +++-- src/node/services/workspaceService.test.ts | 44 +++++++++++++++++++ src/node/services/workspaceService.ts | 12 +++-- 4 files changed, 87 insertions(+), 7 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index b89d1185215..dde03f870ef 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -32,6 +32,32 @@ describe("agent workflow run references", () => { } }); + test("collapses persisted duplicate entries to the newest sane timestamp", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Corrupted files can carry duplicates in either order; order-sensitive consumers must + // never observe a stale duplicate ahead of a legitimate re-record. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_dup", createdAtMs: 1_000 }, + { runId: "wfr_dup", createdAtMs: 2_000 }, + { runId: "wfr_dup_reversed", createdAtMs: 2_000 }, + { runId: "wfr_dup_reversed", createdAtMs: 1_000 }, + ], + }) + ); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(2); + expect(references).toContainEqual({ runId: "wfr_dup", createdAtMs: 2_000 }); + expect(references).toContainEqual({ runId: "wfr_dup_reversed", createdAtMs: 2_000 }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("drops persisted future-dated references and repairs them on the next record", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 90992b5a900..5aec88b834a 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -29,7 +29,7 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { return []; } - const parsed: AgentWorkflowRunReference[] = []; + const parsedByRunId = new Map(); const now = Date.now(); for (const reference of references) { if (reference == null || typeof reference !== "object") { @@ -49,9 +49,15 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (record.createdAtMs > now) { continue; } - parsed.push({ runId: record.runId, createdAtMs: record.createdAtMs }); + // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive + // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run + // superseded. + const existing = parsedByRunId.get(record.runId); + if (existing == null || record.createdAtMs > existing.createdAtMs) { + parsedByRunId.set(record.runId, { runId: record.runId, createdAtMs: record.createdAtMs }); + } } - return parsed; + return Array.from(parsedByRunId.values()); } export async function readAgentWorkflowRunReferences( diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 08ecf7a1c05..407ad8242c3 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6031,6 +6031,50 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("does not treat sidecar references as current after a full history clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-cleared"; + const runId = "wfr_currentness_cleared"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-cleared", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A full clear (truncateHistory) removes every row without appending a reset boundary + // and leaves the sidecar intact; the surviving reference must not inject a workflow + // result into the freshly cleared conversation. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + }); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index b04e43a550f..41c99a20d80 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10986,9 +10986,13 @@ export class WorkspaceService extends EventEmitter { // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the // agent-workflow-runs sidecar: a reference recorded after that boundary counts as the // current invocation. For a consumed boundary that means a background resume/retry issued - // after the prior result was delivered. A boundary without a durable timestamp fails safe - // to not-current, mirroring TaskService.listAgentReferencedWorkflowRunIds. - if (outcome !== null && boundaryAtMs === null) { + // after the prior result was delivered. The fallback requires a datable boundary: an + // undatable boundary fails safe to not-current (mirroring + // TaskService.listAgentReferencedWorkflowRunIds), and so does a decision-free history, + // because a full clear (truncateHistory) removes every row WITHOUT appending a reset + // boundary while leaving the sidecar intact — a surviving reference must not inject a + // workflow result into the freshly cleared conversation. + if (boundaryAtMs === null) { return false; } const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); @@ -10996,7 +11000,7 @@ export class WorkspaceService extends EventEmitter { if (reference == null) { return false; } - return boundaryAtMs === null || reference.createdAtMs > boundaryAtMs; + return reference.createdAtMs > boundaryAtMs; } /** From 59ce349492310c223b00f63de909829d52573f0f Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:28:39 +0000 Subject: [PATCH 06/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20tolerate=20bounded?= =?UTF-8?q?=20backward-clock=20skew=20in=20sidecar=20reference=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 19 +++++++++++++++++++ .../services/agentWorkflowRunReferences.ts | 16 +++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index dde03f870ef..2747eb1fe65 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -95,6 +95,25 @@ describe("agent workflow run references", () => { } }); + test("keeps references within the backward-clock skew tolerance", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A backward clock correction makes a legitimately recorded reference look slightly + // future-dated; dropping it would strand the run's terminal wake. + const slightlyFutureMs = Date.now() + 5 * 60_000; + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ references: [{ runId: "wfr_clock_skew", createdAtMs: slightlyFutureMs }] }) + ); + + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([ + { runId: "wfr_clock_skew", createdAtMs: slightlyFutureMs }, + ]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("clamps future-dated createdAtMs to the current time", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 5aec88b834a..11d7bb6c870 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -13,6 +13,11 @@ export interface AgentWorkflowRunReference { const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; +// Backward clock corrections (e.g. an NTP step after booting with a fast clock) can make a +// legitimately recorded reference look future-dated. Tolerate that bounded skew so the run's +// terminal wake is not dropped; only implausibly future values are treated as corruption. +const MAX_FUTURE_SKEW_MS = 60 * 60_000; + const referenceFileLocks = new MutexMap(); function referencesPath(workspaceSessionDir: string): string { @@ -42,11 +47,12 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (typeof record.createdAtMs !== "number" || !Number.isFinite(record.createdAtMs)) { continue; } - // Reject future-dated references (clock correction, corruption) instead of clamping at - // read time: a per-read clamp re-evaluates to "now" on every read, so the entry would - // outrank every later user/reset boundary until wall time catches up. Rejected entries are - // replaced with a sane timestamp by the next legitimate record. - if (record.createdAtMs > now) { + // Reject implausibly future-dated references (corruption) instead of clamping at read + // time: a per-read clamp re-evaluates to "now" on every read, so the entry would outrank + // every later user/reset boundary until wall time catches up. Values within + // MAX_FUTURE_SKEW_MS are kept as-is (backward clock correction, not corruption). Rejected + // entries are replaced with a sane timestamp by the next legitimate record. + if (record.createdAtMs > now + MAX_FUTURE_SKEW_MS) { continue; } // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive From 76e6e8070d8c0211afa522120a14987118b534df Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:07:54 +0000 Subject: [PATCH 07/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20decide=20kernel=20w?= =?UTF-8?q?orkflow=20currentness=20by=20boundary-row=20identity,=20not=20w?= =?UTF-8?q?all=20clock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/agentWorkflowRunReferences.ts | 31 ++++- src/node/services/taskService.ts | 11 ++ src/node/services/tools/toolUtils.ts | 17 ++- src/node/services/tools/workflow_run.test.ts | 15 ++- src/node/services/workspaceService.test.ts | 108 ++++++++++++++++++ src/node/services/workspaceService.ts | 107 +++++++++++------ 6 files changed, 249 insertions(+), 40 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 11d7bb6c870..58f23023f9e 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -9,6 +9,14 @@ import { MutexMap } from "@/node/utils/concurrency/mutexMap"; export interface AgentWorkflowRunReference { runId: string; createdAtMs: number; + /** + * Message ID of the newest invocation-decision row (manual user/reset supersession, consumed + * terminal result for this run, or direct invocation part) at record time; null when history + * had none. Row identity, not wall clock: currentness compares this against the row the + * history walk stops at, so a backward clock correction cannot reorder the comparison. + * Absent on legacy entries, which fail safe to not-current. + */ + afterBoundaryMessageId?: string | null; } const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; @@ -55,12 +63,23 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (record.createdAtMs > now + MAX_FUTURE_SKEW_MS) { continue; } + const boundaryRaw = record.afterBoundaryMessageId; + const afterBoundaryMessageId = + typeof boundaryRaw === "string" && boundaryRaw.length > 0 + ? boundaryRaw + : boundaryRaw === null + ? null + : undefined; // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run - // superseded. + // superseded. The chosen record is kept wholesale, including its boundary snapshot. const existing = parsedByRunId.get(record.runId); if (existing == null || record.createdAtMs > existing.createdAtMs) { - parsedByRunId.set(record.runId, { runId: record.runId, createdAtMs: record.createdAtMs }); + parsedByRunId.set(record.runId, { + runId: record.runId, + createdAtMs: record.createdAtMs, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + }); } } return Array.from(parsedByRunId.values()); @@ -84,6 +103,7 @@ export async function recordAgentWorkflowRunReference(input: { workspaceSessionDir: string; runId: string; createdAtMs?: number; + afterBoundaryMessageId?: string | null; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -98,8 +118,13 @@ export async function recordAgentWorkflowRunReference(input: { runId: input.runId, // Latest record wins: workflow_resume re-records the reference, and a resume issued after // a manual user message must re-establish provenance for supersession-timestamp - // comparisons (isWorkflowInvocationCurrent, listAgentReferencedWorkflowRunIds). + // comparisons (listAgentReferencedWorkflowRunIds). createdAtMs: previous ? Math.max(previous.createdAtMs, createdAtMs) : createdAtMs, + // The new record event defines currentness provenance wholesale; a caller without + // boundary knowledge produces a legacy-style entry that fails safe to not-current. + ...(input.afterBoundaryMessageId !== undefined + ? { afterBoundaryMessageId: input.afterBoundaryMessageId } + : {}), }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 15bb3c686f7..42ef463b37b 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7730,6 +7730,17 @@ export class TaskService { ); } + /** + * Tool-path access to the invocation-boundary snapshot recorded into the + * agent-workflow-runs sidecar (see recordBackgroundWorkflowRunReference). + */ + async getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise { + return this.workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); + } + async markWorkflowRunTerminalAttentionConsumed(params: { ownerWorkspaceId: string; runId: string; diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index eb60511d88c..2fa64f7a332 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -98,7 +98,22 @@ export async function recordBackgroundWorkflowRunReference( } try { - await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs }); + // Snapshot which invocation-decision row is newest at launch so the terminal-wake + // currentness check compares row identity instead of wall-clock order, which clock + // corrections can reorder (see WorkspaceService.isWorkflowInvocationCurrent). + const afterBoundaryMessageId = + config.workspaceId != null + ? ((await config.taskService?.getWorkflowInvocationBoundaryMessageId?.( + config.workspaceId, + runId + )) ?? null) + : null; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs, + afterBoundaryMessageId, + }); } catch (error: unknown) { log.warn("Failed to record agent workflow run reference", { runId, diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index 0636a54d8b0..d8779d47e1b 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -13,6 +13,7 @@ import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { resolveWorkflowScript } from "@/node/services/workflows/workflowScriptResolver"; import { TestTempDir, createTestToolConfig, writeProjectSkill } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import type { TaskService } from "@/node/services/taskService"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkflowRunRecord } from "@/common/types/workflow"; @@ -650,9 +651,11 @@ describe("workflow_run tool", () => { result: null, })); const getRun = mock(async () => null); + const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); const tool = createWorkflowRunTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, workflowService: { startWorkflow, startWorkflowInBackground, @@ -665,8 +668,18 @@ describe("workflow_run tool", () => { mockToolCallOptions ); + // The reference must persist the invocation-boundary snapshot: currentness compares row + // identity, so a reference recorded without it fails safe and the wake is dropped. const references = await readAgentWorkflowRunReferences(tempDir.path); - expect(references.map((reference) => reference.runId)).toContain("wfr_background"); + expect(references).toHaveLength(1); + expect(references[0]).toMatchObject({ + runId: "wfr_background", + afterBoundaryMessageId: "boundary-row-1", + }); + expect(getWorkflowInvocationBoundaryMessageId).toHaveBeenCalledWith( + "workspace-1", + "wfr_background" + ); expect(startWorkflowInBackground).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 407ad8242c3..7ba2643cf5e 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5981,6 +5981,7 @@ describe("WorkspaceService workflow invocation events", () => { workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); @@ -5999,6 +6000,7 @@ describe("WorkspaceService workflow invocation events", () => { workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_250, + afterBoundaryMessageId: "manual-user-2", }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); @@ -6023,6 +6025,7 @@ describe("WorkspaceService workflow invocation events", () => { workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_350, + afterBoundaryMessageId: "workflow-result", }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); workspaceService.disposeSession(workspaceId); @@ -6066,6 +6069,7 @@ describe("WorkspaceService workflow invocation events", () => { workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); @@ -6075,6 +6079,110 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-clock"; + const runId = "wfr_currentness_clock"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-clock", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // A backward clock correction after recording makes the reference timestamp future-dated + // relative to every later history row; identity comparison must still deliver the wake. + const skewedCreatedAtMs = Date.now() + 30 * 60_000; + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: skewedCreatedAtMs, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A user message written after the correction has a smaller timestamp than the reference; + // wall-clock ordering would keep the stale reference current, identity must not. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("fails safe for legacy sidecar references without a boundary snapshot", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-legacy"; + const runId = "wfr_currentness_legacy"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-legacy", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // Entries written before boundary snapshots existed carry only a timestamp; without an + // orderable identity they must not count as the current invocation. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 41c99a20d80..73cc90e1a16 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10936,17 +10936,63 @@ export class WorkspaceService extends EventEmitter { assert(workspaceId.length > 0, "isWorkflowInvocationCurrent requires workspaceId"); assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); - let outcome: "invocation" | "consumed" | "superseded" | null = null; - let boundaryAtMs: number | null = null; + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + if (decision.status === "error") { + return false; + } + if (decision.status === "found" && decision.outcome === "invocation") { + return true; + } + + // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave + // no recognizable invocation part in history, so the backward walk above stops at the prior + // real user message (or, after a delivered result, at that consumed terminal message) and + // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the + // agent-workflow-runs sidecar, which snapshots the ID of the decision row that was newest + // at record time: the run is current exactly when that row is still the newest decision row. + // Row identity, not wall-clock ordering, so a backward clock correction can neither strand + // a legitimate wake nor let a pre-supersession reference outrank a newer boundary. For a + // consumed boundary, equality means a background resume/retry was recorded after the prior + // result was delivered. A decision-free history fails safe to not-current, because a full + // clear (truncateHistory) removes every row WITHOUT appending a reset boundary while + // leaving the sidecar intact — a surviving reference must not inject a workflow result into + // the freshly cleared conversation. Legacy references without a boundary snapshot fail safe + // the same way. + if (decision.status === "none") { + return false; + } + const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + const reference = references.find((candidate) => candidate.runId === runId); + if (reference?.afterBoundaryMessageId == null) { + return false; + } + return reference.afterBoundaryMessageId === decision.messageId; + } + + /** + * The newest invocation-decision row for this run: a manual user/reset supersession, a + * consumed terminal result for the run, or a direct invocation part. Shared by + * isWorkflowInvocationCurrent and the sidecar record path so both sides of the identity + * comparison classify rows identically. + */ + private async findWorkflowInvocationDecisionRow( + workspaceId: string, + runId: string + ): Promise< + | { status: "found"; outcome: "invocation" | "consumed" | "superseded"; messageId: string } + | { status: "none" } + | { status: "error" } + > { + const state: { + found: { outcome: "invocation" | "consumed" | "superseded"; messageId: string } | null; + } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", (messages) => { for (const message of messages) { if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { - outcome = "superseded"; - const timestamp = message.metadata?.timestamp; - boundaryAtMs = typeof timestamp === "number" ? timestamp : null; + state.found = { outcome: "superseded", messageId: message.id }; return false; } if ( @@ -10954,13 +11000,11 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - outcome = "consumed"; - const timestamp = message.metadata?.timestamp; - boundaryAtMs = typeof timestamp === "number" ? timestamp : null; + state.found = { outcome: "consumed", messageId: message.id }; return false; } if (isWorkflowInvocationMessage(message, runId)) { - outcome = "invocation"; + state.found = { outcome: "invocation", messageId: message.id }; return false; } } @@ -10973,34 +11017,27 @@ export class WorkspaceService extends EventEmitter { runId, error: historyResult.error, }); - return false; - } - - if (outcome === "invocation") { - return true; + return { status: "error" }; } + return state.found != null + ? { status: "found", outcome: state.found.outcome, messageId: state.found.messageId } + : { status: "none" }; + } - // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave - // no recognizable invocation part in history, so the backward walk above stops at the prior - // real user message (or, after a delivered result, at that consumed terminal message) and - // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the - // agent-workflow-runs sidecar: a reference recorded after that boundary counts as the - // current invocation. For a consumed boundary that means a background resume/retry issued - // after the prior result was delivered. The fallback requires a datable boundary: an - // undatable boundary fails safe to not-current (mirroring - // TaskService.listAgentReferencedWorkflowRunIds), and so does a decision-free history, - // because a full clear (truncateHistory) removes every row WITHOUT appending a reset - // boundary while leaving the sidecar intact — a surviving reference must not inject a - // workflow result into the freshly cleared conversation. - if (boundaryAtMs === null) { - return false; - } - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - const reference = references.find((candidate) => candidate.runId === runId); - if (reference == null) { - return false; - } - return reference.createdAtMs > boundaryAtMs; + /** + * Boundary snapshot for the agent-workflow-runs sidecar: the message ID of the newest + * invocation-decision row for this run, or null when history has none. Recorded at + * background launch/resume so isWorkflowInvocationCurrent can compare row identity instead + * of wall-clock timestamps, which clock corrections can reorder. + */ + async getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise { + assert(workspaceId.length > 0, "getWorkflowInvocationBoundaryMessageId requires workspaceId"); + assert(runId.length > 0, "getWorkflowInvocationBoundaryMessageId requires runId"); + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + return decision.status === "found" ? decision.messageId : null; } /** From 046286a0d1c344f9184f8a5200a9a31bb8b60d98 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:40:55 +0000 Subject: [PATCH 08/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20never=20persist=20a?= =?UTF-8?q?=20boundary=20snapshot=20from=20an=20unreadable=20history;=20de?= =?UTF-8?q?fer=20drains=20on=20indeterminate=20currentness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 53 +++++++++++++++ src/node/services/taskService.ts | 58 +++++++++++----- src/node/services/tools/toolUtils.ts | 35 +++++++--- src/node/services/tools/workflow_run.test.ts | 38 +++++++++++ src/node/services/workspaceService.test.ts | 71 ++++++++++++++++++++ src/node/services/workspaceService.ts | 34 ++++++++-- 6 files changed, 256 insertions(+), 33 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index b4d4131a4d6..d444e4402b3 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -665,6 +665,14 @@ function createWorkspaceServiceMocks( mock((_workspaceId: string, _message: WorkspaceChatMessage) => undefined); const isWorkflowInvocationCurrent = overrides?.isWorkflowInvocationCurrent ?? mock(() => Promise.resolve(true)); + // Derived from the boolean mock so tests that override isWorkflowInvocationCurrent keep + // steering the drain's three-state check. + const getWorkflowInvocationCurrentness = mock( + async (workspaceId: string, runId: string) => + ((await isWorkflowInvocationCurrent(workspaceId, runId)) === true + ? "current" + : "not_current") as "current" | "not_current" | "indeterminate" + ); const countQueuedAgentPeerMessages = overrides?.countQueuedAgentPeerMessages ?? mock(() => 0); // Granted by default (no live user activity): interrupt_active tests exercise the // interruption/archive flow; the hold's own refusal logic lives in workspaceService.test.ts. @@ -728,6 +736,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, + getWorkflowInvocationCurrentness, countQueuedAgentPeerMessages, } as unknown as WorkspaceService, create, @@ -6239,6 +6248,50 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("terminal workflow wake-up defers when history is unreadable", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_defer"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // History unreadable at drain time: currentness is indeterminate, so the notification must + // stay pending for a later drain instead of being tombstoned as superseded. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 42ef463b37b..17cc65a065d 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7893,7 +7893,9 @@ export class TaskService { private async buildWorkflowTerminalPrompt( ownerWorkspaceId: string, runId: string - ): Promise { + ): Promise< + { outcome: "deliver"; prompt: string } | { outcome: "superseded" } | { outcome: "defer" } + > { assert(ownerWorkspaceId.length > 0, "buildWorkflowTerminalPrompt requires ownerWorkspaceId"); assert(runId.length > 0, "buildWorkflowTerminalPrompt requires runId"); const runStore = new WorkflowRunStore({ @@ -7908,25 +7910,39 @@ export class TaskService { runId, error: getErrorMessage(error), }); - return null; + return { outcome: "superseded" }; } if ( run.workspaceId !== ownerWorkspaceId || run.parentWorkflow != null || - !isTerminalWorkflowRunStatus(run.status) || - !(await this.workspaceService.isWorkflowInvocationCurrent(ownerWorkspaceId, run.id)) + !isTerminalWorkflowRunStatus(run.status) ) { - return null; + return { outcome: "superseded" }; + } + const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( + ownerWorkspaceId, + run.id + ); + // Indeterminate means history was unreadable, not that the run was superseded: tombstoning + // now would permanently drop the wake over a transient fault, so defer and retry instead. + if (currentness === "indeterminate") { + return { outcome: "defer" }; + } + if (currentness === "not_current") { + return { outcome: "superseded" }; } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; - return buildWorkflowResultContextMessage({ - rawCommand: `workflow_run ${scriptPath}`, - name: scriptPath, - runId: run.id, - status: run.status, - result: null, - run, - }); + return { + outcome: "deliver", + prompt: buildWorkflowResultContextMessage({ + rawCommand: `workflow_run ${scriptPath}`, + name: scriptPath, + runId: run.id, + status: run.status, + result: null, + run, + }), + }; } private async ensureAgentTerminalMessages( @@ -8242,7 +8258,19 @@ export class TaskService { ownerWorkspaceId, notification.sourceId ); - if (workflowPrompt == null) { + if (workflowPrompt.outcome === "defer") { + // Currentness was indeterminate (history unreadable): keep the notification pending so + // the next drain trigger (a later terminal event, idle scheduling, or startup recovery) + // retries it, rather than permanently dropping the wake. No active reschedule here: an + // idle-wait resolves immediately on an idle owner and would busy-loop while the fault + // persists. + log.warn("Deferring workflow terminal attention; history unavailable", { + ownerWorkspaceId, + runId: notification.sourceId, + }); + continue; + } + if (workflowPrompt.outcome === "superseded") { // Dropping a notify_on_terminal wake strands the run's owner; keep the drop diagnosable. log.warn("Dropping superseded workflow terminal attention", { ownerWorkspaceId, @@ -8252,7 +8280,7 @@ export class TaskService { continue; } deliverableWorkflowNotificationIds.add(notification.id); - promptSections.push(workflowPrompt); + promptSections.push(workflowPrompt.prompt); } // Sub-agent reports and failures are already durable user-context messages. Resume from history diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 2fa64f7a332..a56f4574762 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -97,22 +97,35 @@ export async function recordBackgroundWorkflowRunReference( return; } + // Snapshot which invocation-decision row is newest at launch so the terminal-wake + // currentness check compares row identity instead of wall-clock order, which clock + // corrections can reorder (see WorkspaceService.isWorkflowInvocationCurrent). A history read + // failure must not be persisted as a verified-empty boundary (null): record without the + // field instead, so the run stays rediscoverable (listAgentReferencedWorkflowRunIds) and a + // later workflow_resume re-record can repair provenance, while the unverifiable boundary + // fails safe for wake delivery. + let afterBoundaryMessageId: string | null | undefined; + const taskService = config.taskService; + if (config.workspaceId != null && taskService?.getWorkflowInvocationBoundaryMessageId != null) { + try { + afterBoundaryMessageId = await taskService.getWorkflowInvocationBoundaryMessageId( + config.workspaceId, + runId + ); + } catch (error: unknown) { + log.error("Failed to snapshot workflow invocation boundary for run reference", { + runId, + error: getErrorMessage(error), + }); + } + } + try { - // Snapshot which invocation-decision row is newest at launch so the terminal-wake - // currentness check compares row identity instead of wall-clock order, which clock - // corrections can reorder (see WorkspaceService.isWorkflowInvocationCurrent). - const afterBoundaryMessageId = - config.workspaceId != null - ? ((await config.taskService?.getWorkflowInvocationBoundaryMessageId?.( - config.workspaceId, - runId - )) ?? null) - : null; await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs, - afterBoundaryMessageId, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), }); } catch (error: unknown) { log.warn("Failed to record agent workflow run reference", { diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index d8779d47e1b..05064896ffa 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -694,6 +694,44 @@ describe("workflow_run tool", () => { expect(result).toEqual({ status: "running", runId: "wfr_background", result: null }); }); + test("records a rediscovery-only reference when the boundary snapshot fails", async () => { + using tempDir = new TestTempDir("test-workflow-run-tool-boundary-error"); + const scriptPath = await writeWorkflowScript(tempDir.path); + const startWorkflowInBackground = mock(async () => ({ + runId: "wfr_boundary_error", + status: "running" as const, + result: null, + })); + const getWorkflowInvocationBoundaryMessageId = mock(async () => { + throw new Error("history read failed"); + }); + const tool = createWorkflowRunTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, + workflowService: { + startWorkflow: mock(async () => { + throw new Error("foreground start should not be used"); + }), + startWorkflowInBackground, + getRun: mock(async () => null), + }, + }); + + const result = await tool.execute!( + { script_path: scriptPath, args: { topic: "workflow tools" }, run_in_background: true }, + mockToolCallOptions + ); + expect(result).toEqual({ status: "running", runId: "wfr_boundary_error", result: null }); + + // A read failure must not persist a verified-empty boundary (null): the entry keeps the run + // rediscoverable while a later resume re-record can repair provenance. + const references = await readAgentWorkflowRunReferences(tempDir.path); + expect(references).toHaveLength(1); + expect(references[0]?.runId).toBe("wfr_boundary_error"); + expect(references[0] != null && "afterBoundaryMessageId" in references[0]).toBe(false); + }); + test("requires the workflow service", async () => { using tempDir = new TestTempDir("test-workflow-run-tool-missing"); const scriptPath = await writeWorkflowScript(tempDir.path); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7ba2643cf5e..d9eee2e3740 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6183,6 +6183,77 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("treats an unreadable history as indeterminate, not superseded", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-io-error"; + const runId = "wfr_currentness_io_error"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-io-error", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + const readSpy = spyOn(historyService, "iterateFullHistory").mockResolvedValue( + Err("disk read failed") + ); + try { + // The drain distinguishes a read failure (retain and retry) from supersession + // (tombstone); the boolean view stays fail-safe false for non-destructive callers. + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + // The record path must fail loudly instead of persisting a verified-empty boundary that + // would permanently strand the run's wake after storage recovers. + let boundaryError: unknown; + try { + await workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); + } catch (error: unknown) { + boundaryError = error; + } + expect(String(boundaryError)).toContain("boundary unavailable"); + } finally { + readSpy.mockRestore(); + } + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "current" + ); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 73cc90e1a16..472fc748390 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10933,15 +10933,29 @@ export class WorkspaceService extends EventEmitter { } async isWorkflowInvocationCurrent(workspaceId: string, runId: string): Promise { - assert(workspaceId.length > 0, "isWorkflowInvocationCurrent requires workspaceId"); - assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); + return (await this.getWorkflowInvocationCurrentness(workspaceId, runId)) === "current"; + } + + /** + * Three-state currentness: "indeterminate" means history could not be read, so the answer is + * unknown rather than no. Callers that would permanently drop a terminal wake on a negative + * answer (the terminal-attention drain tombstones notifications) must retain and retry on + * "indeterminate" instead; boolean callers treat it as not-current, the pre-existing + * fail-safe for non-destructive decisions. + */ + async getWorkflowInvocationCurrentness( + workspaceId: string, + runId: string + ): Promise<"current" | "not_current" | "indeterminate"> { + assert(workspaceId.length > 0, "getWorkflowInvocationCurrentness requires workspaceId"); + assert(runId.length > 0, "getWorkflowInvocationCurrentness requires runId"); const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); if (decision.status === "error") { - return false; + return "indeterminate"; } if (decision.status === "found" && decision.outcome === "invocation") { - return true; + return "current"; } // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave @@ -10959,14 +10973,14 @@ export class WorkspaceService extends EventEmitter { // the freshly cleared conversation. Legacy references without a boundary snapshot fail safe // the same way. if (decision.status === "none") { - return false; + return "not_current"; } const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); const reference = references.find((candidate) => candidate.runId === runId); if (reference?.afterBoundaryMessageId == null) { - return false; + return "not_current"; } - return reference.afterBoundaryMessageId === decision.messageId; + return reference.afterBoundaryMessageId === decision.messageId ? "current" : "not_current"; } /** @@ -11037,6 +11051,12 @@ export class WorkspaceService extends EventEmitter { assert(workspaceId.length > 0, "getWorkflowInvocationBoundaryMessageId requires workspaceId"); assert(runId.length > 0, "getWorkflowInvocationBoundaryMessageId requires runId"); const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + // A read failure must not masquerade as a verified-empty history: persisting null would + // permanently fail the run's currentness check even after storage recovers. Throw so the + // record path can distinguish and record a rediscovery-only reference instead. + if (decision.status === "error") { + throw new Error("workflow invocation boundary unavailable: history read failed"); + } return decision.status === "found" ? decision.messageId : null; } From e5de1dd3d12a3d8cb05da80e2db5c308f933860c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:00:58 +0000 Subject: [PATCH 09/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20migrate=20pre-snaps?= =?UTF-8?q?hot=20sidecar=20references=20through=20a=20wall-clock=20fallbac?= =?UTF-8?q?k?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/workspaceService.test.ts | 22 ++++++++-- src/node/services/workspaceService.ts | 49 ++++++++++++++++++---- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index d9eee2e3740..22cb999be36 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6137,7 +6137,7 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("fails safe for legacy sidecar references without a boundary snapshot", async () => { + test("migrates legacy sidecar references through the wall-clock fallback", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-legacy"; const runId = "wfr_currentness_legacy"; @@ -6169,13 +6169,29 @@ describe("WorkspaceService workflow invocation events", () => { workspaceId, createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) ); - // Entries written before boundary snapshots existed carry only a timestamp; without an - // orderable identity they must not count as the current invocation. + // Entries written before boundary snapshots existed carry only a timestamp. An in-flight + // run recorded after the newest boundary must keep its wake across the upgrade. await recordAgentWorkflowRunReference({ workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A newer boundary still supersedes a legacy entry. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // An undatable boundary cannot be ordered against a legacy timestamp: fail safe. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-undated", "user", "another instruction", {}) + ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); workspaceService.disposeSession(workspaceId); } finally { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 472fc748390..6dffcdb66b0 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10970,14 +10970,29 @@ export class WorkspaceService extends EventEmitter { // result was delivered. A decision-free history fails safe to not-current, because a full // clear (truncateHistory) removes every row WITHOUT appending a reset boundary while // leaving the sidecar intact — a surviving reference must not inject a workflow result into - // the freshly cleared conversation. Legacy references without a boundary snapshot fail safe - // the same way. + // the freshly cleared conversation. References without a boundary snapshot (pre-upgrade + // entries, record-time read failures) take a wall-clock migration fallback below. if (decision.status === "none") { return "not_current"; } const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); const reference = references.find((candidate) => candidate.runId === runId); - if (reference?.afterBoundaryMessageId == null) { + if (reference == null) { + return "not_current"; + } + if (reference.afterBoundaryMessageId === undefined) { + // Migration fallback: entries written before boundary snapshots existed (or after a + // record-time history read failure) carry only a timestamp, and an in-flight run must not + // lose its wake across the upgrade. Fall back to wall-clock ordering against a datable + // boundary; an undatable boundary fails safe. All new records take the identity path + // above, so clock-correction edge cases are confined to this shrinking population. + if (decision.timestampMs == null) { + return "not_current"; + } + return reference.createdAtMs > decision.timestampMs ? "current" : "not_current"; + } + if (reference.afterBoundaryMessageId === null) { + // Verified-empty snapshot: a decision row now exists, so it appeared after the record. return "not_current"; } return reference.afterBoundaryMessageId === decision.messageId ? "current" : "not_current"; @@ -10993,20 +11008,31 @@ export class WorkspaceService extends EventEmitter { workspaceId: string, runId: string ): Promise< - | { status: "found"; outcome: "invocation" | "consumed" | "superseded"; messageId: string } + | { + status: "found"; + outcome: "invocation" | "consumed" | "superseded"; + messageId: string; + timestampMs: number | null; + } | { status: "none" } | { status: "error" } > { const state: { - found: { outcome: "invocation" | "consumed" | "superseded"; messageId: string } | null; + found: { + outcome: "invocation" | "consumed" | "superseded"; + messageId: string; + timestampMs: number | null; + } | null; } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", (messages) => { for (const message of messages) { + const timestamp = message.metadata?.timestamp; + const timestampMs = typeof timestamp === "number" ? timestamp : null; if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { - state.found = { outcome: "superseded", messageId: message.id }; + state.found = { outcome: "superseded", messageId: message.id, timestampMs }; return false; } if ( @@ -11014,11 +11040,11 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - state.found = { outcome: "consumed", messageId: message.id }; + state.found = { outcome: "consumed", messageId: message.id, timestampMs }; return false; } if (isWorkflowInvocationMessage(message, runId)) { - state.found = { outcome: "invocation", messageId: message.id }; + state.found = { outcome: "invocation", messageId: message.id, timestampMs }; return false; } } @@ -11034,7 +11060,12 @@ export class WorkspaceService extends EventEmitter { return { status: "error" }; } return state.found != null - ? { status: "found", outcome: state.found.outcome, messageId: state.found.messageId } + ? { + status: "found", + outcome: state.found.outcome, + messageId: state.found.messageId, + timestampMs: state.found.timestampMs, + } : { status: "none" }; } From 60bb82d89f7325043832fd833c14df998ae2e3f6 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:14:54 +0000 Subject: [PATCH 10/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20deliver=20kernel=20?= =?UTF-8?q?workflow=20wakes=20launched=20from=20a=20decision-free=20histor?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A kernel background workflow launched from a synthetic turn in a new or fully cleared workspace records a verified-empty boundary snapshot (afterBoundaryMessageId: null), but getWorkflowInvocationCurrentness declared every decision-free history not_current, permanently superseding the run's terminal wake. Honor the verified-empty snapshot: null matching a still decision-free history means the launch context is unchanged, so the run is current. References pointing at a cleared row or lacking a verified snapshot keep failing safe, preserving the full-clear protection. --- src/node/services/workspaceService.test.ts | 65 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 21 ++++--- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 22cb999be36..0ac43c1a3cd 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6079,6 +6079,71 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("delivers kernel launches recorded against a decision-free history", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-empty"; + const runId = "wfr_currentness_empty"; + const legacyRunId = "wfr_currentness_empty_legacy"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-empty", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A kernel launch from a synthetic turn in a new (or fully cleared) workspace records a + // verified-empty snapshot (null). History still having no decision row means the launch + // context is unchanged, so the wake must deliver. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A reference without a verified snapshot cannot claim the empty history as its launch + // context; it may merely have survived a full clear. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: legacyRunId, + createdAtMs: 1_150, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, legacyRunId)).toBe( + false + ); + + // A decision row appearing after the launch supersedes the verified-empty snapshot. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6dffcdb66b0..1c6d3c18389 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10967,16 +10967,21 @@ export class WorkspaceService extends EventEmitter { // Row identity, not wall-clock ordering, so a backward clock correction can neither strand // a legitimate wake nor let a pre-supersession reference outrank a newer boundary. For a // consumed boundary, equality means a background resume/retry was recorded after the prior - // result was delivered. A decision-free history fails safe to not-current, because a full - // clear (truncateHistory) removes every row WITHOUT appending a reset boundary while - // leaving the sidecar intact — a surviving reference must not inject a workflow result into - // the freshly cleared conversation. References without a boundary snapshot (pre-upgrade - // entries, record-time read failures) take a wall-clock migration fallback below. - if (decision.status === "none") { - return "not_current"; - } + // result was delivered. References without a boundary snapshot (pre-upgrade entries, + // record-time read failures) take a wall-clock migration fallback below. const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); const reference = references.find((candidate) => candidate.runId === runId); + if (decision.status === "none") { + // A decision-free history is current only for a reference whose snapshot verified an + // empty history at record time (null): kernel launches from a new or fully cleared + // workspace (e.g. a heartbeat turn) have no decision row before or after, and their wake + // must still deliver. Every other surviving reference fails safe, because a full clear + // (truncateHistory) removes every row WITHOUT appending a reset boundary while leaving + // the sidecar intact, and a reference pointing at a cleared row, or one without a + // verified snapshot, must not inject a workflow result into the freshly cleared + // conversation. + return reference?.afterBoundaryMessageId === null ? "current" : "not_current"; + } if (reference == null) { return "not_current"; } From a73a35428447759bce3eefc3b18f19b7b09ac5da Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:36:23 +0000 Subject: [PATCH 11/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20harden=20kernel=20w?= =?UTF-8?q?orkflow=20wake=20delivery=20against=20sidecar=20faults?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reject sidecar entries whose afterBoundaryMessageId is present but invalid (empty string or non-string) instead of migrating them into the wall-clock legacy fallback, where they could outrank a newer boundary during tolerated clock skew. - Propagate non-ENOENT sidecar read failures instead of flattening them to an empty list; workflow currentness reports indeterminate so the terminal drain defers rather than tombstoning the wake, while rediscovery listings skip the pass and record keeps its atomic rewrite. - Arm a bounded per-owner retry timer when a drain defers on indeterminate currentness: an already-idle owner otherwise produces no further drain trigger until restart. --- .../agentWorkflowRunReferences.test.ts | 55 +++++++++++++++ .../services/agentWorkflowRunReferences.ts | 44 +++++++++--- src/node/services/taskService.test.ts | 63 +++++++++++++++++ src/node/services/taskService.ts | 47 +++++++++++-- src/node/services/workspaceService.test.ts | 69 +++++++++++++++++++ src/node/services/workspaceService.ts | 20 +++++- 6 files changed, 282 insertions(+), 16 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 2747eb1fe65..d4e893ef177 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -114,6 +114,61 @@ describe("agent workflow run references", () => { } }); + test("rejects entries with a present-but-invalid boundary snapshot", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // "" and non-string values are corruption, not legacy records: migrating them into the + // wall-clock fallback could outrank a newer boundary during tolerated clock skew. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_empty_boundary", createdAtMs: 1_000, afterBoundaryMessageId: "" }, + { runId: "wfr_numeric_boundary", createdAtMs: 1_000, afterBoundaryMessageId: 42 }, + { runId: "wfr_valid_boundary", createdAtMs: 1_000, afterBoundaryMessageId: "row-1" }, + { runId: "wfr_null_boundary", createdAtMs: 1_000, afterBoundaryMessageId: null }, + ], + }) + ); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(new Set(references.map((reference) => reference.runId))).toEqual( + new Set(["wfr_valid_boundary", "wfr_null_boundary"]) + ); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("propagates non-ENOENT read failures instead of flattening them to empty", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A directory at the file path fails reads with EISDIR. Callers deciding wake delivery + // must observe the failure rather than "no references". + await fs.mkdir(path.join(workspaceSessionDir, "agent-workflow-runs.json")); + let readError: unknown; + try { + await readAgentWorkflowRunReferences(workspaceSessionDir); + } catch (error: unknown) { + readError = error; + } + expect(String(readError)).toContain("EISDIR"); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("self-heals unparseable file contents to empty", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Unlike a failed read, corrupted contents cannot be repaired by rereading. + await fs.writeFile(path.join(workspaceSessionDir, "agent-workflow-runs.json"), "{not json"); + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("clamps future-dated createdAtMs to the current time", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 58f23023f9e..04aab142e65 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -63,13 +63,24 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (record.createdAtMs > now + MAX_FUTURE_SKEW_MS) { continue; } + const hasBoundary = "afterBoundaryMessageId" in record; const boundaryRaw = record.afterBoundaryMessageId; - const afterBoundaryMessageId = - typeof boundaryRaw === "string" && boundaryRaw.length > 0 + // A present-but-invalid snapshot ("" or a non-string) is corruption, not a legacy record: + // migrating it into the wall-clock fallback could let a stale reference outrank a newer + // boundary within the tolerated clock skew. Reject the entry; absence stays reserved for + // records that genuinely predate the field. + if ( + hasBoundary && + boundaryRaw !== null && + (typeof boundaryRaw !== "string" || boundaryRaw.length === 0) + ) { + continue; + } + const afterBoundaryMessageId = hasBoundary + ? typeof boundaryRaw === "string" ? boundaryRaw - : boundaryRaw === null - ? null - : undefined; + : null + : undefined; // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run // superseded. The chosen record is kept wholesale, including its boundary snapshot. @@ -88,13 +99,23 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { export async function readAgentWorkflowRunReferences( workspaceSessionDir: string ): Promise { + let raw: string; try { - const raw = await fs.readFile(referencesPath(workspaceSessionDir), "utf-8"); - return parseReferences(JSON.parse(raw) as unknown); + raw = await fs.readFile(referencesPath(workspaceSessionDir), "utf-8"); } catch (error: unknown) { if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { return []; } + // For kernel-launched runs this file is the only durable invocation evidence, and callers + // deciding wake delivery must distinguish "no reference" from "cannot know right now": + // flattening a transient read failure into [] would let the terminal drain tombstone the + // run's wake. Corrupted contents below stay self-healing because rereading cannot repair + // them, while a failed read can succeed later. + throw error; + } + try { + return parseReferences(JSON.parse(raw) as unknown); + } catch { return []; } } @@ -109,7 +130,14 @@ export async function recordAgentWorkflowRunReference(input: { const filePath = referencesPath(input.workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { - const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + let existing: AgentWorkflowRunReference[]; + try { + existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + } catch { + // Recording must survive an unreadable file: the atomic rewrite below replaces it, and + // failing here would leave the new run without any sidecar entry, stranding its wake. + existing = []; + } const byRunId = new Map(existing.map((reference) => [reference.runId, reference])); // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index d444e4402b3..2466f5292ea 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6292,6 +6292,69 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); }); + test("deferred terminal wake-up retries on the bounded timer", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_defer_retry"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // The first drain sees a transient storage fault that clears before the retry fires. An + // already-idle owner produces no other drain trigger, so only the bounded retry delivers. + let currentnessCalls = 0; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => { + currentnessCalls += 1; + return Promise.resolve(currentnessCalls === 1 ? "indeterminate" : "current"); + }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + ( + taskService as unknown as { terminalAttentionDeferRetryDelayMs: number } + ).terminalAttentionDeferRetryDelayMs = 10; + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + + // Real timers: poll until the armed retry fires and the follow-up drain delivers. + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && sendMessage.mock.calls.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 10)); + await flushTerminalAttentionDrains(taskService); + } + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 17cc65a065d..785df8ba671 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -862,6 +862,11 @@ function isWorkspaceBusyIdleOnlySend(error: unknown): boolean { const REMOVED_AGENT_TASKS_DIR = "removed-agent-tasks"; const COMPLETED_REPORT_CACHE_MAX_ENTRIES = 128; +// Retry cadence for terminal-attention drains deferred by indeterminate workflow currentness +// (unreadable history/sidecar). There is no deterministic "storage recovered" signal, so a +// bounded timer is the re-trigger; each retry that defers again arms the next one. +const TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS = 30_000; + /** Maximum consecutive auto-resumes before stopping. Prevents infinite loops when descendants are stuck. */ // Task-recovery paths must stay deterministic and editing-capable even when // workspace/default agent preferences evolve (e.g., auto router defaults). @@ -1538,6 +1543,13 @@ export class TaskService { // tests and shutdown can await them; drains are idempotent and re-triggered on owner idle events. private readonly pendingTerminalAttentionDrainsByOwner = new Map>(); private readonly pendingTerminalAttentionDrains = new Set>(); + // One armed defer-retry timer per owner (see scheduleTerminalAttentionDeferRetry). The delay + // is a field, not a constant, so tests can shrink it without waiting out the real backoff. + private readonly terminalAttentionDeferRetryTimers = new Map< + string, + ReturnType + >(); + private terminalAttentionDeferRetryDelayMs = TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS; private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); // Tracks workspaces currently blocked in a foreground wait (e.g. a task tool call awaiting @@ -1690,7 +1702,13 @@ export class TaskService { } const runIds = new Set(); - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + let references: Awaited> = []; + try { + references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error: unknown) { + // Rediscovery is non-destructive and re-runs on the next listing; skip this pass. + log.warn("Failed to read agent workflow run references", { workspaceId, error }); + } for (const reference of references) { // If the latest user/reset supersession has no durable timestamp, fail safe: only trust // workflow provenance re-established by current/post-supersession assistant output below. @@ -7872,6 +7890,24 @@ export class TaskService { this.pendingTerminalAttentionDrains.add(promise); } + /** + * A deferred (indeterminate) terminal wake has no deterministic "storage recovered" signal + * to re-trigger the drain, and an idle-wait would resolve immediately on an already-idle + * owner and busy-loop while the fault persists. Retry on a bounded timer instead, one armed + * timer per owner; each retry that defers again arms the next one. + */ + private scheduleTerminalAttentionDeferRetry(ownerWorkspaceId: string): void { + if (this.terminalAttentionDeferRetryTimers.has(ownerWorkspaceId)) { + return; + } + const timer = setTimeout(() => { + this.terminalAttentionDeferRetryTimers.delete(ownerWorkspaceId); + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + }, this.terminalAttentionDeferRetryDelayMs); + timer.unref?.(); + this.terminalAttentionDeferRetryTimers.set(ownerWorkspaceId, timer); + } + private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { const promise = this.workspaceService .waitForIdleAndNoQueuedMessages(ownerWorkspaceId) @@ -8259,15 +8295,14 @@ export class TaskService { notification.sourceId ); if (workflowPrompt.outcome === "defer") { - // Currentness was indeterminate (history unreadable): keep the notification pending so - // the next drain trigger (a later terminal event, idle scheduling, or startup recovery) - // retries it, rather than permanently dropping the wake. No active reschedule here: an - // idle-wait resolves immediately on an idle owner and would busy-loop while the fault - // persists. + // Currentness was indeterminate (history or sidecar unreadable): keep the notification + // pending rather than permanently dropping the wake, and arm a bounded retry, because an + // already-idle owner produces no further drain trigger on its own. log.warn("Deferring workflow terminal attention; history unavailable", { ownerWorkspaceId, runId: notification.sourceId, }); + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); continue; } if (workflowPrompt.outcome === "superseded") { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 0ac43c1a3cd..9fb652f6cf0 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6335,6 +6335,75 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("treats an unreadable sidecar as indeterminate, not superseded", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-sidecar-error"; + const runId = "wfr_currentness_sidecar_error"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-sidecar-error", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // The sidecar is the only invocation evidence for kernel-launched runs: an unreadable + // file must read as "cannot know right now", not "no reference", or the drain would + // tombstone the wake on a transient storage fault. + const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); + await fsPromises.rm(sidecarPath); + await fsPromises.mkdir(sidecarPath); + try { + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + } finally { + await fsPromises.rmdir(sidecarPath); + } + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "current" + ); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 1c6d3c18389..c9a3182dd62 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3,7 +3,10 @@ import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { EventEmitter } from "events"; import * as path from "path"; import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; -import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import { + readAgentWorkflowRunReferences, + type AgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; @@ -10969,7 +10972,20 @@ export class WorkspaceService extends EventEmitter { // consumed boundary, equality means a background resume/retry was recorded after the prior // result was delivered. References without a boundary snapshot (pre-upgrade entries, // record-time read failures) take a wall-clock migration fallback below. - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + let references: AgentWorkflowRunReference[]; + try { + references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error: unknown) { + // The sidecar is the only invocation evidence a kernel-launched run has, so an + // unreadable file is "cannot know right now", not "no reference": defer wake decisions + // exactly like an unreadable history. + log.warn("Could not read workflow run references for currentness", { + workspaceId, + runId, + error, + }); + return "indeterminate"; + } const reference = references.find((candidate) => candidate.runId === runId); if (decision.status === "none") { // A decision-free history is current only for a reference whose snapshot verified an From 983d3c5eac4e32446262960e774cb4d0f0e16b8c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:46:45 +0000 Subject: [PATCH 12/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20retire=20kernel=20w?= =?UTF-8?q?orkflow=20run=20references=20on=20a=20full=20history=20clear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verified-empty (null) boundary snapshot recorded before a full clear is indistinguishable from one recorded after it, because the clear removes every row without appending a reset boundary. A pre-clear reference could therefore inject its workflow result into the freshly cleared conversation. Retire the sidecar with the transcript, durably like the post-compaction carryover discard; a post-clear resume re-records provenance. --- .../services/agentWorkflowRunReferences.ts | 14 +++++ src/node/services/workspaceService.test.ts | 57 +++++++++++++++++++ src/node/services/workspaceService.ts | 15 +++++ 3 files changed, 86 insertions(+) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 04aab142e65..d4ef37394be 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -120,6 +120,20 @@ export async function readAgentWorkflowRunReferences( } } +/** + * Retire every reference for this workspace. A full history clear removes all rows without + * appending a reset boundary, which makes a verified-empty (null) boundary snapshot recorded + * before the clear indistinguishable from one recorded after it; retiring the references with + * the transcript keeps pre-clear workflow results out of the fresh conversation. A post-clear + * workflow_resume re-records provenance. + */ +export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { + const filePath = referencesPath(workspaceSessionDir); + await referenceFileLocks.withLock(filePath, async () => { + await fs.rm(filePath, { force: true }); + }); +} + export async function recordAgentWorkflowRunReference(input: { workspaceSessionDir: string; runId: string; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 9fb652f6cf0..ed803ed0890 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6144,6 +6144,63 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("retires kernel workflow run references on a full history clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire"; + const runId = "wfr_currentness_retire"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // Launched from a decision-free history: the verified-empty snapshot delivers. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A full clear returns history to decision-free, making the pre-clear null snapshot + // indistinguishable from a fresh empty-history launch; the clear must retire the + // reference so the stale result cannot inject into the fresh conversation. + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(true); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c9a3182dd62..6dfde259062 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4,6 +4,7 @@ import { EventEmitter } from "events"; import * as path from "path"; import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; import { + clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, type AgentWorkflowRunReference, } from "@/node/services/agentWorkflowRunReferences"; @@ -12958,6 +12959,20 @@ export class WorkspaceService extends EventEmitter { `be re-injected after a restart; retry once the session storage is writable.` ); } + // Kernel workflow run references belong to the cleared conversation: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one + // recorded after it, so a surviving reference could inject a pre-clear workflow result + // into the fresh conversation. Retire them with the transcript (a post-clear resume + // re-records provenance), durably like the carryover discard above. + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `History was cleared, but stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + + `result into the cleared conversation; retry once the session storage is writable.` + ); + } // The persistent RLM sandbox holds context DERIVED from the cleared // transcript (vars populated by code execution), and its latest durable // snapshot would restore it after a restart — later turns could read From 56c789b44da7e8229cf393bc894a5b169c2985e9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:13:15 +0000 Subject: [PATCH 13/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20close=20round-10=20?= =?UTF-8?q?wake-delivery=20gaps:=20retirement=20ordering,=20record=20clobb?= =?UTF-8?q?er,=20coalesced=20replay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Retire kernel workflow run references immediately after the truncation commits, before post-clear steps (goal acknowledgment, carryover discard) that can fail and return early while leaving the transcript deleted. - Propagate a sidecar read failure out of recordAgentWorkflowRunReference instead of treating the file as empty: the atomic rewrite would replace valid-but-unreadable contents with only the new run, destroying every other run's durable provenance. The failed record is retryable; parse corruption still self-heals. - Recognize coalesced terminal-attention prompts as consumption during currentness checks: the drain's synthetic user row carries no workflow-result metadata, so a crash between durable acceptance and the outbox delivery mark would otherwise replay the same terminal result after restart. Payload blocks are parsed back and matched on the exact workflow.runId the builder wrote. --- src/common/utils/workflowRunMessages.ts | 35 +++++ .../agentWorkflowRunReferences.test.ts | 32 ++++ .../services/agentWorkflowRunReferences.ts | 14 +- src/node/services/workspaceService.test.ts | 147 ++++++++++++++++++ src/node/services/workspaceService.ts | 49 ++++-- 5 files changed, 255 insertions(+), 22 deletions(-) diff --git a/src/common/utils/workflowRunMessages.ts b/src/common/utils/workflowRunMessages.ts index 9fc393e9f7e..66a2b3b5fa9 100644 --- a/src/common/utils/workflowRunMessages.ts +++ b/src/common/utils/workflowRunMessages.ts @@ -190,6 +190,41 @@ export function buildWorkflowResultContextMessage(input: { ].join("\n\n"); } +/** + * Recognize this run's result payload inside a coalesced terminal-attention prompt from the + * builder's own output format. The drain's synthetic user row can coalesce several runs into + * one message and carries no workflow-result metadata, so currentness checks must read the + * consumption evidence out of the text: each payload block is parsed back and matched on the + * exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted + * inside another run's report cannot count as consumption. + */ +export function textContainsWorkflowResultPayload(text: string, runId: string): boolean { + assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required"); + if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) { + return false; + } + const blockPattern = new RegExp( + `<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n`, + "g" + ); + for (const match of text.matchAll(blockPattern)) { + let payload: unknown; + try { + payload = JSON.parse(match[1] ?? ""); + } catch { + continue; + } + if (!isRecordValue(payload)) { + continue; + } + const workflow = payload.workflow; + if (isRecordValue(workflow) && workflow.runId === runId) { + return true; + } + } + return false; +} + export interface WorkflowRunCardInput { scriptPath?: string; scriptSource?: string; diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index d4e893ef177..69171c081cf 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -158,6 +158,38 @@ describe("agent workflow run references", () => { } }); + test("record propagates a sidecar read failure instead of clobbering existing references", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_existing", + createdAtMs: 1_000, + }); + const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); + // Unreadable file, writable directory: the atomic rewrite could replace contents it + // never saw, destroying every other run's only durable provenance. + await fs.chmod(filePath, 0o000); + let recordError: unknown; + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_new", + createdAtMs: 2_000, + }); + } catch (error: unknown) { + recordError = error; + } finally { + await fs.chmod(filePath, 0o600); + } + expect(String(recordError)).toContain("EACCES"); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references.map((reference) => reference.runId)).toEqual(["wfr_existing"]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("self-heals unparseable file contents to empty", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index d4ef37394be..43a9d23a778 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -144,14 +144,12 @@ export async function recordAgentWorkflowRunReference(input: { const filePath = referencesPath(input.workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { - let existing: AgentWorkflowRunReference[]; - try { - existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); - } catch { - // Recording must survive an unreadable file: the atomic rewrite below replaces it, and - // failing here would leave the new run without any sidecar entry, stranding its wake. - existing = []; - } + // A read failure propagates instead of being treated as empty: the atomic rewrite below + // would otherwise replace valid-but-momentarily-unreadable contents with only this run, + // destroying every other active run's sole durable provenance. The failed record is + // retryable (workflow_resume re-records), while parse corruption still self-heals to + // empty inside readAgentWorkflowRunReferences because rereading cannot repair it. + const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); const byRunId = new Map(existing.map((reference) => [reference.runId, reference])); // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ed803ed0890..daac7889e48 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -57,6 +57,7 @@ import { WORKFLOW_RESULT_METADATA_TYPE, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, + buildWorkflowResultContextMessage, } from "@/common/utils/workflowRunMessages"; import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; @@ -6201,6 +6202,152 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("retires kernel workflow run references even when a later post-clear step fails", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire-early"; + const runId = "wfr_currentness_retire_early"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire-early", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + + // The truncation commits, then a later post-clear step fails. Retirement must already + // have happened, or the stale null-snapshot reference survives the committed clear and + // reads current against the emptied history. + const sessionAccessor = workspaceService as unknown as { + getOrCreateSession(id: string): { clearPostCompactionState(): Promise }; + }; + const session = sessionAccessor.getOrCreateSession(workspaceId); + const carryoverSpy = spyOn(session, "clearPostCompactionState").mockImplementationOnce(() => + Promise.reject(new Error("carryover discard failed")) + ); + try { + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(false); + } finally { + carryoverSpy.mockRestore(); + } + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-coalesced"; + const runId = "wfr_currentness_coalesced"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-coalesced", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // Another run's payload quoting nothing about this run must not count as consumption. + await historyService.appendToHistory( + workspaceId, + createMuxMessage( + "coalesced-other", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run other.js", + name: "other.js", + runId: "wfr_currentness_other", + status: "completed", + result: { reportMarkdown: "other done" }, + run: null, + }), + { timestamp: 1_250, synthetic: true } + ) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a + // crash between durable acceptance and the outbox delivery mark, this row is the only + // evidence the result already reached history; it must read as consumption or restart + // recovery injects the same terminal result again. + await historyService.appendToHistory( + workspaceId, + createMuxMessage( + "coalesced-result", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run research.js", + name: "research.js", + runId, + status: "completed", + result: { reportMarkdown: "done" }, + run: null, + }), + { timestamp: 1_300, synthetic: true } + ) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6dfde259062..bb95cabd55c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -204,6 +204,7 @@ import { } from "@/node/services/workflows/workflowArchiveAdmission"; import { WORKFLOW_RESULT_METADATA_TYPE, + textContainsWorkflowResultPayload, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowRunCardMessage, @@ -475,6 +476,23 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) ); } +/** + * The terminal-attention drain delivers workflow results as one synthetic user prompt that may + * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands + * between the send's durable acceptance and the outbox delivery mark, restart recovery drains + * the notification again; recognizing the accepted row as consumption is what suppresses the + * replay. Only synthetic rows qualify: a manual user message is a supersession boundary and is + * classified before this check runs. + */ +function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { + if (message.role !== "user" || message.metadata?.synthetic !== true) { + return false; + } + return message.parts.some( + (part) => part.type === "text" && textContainsWorkflowResultPayload(part.text, runId) + ); +} + function isResetBoundaryMessage(message: MuxMessage): boolean { return message.metadata?.contextBoundaryKind === CONTEXT_BOUNDARY_KINDS.RESET; } @@ -11059,6 +11077,7 @@ export class WorkspaceService extends EventEmitter { } if ( isWorkflowResultContinuationMessage(message, runId) || + isCoalescedWorkflowResultMessage(message, runId) || isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { @@ -12901,6 +12920,22 @@ export class WorkspaceService extends EventEmitter { // admitted afterwards (their content references the discarded context). if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); + // Kernel workflow run references belong to the cleared conversation: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one + // recorded after it, so a surviving reference could inject a pre-clear workflow result + // into the fresh conversation. Retire them immediately after the truncation commits, + // before any later post-clear step that can fail and return early (goal acknowledgment, + // carryover discard), or the stale reference would survive the committed clear. A + // post-clear resume re-records provenance. + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `History was cleared, but stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + + `result into the cleared conversation; retry once the session storage is writable.` + ); + } } // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the @@ -12959,20 +12994,6 @@ export class WorkspaceService extends EventEmitter { `be re-injected after a restart; retry once the session storage is writable.` ); } - // Kernel workflow run references belong to the cleared conversation: a verified-empty - // (null) boundary snapshot recorded before the clear is indistinguishable from one - // recorded after it, so a surviving reference could inject a pre-clear workflow result - // into the fresh conversation. Retire them with the transcript (a post-clear resume - // re-records provenance), durably like the carryover discard above. - try { - await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error) { - return Err( - `History was cleared, but stale workflow run references could not be retired ` + - `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + - `result into the cleared conversation; retry once the session storage is writable.` - ); - } // The persistent RLM sandbox holds context DERIVED from the cleared // transcript (vars populated by code execution), and its latest durable // snapshot would restore it after a restart — later turns could read From 521fba2b778747b7a727fd963e9a2b463f87274c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:30:53 +0000 Subject: [PATCH 14/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20close=20round-11=20?= =?UTF-8?q?gaps:=20record=20retry,=20indeterminate=20recovery,=20clear-emi?= =?UTF-8?q?t=20ordering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A failed provenance record now schedules bounded background retries (1s/10s/60s, reusing the launch-time boundary snapshot): the only natural re-record sites are a new dispatch and workflow_resume, which an untouched active run never hits, so a single failed write would permanently supersede its wake once storage recovers. - Startup recovery keeps indeterminate runs: it now consults three-state currentness and skips only not_current, because no pending notification exists yet to arm the drain's defer retry; the drain re-evaluates and defers or supersedes with full context. - A failed sidecar retirement after a committed full clear now emits the DeleteMessage before returning the cleanup error, so the renderer does not keep showing a transcript that no longer exists on disk. --- src/node/services/taskService.test.ts | 45 ++++++++++++++++ src/node/services/taskService.ts | 10 +++- src/node/services/tools/toolUtils.test.ts | 52 ++++++++++++++++++ src/node/services/tools/toolUtils.ts | 56 +++++++++++++++++++- src/node/services/workspaceService.test.ts | 61 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 38 +++++++++----- 6 files changed, 246 insertions(+), 16 deletions(-) create mode 100644 src/node/services/tools/toolUtils.test.ts diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 2466f5292ea..23b0f450909 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6752,6 +6752,51 @@ describe("TaskService", () => { }); }); + test("initialize recovery keeps indeterminate workflow runs enqueued for a later drain", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_recovery_indeterminate"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // History/sidecar unreadable at startup. Recovery is the only reconstruction point for a + // wake that never reached the outbox, and no pending notification exists yet to arm the + // drain's defer retry, so skipping here would strand the run until another restart. The + // boolean wrapper collapses indeterminate to false, which is what recovery must NOT use. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + (workspaceService as unknown as Record).isWorkflowInvocationCurrent = mock( + () => Promise.resolve(false) + ); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + await taskService.initialize(); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + test("initialize recovers terminal notify workspace turns without pending notification", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 785df8ba671..1bc7797e56b 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7596,7 +7596,15 @@ export class TaskService { ) { continue; } - if (!(await this.workspaceService.isWorkflowInvocationCurrent(workspace.id, run.id))) { + const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( + workspace.id, + run.id + ); + // Indeterminate (unreadable history/sidecar) must still enqueue: startup recovery is + // the only reconstruction point for wakes that never reached the outbox, and no + // pending notification exists yet to arm the drain's defer retry. The drain + // re-evaluates currentness and defers or supersedes with full context. + if (currentness === "not_current") { continue; } const created = await this.terminalAttentionStore.enqueueIfAbsent({ diff --git a/src/node/services/tools/toolUtils.test.ts b/src/node/services/tools/toolUtils.test.ts new file mode 100644 index 00000000000..7552b08a7da --- /dev/null +++ b/src/node/services/tools/toolUtils.test.ts @@ -0,0 +1,52 @@ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; + +import { describe, expect, test } from "bun:test"; + +import type { ToolConfiguration } from "@/common/utils/tools/tools"; +import { + readAgentWorkflowRunReferences, + recordAgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; +import { recordBackgroundWorkflowRunReference } from "@/node/services/tools/toolUtils"; + +describe("recordBackgroundWorkflowRunReference", () => { + test("retries a failed provenance record until storage recovers", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-record-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_existing", + createdAtMs: 1_000, + }); + const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); + // Unreadable at record time: the tool has already returned by the time storage recovers, + // and an untouched active run never hits a natural re-record site, so only the bounded + // background retry can persist provenance for the terminal wake. + await fs.chmod(filePath, 0o000); + await recordBackgroundWorkflowRunReference( + { workspaceSessionDir } as unknown as ToolConfiguration, + "wfr_retry", + 2_000, + [25, 25, 25] + ); + await fs.chmod(filePath, 0o600); + + const deadline = Date.now() + 5_000; + let runIds: string[] = []; + while (Date.now() < deadline) { + runIds = (await readAgentWorkflowRunReferences(workspaceSessionDir)).map( + (reference) => reference.runId + ); + if (runIds.includes("wfr_retry")) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(new Set(runIds)).toEqual(new Set(["wfr_existing", "wfr_retry"])); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index a56f4574762..bf90c448775 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -79,6 +79,51 @@ export async function emitWorkflowRunAttachedEvent(input: { await input.config.emitChatEvent(event); } +// Bounded background retries for a transiently unreadable sidecar. The only natural re-record +// sites are a new dispatch and workflow_resume, which an untouched active run never hits, so +// giving up after one failed write would permanently supersede the run's terminal wake once +// storage recovers. Retries reuse the launch-time boundary snapshot: provenance describes the +// launch, not the retry moment. +const RECORD_REFERENCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; + +function scheduleRecordReferenceRetry(input: { + workspaceSessionDir: string; + runId: string; + createdAtMs: number; + afterBoundaryMessageId: string | null | undefined; + retryDelaysMs: readonly number[]; + attempt: number; +}): void { + const delayMs = input.retryDelaysMs[input.attempt]; + if (delayMs == null) { + log.error("Giving up on agent workflow run reference record after retries", { + runId: input.runId, + attempts: input.attempt, + }); + return; + } + const timer = setTimeout(() => { + // Detached by design: the launching tool already returned, so only this chain can finish + // the write. Failures reschedule until the bounded delays are exhausted. + void recordAgentWorkflowRunReference({ + workspaceSessionDir: input.workspaceSessionDir, + runId: input.runId, + createdAtMs: input.createdAtMs, + ...(input.afterBoundaryMessageId !== undefined + ? { afterBoundaryMessageId: input.afterBoundaryMessageId } + : {}), + }).catch((error: unknown) => { + log.warn("Agent workflow run reference record retry failed", { + runId: input.runId, + attempt: input.attempt + 1, + error: getErrorMessage(error), + }); + scheduleRecordReferenceRetry({ ...input, attempt: input.attempt + 1 }); + }); + }, delayMs); + timer.unref?.(); +} + /** * Persist agent provenance for a workflow run that outlives the current turn (background * start/resume, or a foreground run that backgrounded itself). TaskService reads these @@ -89,7 +134,8 @@ export async function emitWorkflowRunAttachedEvent(input: { export async function recordBackgroundWorkflowRunReference( config: ToolConfiguration, runId: string, - createdAtMs: number + createdAtMs: number, + retryDelaysMs?: readonly number[] | null ): Promise { const workspaceSessionDir = config.workspaceSessionDir; if (workspaceSessionDir == null || workspaceSessionDir.length === 0) { @@ -132,5 +178,13 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); + scheduleRecordReferenceRetry({ + workspaceSessionDir, + runId, + createdAtMs, + afterBoundaryMessageId, + retryDelaysMs: retryDelaysMs ?? RECORD_REFERENCE_RETRY_DELAYS_MS, + attempt: 0, + }); } } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index daac7889e48..45322ca77af 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6266,6 +6266,67 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a failed reference retirement still emits the committed clear to the renderer", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire-emit"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire-emit", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "hello", { timestamp: 1_000 }) + ); + const sessionAccessor = workspaceService as unknown as { + getOrCreateSession(id: string): { emitChatEvent(message: unknown): void }; + }; + const session = sessionAccessor.getOrCreateSession(workspaceId); + const emitSpy = spyOn(session, "emitChatEvent"); + // A directory at the sidecar path makes retirement fail after the truncation committed. + const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); + await fsPromises.mkdir(sidecarPath); + try { + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(false); + if (!clearResult.success) { + expect(clearResult.error).toContain("workflow run references"); + } + // The transcript is already gone on disk and the deleted sequences cannot be recovered + // by a retry; the renderer must learn about the deletion even though the cleanup error + // aborts the remaining post-clear steps. + expect( + emitSpy.mock.calls.some((call) => (call[0] as { type?: string }).type === "delete") + ).toBe(true); + } finally { + emitSpy.mockRestore(); + await fsPromises.rmdir(sidecarPath); + } + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-coalesced"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index bb95cabd55c..2069613ee69 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12918,6 +12918,28 @@ export class WorkspaceService extends EventEmitter { // r41: the discard is durable — sends that entered before it must not be // admitted afterwards (their content references the discarded context). + // The truncation is committed: any early error return below must first emit the deletion, + // or the renderer keeps showing a transcript that no longer exists on disk (the original + // deletedSequences cannot be recovered by a retry). + const deletedSequences = truncateResult.data; + let deletionsEmitted = false; + const emitDeletedSequences = () => { + if (deletionsEmitted || deletedSequences.length === 0) { + return; + } + deletionsEmitted = true; + const deleteMessage: DeleteMessage = { + type: "delete", + historySequences: deletedSequences, + }; + // Emit through the session so ORPC subscriptions receive the event + if (session) { + session.emitChatEvent(deleteMessage); + } else { + // Fallback to direct emit (legacy path) + this.emit("chat", { workspaceId, message: deleteMessage }); + } + }; if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); // Kernel workflow run references belong to the cleared conversation: a verified-empty @@ -12930,6 +12952,7 @@ export class WorkspaceService extends EventEmitter { try { await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); } catch (error) { + emitDeletedSequences(); return Err( `History was cleared, but stale workflow run references could not be retired ` + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + @@ -12952,20 +12975,7 @@ export class WorkspaceService extends EventEmitter { await clearPendingBranchSummary(workspaceId); } - const deletedSequences = truncateResult.data; - if (deletedSequences.length > 0) { - const deleteMessage: DeleteMessage = { - type: "delete", - historySequences: deletedSequences, - }; - // Emit through the session so ORPC subscriptions receive the event - if (session) { - session.emitChatEvent(deleteMessage); - } else { - // Fallback to direct emit (legacy path) - this.emit("chat", { workspaceId, message: deleteMessage }); - } - } + emitDeletedSequences(); // On full clear, also delete plan file and clear file change tracking if (isFullClear) { From 3eb19a5229be8df68efb4959bd19492eccf433db Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:21:42 +0000 Subject: [PATCH 15/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20close=20round-12=20?= =?UTF-8?q?lifecycle=20gaps=20for=20kernel=20workflow=20wake=20provenance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detached record retries are lifecycle-governed: they only fill absence (onlyIfAbsent), so a later dispatch or workflow_resume record wins, and a full history clear cancels the sidecar path's pending retries (with a registry-identity guard against a raced timer), so a stale retry can neither overwrite newer provenance nor resurrect a retired reference. - Sidecar retirement now happens durably BEFORE the truncation, like the r41 retry discard: no crash window exists in which the transcript is gone but the sidecar survives, and a retirement failure aborts the clear with the transcript intact instead of returning a partial cleanup error after commit. - Terminal-attention drain sends carry an admissionStale probe bound to the context-mutation epoch captured before prompt validation, so a full clear between the currentness check and send admission refuses the stale workflow result instead of injecting it into the cleared conversation; the refused send leaves notifications pending. --- .../agentWorkflowRunReferences.test.ts | 52 +++++++++ .../services/agentWorkflowRunReferences.ts | 102 +++++++++++++++++- src/node/services/taskService.test.ts | 55 ++++++++++ src/node/services/taskService.ts | 16 ++- src/node/services/tools/toolUtils.ts | 57 ++-------- src/node/services/workspaceService.test.ts | 17 +-- src/node/services/workspaceService.ts | 79 +++++++------- 7 files changed, 280 insertions(+), 98 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 69171c081cf..7f5b66a0636 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -5,8 +5,10 @@ import * as path from "node:path"; import { describe, expect, test } from "bun:test"; import { + clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, + scheduleAgentWorkflowRunReferenceRecordRetry, } from "@/node/services/agentWorkflowRunReferences"; describe("agent workflow run references", () => { @@ -190,6 +192,56 @@ describe("agent workflow run references", () => { } }); + test("a pending record retry never overwrites newer provenance", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // The retry carries the stale launch-time snapshot; a workflow_resume records newer + // provenance before the timer fires. Fill-absence semantics must let the newer record win. + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir, + runId: "wfr_lifecycle", + createdAtMs: 1_000, + afterBoundaryMessageId: "stale-row", + retryDelaysMs: [150], + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_lifecycle", + createdAtMs: 2_000, + afterBoundaryMessageId: "resume-row", + }); + await new Promise((resolve) => setTimeout(resolve, 400)); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(1); + expect(references[0]).toMatchObject({ + runId: "wfr_lifecycle", + afterBoundaryMessageId: "resume-row", + }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("a full history clear cancels pending record retries", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A stale detached retry must not resurrect a reference the clear retired; against the + // then decision-free history it would read current and inject the pre-clear result. + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir, + runId: "wfr_cleared", + createdAtMs: 1_000, + afterBoundaryMessageId: null, + retryDelaysMs: [100], + }); + await clearAgentWorkflowRunReferences(workspaceSessionDir); + await new Promise((resolve) => setTimeout(resolve, 350)); + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("self-heals unparseable file contents to empty", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 43a9d23a778..b1f55971d6f 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -5,6 +5,7 @@ import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; +import { log } from "@/node/services/log"; export interface AgentWorkflowRunReference { runId: string; @@ -28,6 +29,26 @@ const MAX_FUTURE_SKEW_MS = 60 * 60_000; const referenceFileLocks = new MutexMap(); +// Detached record retries keyed by sidecar path and runId so lifecycle events can govern them: +// a full clear cancels the path's retries so a stale retry cannot resurrect a retired +// reference, and a retry only ever fills absence (onlyIfAbsent), so it cannot overwrite newer +// provenance recorded by a later dispatch or workflow_resume. +const pendingRecordRetryTimersByPath = new Map< + string, + Map> +>(); + +function cancelPendingRecordRetries(filePath: string): void { + const byRunId = pendingRecordRetryTimersByPath.get(filePath); + if (byRunId == null) { + return; + } + for (const timer of byRunId.values()) { + clearTimeout(timer); + } + pendingRecordRetryTimersByPath.delete(filePath); +} + function referencesPath(workspaceSessionDir: string): string { assert(workspaceSessionDir.length > 0, "agent workflow references require session dir"); return path.join(workspaceSessionDir, AGENT_WORKFLOW_RUN_REFERENCES_FILE); @@ -125,11 +146,13 @@ export async function readAgentWorkflowRunReferences( * appending a reset boundary, which makes a verified-empty (null) boundary snapshot recorded * before the clear indistinguishable from one recorded after it; retiring the references with * the transcript keeps pre-clear workflow results out of the fresh conversation. A post-clear - * workflow_resume re-records provenance. + * workflow_resume re-records provenance. Pending record retries are cancelled first so a stale + * detached retry cannot recreate a retired reference. */ export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { const filePath = referencesPath(workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { + cancelPendingRecordRetries(filePath); await fs.rm(filePath, { force: true }); }); } @@ -139,6 +162,8 @@ export async function recordAgentWorkflowRunReference(input: { runId: string; createdAtMs?: number; afterBoundaryMessageId?: string | null; + /** Fill-absence mode for detached retries: an existing entry (any newer record) wins. */ + onlyIfAbsent?: boolean; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -154,6 +179,9 @@ export async function recordAgentWorkflowRunReference(input: { // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); + if (input.onlyIfAbsent === true && previous != null) { + return; + } byRunId.set(input.runId, { runId: input.runId, // Latest record wins: workflow_resume re-records the reference, and a resume issued after @@ -174,3 +202,75 @@ export async function recordAgentWorkflowRunReference(input: { ); }); } + +// Delays for detached record retries; see scheduleAgentWorkflowRunReferenceRecordRetry. +const RECORD_REFERENCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; + +/** + * Retry a failed provenance record in the background. The launching tool has already returned + * and an untouched active run never hits a natural re-record site, so a single failed write + * would permanently supersede the run's terminal wake once storage recovers. Retries reuse the + * launch-time boundary snapshot, only fill absence (a later successful record wins), and are + * cancelled by a full history clear. + */ +export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { + workspaceSessionDir: string; + runId: string; + createdAtMs: number; + afterBoundaryMessageId?: string | null; + retryDelaysMs?: readonly number[] | null; + attempt?: number; +}): void { + const retryDelaysMs = input.retryDelaysMs ?? RECORD_REFERENCE_RETRY_DELAYS_MS; + const attempt = input.attempt ?? 0; + const delayMs = retryDelaysMs[attempt]; + if (delayMs == null) { + log.error("Giving up on agent workflow run reference record after retries", { + runId: input.runId, + attempts: attempt, + }); + return; + } + const filePath = referencesPath(input.workspaceSessionDir); + const timer = setTimeout(() => { + const byRunId = pendingRecordRetryTimersByPath.get(filePath); + // clearTimeout cannot stop a callback Node already dequeued; registry identity is the + // authoritative cancellation signal, so a cancelled-but-raced retry aborts here. + if (byRunId?.get(input.runId) !== timer) { + return; + } + byRunId.delete(input.runId); + if (byRunId.size === 0) { + pendingRecordRetryTimersByPath.delete(filePath); + } + // Detached by design: the launching tool already returned, so only this chain can finish + // the write. Failures reschedule until the bounded delays are exhausted. + void recordAgentWorkflowRunReference({ + workspaceSessionDir: input.workspaceSessionDir, + runId: input.runId, + createdAtMs: input.createdAtMs, + onlyIfAbsent: true, + ...(input.afterBoundaryMessageId !== undefined + ? { afterBoundaryMessageId: input.afterBoundaryMessageId } + : {}), + }).catch((error: unknown) => { + log.warn("Agent workflow run reference record retry failed", { + runId: input.runId, + attempt: attempt + 1, + error, + }); + scheduleAgentWorkflowRunReferenceRecordRetry({ ...input, attempt: attempt + 1 }); + }); + }, delayMs); + timer.unref?.(); + let byRunId = pendingRecordRetryTimersByPath.get(filePath); + if (byRunId == null) { + byRunId = new Map(); + pendingRecordRetryTimersByPath.set(filePath, byRunId); + } + const previousTimer = byRunId.get(input.runId); + if (previousTimer != null) { + clearTimeout(previousTimer); + } + byRunId.set(input.runId, timer); +} diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 23b0f450909..f652dd36f46 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -554,6 +554,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled: ReturnType; emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; + getContextMutationEpoch: ReturnType; create: ReturnType; countQueuedAgentPeerMessages: ReturnType; }> @@ -591,6 +592,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled: ReturnType; emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; + getContextMutationEpoch: ReturnType; create: ReturnType; } { const sendMessage = @@ -660,6 +662,7 @@ function createWorkspaceServiceMocks( const updateAgentStatus = overrides?.updateAgentStatus ?? mock((): Promise => Promise.resolve()); const isExperimentEnabled = overrides?.isExperimentEnabled ?? mock(() => false); + const getContextMutationEpoch = overrides?.getContextMutationEpoch ?? mock(() => 0); const emitChatEvent = overrides?.emitChatEvent ?? mock((_workspaceId: string, _message: WorkspaceChatMessage) => undefined); @@ -736,6 +739,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, + getContextMutationEpoch, getWorkflowInvocationCurrentness, countQueuedAgentPeerMessages, } as unknown as WorkspaceService, @@ -772,6 +776,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, + getContextMutationEpoch, }; } @@ -6355,6 +6360,56 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("drain sends carry a staleness probe that trips after a full clear", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_admission_stale"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + // The prompt is validated against history before the send is admitted; a full clear in + // that window advances the context-mutation epoch. The probe handed to sendMessage must + // observe the live epoch so admission can refuse the stale prompt. + let epoch = 1; + const { workspaceService } = createWorkspaceServiceMocks({ + sendMessage, + getContextMutationEpoch: mock(() => epoch), + }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const internal = sendMessage.mock.calls[0]?.[3] as { admissionStale?: () => boolean }; + expect(typeof internal.admissionStale).toBe("function"); + expect(internal.admissionStale?.()).toBe(false); + epoch = 2; + expect(internal.admissionStale?.()).toBe(true); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 1bc7797e56b..27773cfb84f 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8288,6 +8288,13 @@ export class TaskService { isPersistentChildContinuation ? record.workspaceId : notification.sourceId ); } + // Workflow prompts are validated against history well before the send is admitted; a full + // clear in that window truncates history and retires the sidecar, so the send must refuse + // (admissionStale) rather than inject the stale result into the freshly cleared + // conversation. A refused send leaves the notifications pending for the next drain. + const admissionEpoch = this.workspaceService.getContextMutationEpoch(ownerWorkspaceId); + const sendAdmissionStale = () => + this.workspaceService.getContextMutationEpoch(ownerWorkspaceId) !== admissionEpoch; const workflowNotifications = pending.filter( (notification) => notification.sourceKind === "workflow_run" ); @@ -8394,7 +8401,13 @@ export class TaskService { prompt, sendOptions, // Synthetic, idle-only auto-resume — same flags as the active-work auto-resume path. - { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, requireIdle: true } + { + skipAutoResumeReset: true, + synthetic: true, + agentInitiated: true, + requireIdle: true, + admissionStale: sendAdmissionStale, + } ); if (!sendResult.success && isWorkspaceBusyIdleOnlySend(sendResult.error)) { @@ -8416,6 +8429,7 @@ export class TaskService { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, + admissionStale: sendAdmissionStale, onCanceled: () => { this.scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId); }, diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index bf90c448775..31e423cbffd 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -7,7 +7,10 @@ import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkspaceChatMessage } from "@/common/orpc/types"; import type { ToolConfiguration } from "@/common/utils/tools/tools"; -import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; +import { + recordAgentWorkflowRunReference, + scheduleAgentWorkflowRunReferenceRecordRetry, +} from "@/node/services/agentWorkflowRunReferences"; import { log } from "@/node/services/log"; import type { TaskService } from "@/node/services/taskService"; @@ -79,51 +82,6 @@ export async function emitWorkflowRunAttachedEvent(input: { await input.config.emitChatEvent(event); } -// Bounded background retries for a transiently unreadable sidecar. The only natural re-record -// sites are a new dispatch and workflow_resume, which an untouched active run never hits, so -// giving up after one failed write would permanently supersede the run's terminal wake once -// storage recovers. Retries reuse the launch-time boundary snapshot: provenance describes the -// launch, not the retry moment. -const RECORD_REFERENCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; - -function scheduleRecordReferenceRetry(input: { - workspaceSessionDir: string; - runId: string; - createdAtMs: number; - afterBoundaryMessageId: string | null | undefined; - retryDelaysMs: readonly number[]; - attempt: number; -}): void { - const delayMs = input.retryDelaysMs[input.attempt]; - if (delayMs == null) { - log.error("Giving up on agent workflow run reference record after retries", { - runId: input.runId, - attempts: input.attempt, - }); - return; - } - const timer = setTimeout(() => { - // Detached by design: the launching tool already returned, so only this chain can finish - // the write. Failures reschedule until the bounded delays are exhausted. - void recordAgentWorkflowRunReference({ - workspaceSessionDir: input.workspaceSessionDir, - runId: input.runId, - createdAtMs: input.createdAtMs, - ...(input.afterBoundaryMessageId !== undefined - ? { afterBoundaryMessageId: input.afterBoundaryMessageId } - : {}), - }).catch((error: unknown) => { - log.warn("Agent workflow run reference record retry failed", { - runId: input.runId, - attempt: input.attempt + 1, - error: getErrorMessage(error), - }); - scheduleRecordReferenceRetry({ ...input, attempt: input.attempt + 1 }); - }); - }, delayMs); - timer.unref?.(); -} - /** * Persist agent provenance for a workflow run that outlives the current turn (background * start/resume, or a foreground run that backgrounded itself). TaskService reads these @@ -178,13 +136,12 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); - scheduleRecordReferenceRetry({ + scheduleAgentWorkflowRunReferenceRecordRetry({ workspaceSessionDir, runId, createdAtMs, - afterBoundaryMessageId, - retryDelaysMs: retryDelaysMs ?? RECORD_REFERENCE_RETRY_DELAYS_MS, - attempt: 0, + retryDelaysMs, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), }); } } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 45322ca77af..176b45d413d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6266,7 +6266,7 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("a failed reference retirement still emits the committed clear to the renderer", async () => { + test("a failed reference retirement aborts the clear before deleting history", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-retire-emit"; const projectPath = path.join(config.rootDir, "project"); @@ -6302,7 +6302,10 @@ describe("WorkspaceService workflow invocation events", () => { }; const session = sessionAccessor.getOrCreateSession(workspaceId); const emitSpy = spyOn(session, "emitChatEvent"); - // A directory at the sidecar path makes retirement fail after the truncation committed. + // A directory at the sidecar path makes retirement fail. Retirement runs BEFORE the + // truncation, so the failure must abort the whole clear: the transcript survives, the + // renderer sees no deletion, and no crash window exists in which the transcript is gone + // while the sidecar lives on. const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); await fsPromises.mkdir(sidecarPath); try { @@ -6311,12 +6314,14 @@ describe("WorkspaceService workflow invocation events", () => { if (!clearResult.success) { expect(clearResult.error).toContain("workflow run references"); } - // The transcript is already gone on disk and the deleted sequences cannot be recovered - // by a retry; the renderer must learn about the deletion even though the cleanup error - // aborts the remaining post-clear steps. expect( emitSpy.mock.calls.some((call) => (call[0] as { type?: string }).type === "delete") - ).toBe(true); + ).toBe(false); + const survivingHistory = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(survivingHistory.success).toBe(true); + if (survivingHistory.success) { + expect(survivingHistory.data).toHaveLength(1); + } } finally { emitSpy.mockRestore(); await fsPromises.rmdir(sidecarPath); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2069613ee69..da118d47116 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3806,6 +3806,14 @@ export class WorkspaceService extends EventEmitter { } } + /** + * Current context-mutation epoch (see contextMutationEpochs). Lets the terminal-attention + * drain refuse a send whose workflow prompt was validated before a full clear committed. + */ + getContextMutationEpoch(workspaceId: string): number { + return this.contextMutationEpochs.get(workspaceId) ?? 0; + } + /** r41: mark a context-discarding mutation as durably committed (see contextMutationEpochs). */ private advanceContextMutationEpoch(workspaceId: string): void { this.contextMutationEpochs.set( @@ -12902,6 +12910,23 @@ export class WorkspaceService extends EventEmitter { ); } } + // Kernel workflow run references belong to the transcript being discarded: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one recorded + // after it. Retire them durably BEFORE the truncation (like the r41 retry discard above) so + // no crash window exists in which the transcript is gone but the sidecar survives; a crash + // here can only lose a wake for a still-intact conversation, never inject a pre-clear + // result into the cleared one. A post-clear resume re-records provenance, and pending + // record retries are cancelled with the sidecar. + if (isFullClear) { + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `Cannot clear history: stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}); retry once the session storage is writable.` + ); + } + } if (effectivePercentage > 0) { session?.clearUsageState(); } @@ -12918,47 +12943,8 @@ export class WorkspaceService extends EventEmitter { // r41: the discard is durable — sends that entered before it must not be // admitted afterwards (their content references the discarded context). - // The truncation is committed: any early error return below must first emit the deletion, - // or the renderer keeps showing a transcript that no longer exists on disk (the original - // deletedSequences cannot be recovered by a retry). - const deletedSequences = truncateResult.data; - let deletionsEmitted = false; - const emitDeletedSequences = () => { - if (deletionsEmitted || deletedSequences.length === 0) { - return; - } - deletionsEmitted = true; - const deleteMessage: DeleteMessage = { - type: "delete", - historySequences: deletedSequences, - }; - // Emit through the session so ORPC subscriptions receive the event - if (session) { - session.emitChatEvent(deleteMessage); - } else { - // Fallback to direct emit (legacy path) - this.emit("chat", { workspaceId, message: deleteMessage }); - } - }; if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); - // Kernel workflow run references belong to the cleared conversation: a verified-empty - // (null) boundary snapshot recorded before the clear is indistinguishable from one - // recorded after it, so a surviving reference could inject a pre-clear workflow result - // into the fresh conversation. Retire them immediately after the truncation commits, - // before any later post-clear step that can fail and return early (goal acknowledgment, - // carryover discard), or the stale reference would survive the committed clear. A - // post-clear resume re-records provenance. - try { - await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error) { - emitDeletedSequences(); - return Err( - `History was cleared, but stale workflow run references could not be retired ` + - `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + - `result into the cleared conversation; retry once the session storage is writable.` - ); - } } // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the @@ -12975,7 +12961,20 @@ export class WorkspaceService extends EventEmitter { await clearPendingBranchSummary(workspaceId); } - emitDeletedSequences(); + const deletedSequences = truncateResult.data; + if (deletedSequences.length > 0) { + const deleteMessage: DeleteMessage = { + type: "delete", + historySequences: deletedSequences, + }; + // Emit through the session so ORPC subscriptions receive the event + if (session) { + session.emitChatEvent(deleteMessage); + } else { + // Fallback to direct emit (legacy path) + this.emit("chat", { workspaceId, message: deleteMessage }); + } + } // On full clear, also delete plan file and clear file change tracking if (isFullClear) { From 62dd29073a7982f7110aae0925807f5b5624e0cf Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:44:16 +0000 Subject: [PATCH 16/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20round-13=20sidecar?= =?UTF-8?q?=20lifecycle=20hardening:=20boundary=20repair,=20removal=20drai?= =?UTF-8?q?n,=20corrupt-dir=20self-heal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A failed record-time boundary snapshot on a decision-free launch now schedules a bounded background repair: if history is still verified empty at repair time it persists the null snapshot the decision-free currentness branch requires (rows cannot disappear outside a full clear, which retires the sidecar), while a decision row seen at repair time keeps the entry boundary-less and fail safe because it may postdate the launch. - Workspace removal cancels and drains detached sidecar maintenance before deleting the session directory, so a late retry cannot mkdir it back into existence; the registry now tracks in-flight writes and the full clear drains through the same path. - clearAgentWorkflowRunReferences removes a directory at the known sidecar path recursively: force alone refuses directories, which previously failed every subsequent full clear identically with no self-heal. --- .../services/agentWorkflowRunReferences.ts | 151 +++++++++++++----- src/node/services/tools/toolUtils.test.ts | 77 +++++++++ src/node/services/tools/toolUtils.ts | 75 +++++++++ src/node/services/workspaceRemoval.test.ts | 34 ++++ src/node/services/workspaceRemoval.ts | 5 + src/node/services/workspaceService.test.ts | 72 ++++++++- 6 files changed, 370 insertions(+), 44 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index b1f55971d6f..9ca4544cece 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -29,24 +29,101 @@ const MAX_FUTURE_SKEW_MS = 60 * 60_000; const referenceFileLocks = new MutexMap(); -// Detached record retries keyed by sidecar path and runId so lifecycle events can govern them: -// a full clear cancels the path's retries so a stale retry cannot resurrect a retired -// reference, and a retry only ever fills absence (onlyIfAbsent), so it cannot overwrite newer -// provenance recorded by a later dispatch or workflow_resume. -const pendingRecordRetryTimersByPath = new Map< +// Detached sidecar maintenance (record retries, boundary-snapshot repairs) keyed by sidecar +// path and a per-run key so lifecycle events can govern it: a full clear or workspace removal +// cancels the path's timers and drains in-flight writes, so stale maintenance can neither +// resurrect a retired reference nor recreate a deleted session directory, and maintenance +// writes only ever fill gaps (onlyIfAbsent / onlyIfBoundaryAbsent), so they cannot overwrite +// newer provenance recorded by a later dispatch or workflow_resume. +const pendingSidecarMaintenanceTimersByPath = new Map< string, Map> >(); +const inFlightSidecarMaintenanceByPath = new Map>>(); -function cancelPendingRecordRetries(filePath: string): void { - const byRunId = pendingRecordRetryTimersByPath.get(filePath); - if (byRunId == null) { - return; +/** Arm a maintenance timer; a newer schedule for the same key supersedes the older timer. */ +export function registerSidecarMaintenanceTimer( + workspaceSessionDir: string, + key: string, + timer: ReturnType +): void { + const filePath = referencesPath(workspaceSessionDir); + let byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); + if (byKey == null) { + byKey = new Map(); + pendingSidecarMaintenanceTimersByPath.set(filePath, byKey); + } + const previous = byKey.get(key); + if (previous != null) { + clearTimeout(previous); + } + byKey.set(key, timer); +} + +/** + * Consume a fired maintenance timer. clearTimeout cannot stop a callback Node already + * dequeued, so registry identity is the authoritative cancellation signal: a + * cancelled-but-raced callback sees a mismatch and must abort. + */ +export function takeSidecarMaintenanceTimer( + workspaceSessionDir: string, + key: string, + timer: ReturnType +): boolean { + const filePath = referencesPath(workspaceSessionDir); + const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); + if (byKey?.get(key) !== timer) { + return false; } - for (const timer of byRunId.values()) { - clearTimeout(timer); + byKey.delete(key); + if (byKey.size === 0) { + pendingSidecarMaintenanceTimersByPath.delete(filePath); + } + return true; +} + +/** Track a maintenance write so cancelAgentWorkflowRunReferenceMaintenance can drain it. */ +export function trackSidecarMaintenanceWrite( + workspaceSessionDir: string, + work: Promise +): void { + const filePath = referencesPath(workspaceSessionDir); + let inFlight = inFlightSidecarMaintenanceByPath.get(filePath); + if (inFlight == null) { + inFlight = new Set(); + inFlightSidecarMaintenanceByPath.set(filePath, inFlight); + } + inFlight.add(work); + const remove = () => { + inFlight.delete(work); + if (inFlight.size === 0) { + inFlightSidecarMaintenanceByPath.delete(filePath); + } + }; + work.then(remove, remove); +} + +/** + * Cancel this sidecar's pending maintenance timers and drain writes already past their + * identity check, so a full clear or workspace removal cannot race a recreation of the file + * or the session directory it is about to delete. Must run BEFORE taking the sidecar file + * lock: an in-flight write acquires that same lock, so draining inside it would deadlock. + */ +export async function cancelAgentWorkflowRunReferenceMaintenance( + workspaceSessionDir: string +): Promise { + const filePath = referencesPath(workspaceSessionDir); + const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); + if (byKey != null) { + for (const timer of byKey.values()) { + clearTimeout(timer); + } + pendingSidecarMaintenanceTimersByPath.delete(filePath); + } + const inFlight = inFlightSidecarMaintenanceByPath.get(filePath); + if (inFlight != null && inFlight.size > 0) { + await Promise.allSettled([...inFlight]); } - pendingRecordRetryTimersByPath.delete(filePath); } function referencesPath(workspaceSessionDir: string): string { @@ -151,9 +228,12 @@ export async function readAgentWorkflowRunReferences( */ export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { const filePath = referencesPath(workspaceSessionDir); + await cancelAgentWorkflowRunReferenceMaintenance(workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { - cancelPendingRecordRetries(filePath); - await fs.rm(filePath, { force: true }); + // recursive: a directory at this known sidecar path is corruption (it also fails reads + // with EISDIR), and force alone refuses to remove directories, which would fail every + // subsequent full clear identically. Removing it self-heals the workspace. + await fs.rm(filePath, { force: true, recursive: true }); }); } @@ -164,6 +244,12 @@ export async function recordAgentWorkflowRunReference(input: { afterBoundaryMessageId?: string | null; /** Fill-absence mode for detached retries: an existing entry (any newer record) wins. */ onlyIfAbsent?: boolean; + /** + * Boundary-repair mode: only patch an entry that exists and still lacks a boundary + * snapshot. A missing entry means the reference was retired (clear/removal) and must not be + * resurrected; a present boundary means newer provenance already landed. + */ + onlyIfBoundaryAbsent?: boolean; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -182,6 +268,12 @@ export async function recordAgentWorkflowRunReference(input: { if (input.onlyIfAbsent === true && previous != null) { return; } + if ( + input.onlyIfBoundaryAbsent === true && + (previous == null || previous.afterBoundaryMessageId !== undefined) + ) { + return; + } byRunId.set(input.runId, { runId: input.runId, // Latest record wins: workflow_resume re-records the reference, and a resume issued after @@ -203,8 +295,8 @@ export async function recordAgentWorkflowRunReference(input: { }); } -// Delays for detached record retries; see scheduleAgentWorkflowRunReferenceRecordRetry. -const RECORD_REFERENCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; +// Delays for detached sidecar maintenance (record retries, boundary-snapshot repairs). +export const SIDECAR_MAINTENANCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; /** * Retry a failed provenance record in the background. The launching tool has already returned @@ -221,7 +313,7 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { retryDelaysMs?: readonly number[] | null; attempt?: number; }): void { - const retryDelaysMs = input.retryDelaysMs ?? RECORD_REFERENCE_RETRY_DELAYS_MS; + const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; const attempt = input.attempt ?? 0; const delayMs = retryDelaysMs[attempt]; if (delayMs == null) { @@ -231,21 +323,14 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { }); return; } - const filePath = referencesPath(input.workspaceSessionDir); + const key = `record:${input.runId}`; const timer = setTimeout(() => { - const byRunId = pendingRecordRetryTimersByPath.get(filePath); - // clearTimeout cannot stop a callback Node already dequeued; registry identity is the - // authoritative cancellation signal, so a cancelled-but-raced retry aborts here. - if (byRunId?.get(input.runId) !== timer) { + if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { return; } - byRunId.delete(input.runId); - if (byRunId.size === 0) { - pendingRecordRetryTimersByPath.delete(filePath); - } // Detached by design: the launching tool already returned, so only this chain can finish // the write. Failures reschedule until the bounded delays are exhausted. - void recordAgentWorkflowRunReference({ + const work = recordAgentWorkflowRunReference({ workspaceSessionDir: input.workspaceSessionDir, runId: input.runId, createdAtMs: input.createdAtMs, @@ -261,16 +346,8 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { }); scheduleAgentWorkflowRunReferenceRecordRetry({ ...input, attempt: attempt + 1 }); }); + trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); }, delayMs); timer.unref?.(); - let byRunId = pendingRecordRetryTimersByPath.get(filePath); - if (byRunId == null) { - byRunId = new Map(); - pendingRecordRetryTimersByPath.set(filePath, byRunId); - } - const previousTimer = byRunId.get(input.runId); - if (previousTimer != null) { - clearTimeout(previousTimer); - } - byRunId.set(input.runId, timer); + registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer); } diff --git a/src/node/services/tools/toolUtils.test.ts b/src/node/services/tools/toolUtils.test.ts index 7552b08a7da..538c2c249a5 100644 --- a/src/node/services/tools/toolUtils.test.ts +++ b/src/node/services/tools/toolUtils.test.ts @@ -49,4 +49,81 @@ describe("recordBackgroundWorkflowRunReference", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); + + test("repairs a verified-empty boundary snapshot after a transient read failure", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); + try { + // Launch from a decision-free history whose boundary read fails once: the rediscovery + // entry lands boundary-less, and only the repair can restore the verified-empty (null) + // snapshot the decision-free currentness branch requires. + let calls = 0; + const taskService = { + getWorkflowInvocationBoundaryMessageId: (): Promise => { + calls += 1; + return calls === 1 + ? Promise.reject(new Error("history unavailable")) + : Promise.resolve(null); + }, + }; + await recordBackgroundWorkflowRunReference( + { + workspaceSessionDir, + workspaceId: "ws-boundary-repair", + taskService, + } as unknown as ToolConfiguration, + "wfr_boundary_repair", + 2_000, + [25, 25, 25] + ); + let reference: { afterBoundaryMessageId?: string | null } | undefined; + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + [reference] = await readAgentWorkflowRunReferences(workspaceSessionDir); + if (reference?.afterBoundaryMessageId === null) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(reference).toMatchObject({ + runId: "wfr_boundary_repair", + afterBoundaryMessageId: null, + }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("keeps the entry boundary-less when a decision row exists at repair time", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); + try { + // A decision row seen at repair time may postdate the launch; persisting it would + // overclaim currentness, so the entry must stay boundary-less and fail safe. + let calls = 0; + const taskService = { + getWorkflowInvocationBoundaryMessageId: (): Promise => { + calls += 1; + return calls === 1 + ? Promise.reject(new Error("history unavailable")) + : Promise.resolve("manual-user"); + }, + }; + await recordBackgroundWorkflowRunReference( + { + workspaceSessionDir, + workspaceId: "ws-boundary-unsafe", + taskService, + } as unknown as ToolConfiguration, + "wfr_boundary_unsafe", + 2_000, + [25] + ); + await new Promise((resolve) => setTimeout(resolve, 300)); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(1); + expect(references[0]?.runId).toBe("wfr_boundary_unsafe"); + expect(references[0]?.afterBoundaryMessageId).toBeUndefined(); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 31e423cbffd..95b3c8355af 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -8,8 +8,12 @@ import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkspaceChatMessage } from "@/common/orpc/types"; import type { ToolConfiguration } from "@/common/utils/tools/tools"; import { + SIDECAR_MAINTENANCE_RETRY_DELAYS_MS, recordAgentWorkflowRunReference, + registerSidecarMaintenanceTimer, scheduleAgentWorkflowRunReferenceRecordRetry, + takeSidecarMaintenanceTimer, + trackSidecarMaintenanceWrite, } from "@/node/services/agentWorkflowRunReferences"; import { log } from "@/node/services/log"; import type { TaskService } from "@/node/services/taskService"; @@ -82,6 +86,65 @@ export async function emitWorkflowRunAttachedEvent(input: { await input.config.emitChatEvent(event); } +/** + * Repair a missing boundary snapshot in the background. A kernel launch from a decision-free + * history whose record-time boundary read failed persists a boundary-less entry, but the + * decision-free currentness branch accepts only an explicit verified-empty (null) snapshot, + * so without repair the run's terminal wake is permanently superseded once storage recovers. + * Rows never disappear outside a full clear (which retires the sidecar), so a history still + * verified-empty at repair time was also empty at launch and null is faithful launch + * provenance; a decision row seen at repair time may postdate the launch, so persisting it + * would overclaim currentness and the entry stays boundary-less (fail safe). + */ +function scheduleBoundarySnapshotRepair(input: { + workspaceSessionDir: string; + runId: string; + createdAtMs: number; + getBoundary: () => Promise; + retryDelaysMs?: readonly number[] | null; + attempt?: number; +}): void { + const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; + const attempt = input.attempt ?? 0; + const delayMs = retryDelaysMs[attempt]; + if (delayMs == null) { + log.error("Giving up on workflow boundary snapshot repair after retries", { + runId: input.runId, + attempts: attempt, + }); + return; + } + const key = `boundary:${input.runId}`; + const timer = setTimeout(() => { + if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { + return; + } + const work = (async () => { + const boundary = await input.getBoundary(); + if (boundary !== null) { + return; + } + await recordAgentWorkflowRunReference({ + workspaceSessionDir: input.workspaceSessionDir, + runId: input.runId, + createdAtMs: input.createdAtMs, + afterBoundaryMessageId: null, + onlyIfBoundaryAbsent: true, + }); + })().catch((error: unknown) => { + log.warn("Workflow boundary snapshot repair failed", { + runId: input.runId, + attempt: attempt + 1, + error: getErrorMessage(error), + }); + scheduleBoundarySnapshotRepair({ ...input, attempt: attempt + 1 }); + }); + trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); + }, delayMs); + timer.unref?.(); + registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer); +} + /** * Persist agent provenance for a workflow run that outlives the current turn (background * start/resume, or a foreground run that backgrounded itself). TaskService reads these @@ -121,6 +184,18 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); + const workspaceId = config.workspaceId; + const getBoundaryMessageId = + taskService?.getWorkflowInvocationBoundaryMessageId?.bind(taskService); + if (workspaceId != null && getBoundaryMessageId != null) { + scheduleBoundarySnapshotRepair({ + workspaceSessionDir, + runId, + createdAtMs, + getBoundary: () => getBoundaryMessageId(workspaceId, runId), + retryDelaysMs, + }); + } } } diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index 970ccea6d11..e440984e50b 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -8,6 +8,7 @@ import { targetMutationLockFilePath, withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; +import { scheduleAgentWorkflowRunReferenceRecordRetry } from "@/node/services/agentWorkflowRunReferences"; import { acquireProcessFileLock, getProcessBirth } from "@/node/utils/concurrency/fileLock"; import { healRemovalTombstonesForRegisteredWorkspaces, @@ -21,6 +22,39 @@ import { } from "./workspaceRemoval"; describe("workspaceRemoval", () => { + test("removal cancels pending sidecar record retries so they cannot recreate the session dir", async () => { + using tmp = new DisposableTempDir("workspace-removal-sidecar"); + const rootDir = path.join(tmp.path, "xum-home"); + const workspaceId = "ws-removal-sidecar"; + const sessionDir = path.join(rootDir, "sessions", workspaceId); + await fsPromises.mkdir(sessionDir, { recursive: true }); + + // A detached provenance retry armed before removal; on fire it would mkdir the session + // directory back into existence after the deletion below. + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir: sessionDir, + runId: "wfr_removal_race", + createdAtMs: 1_000, + afterBoundaryMessageId: null, + retryDelaysMs: [100], + }); + + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir, + workspaceId, + attemptId: "test-attempt-sidecar", + }); + await new Promise((resolve) => setTimeout(resolve, 350)); + + expect( + await fsPromises.access(sessionDir).then( + () => true, + () => false + ) + ).toBe(false); + }); + test("deletion waits for a live memory writer, then tombstones and deletes (r61)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index 154c7311626..c2fe3b8c4d5 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -27,6 +27,7 @@ * foreign backend's still-running consolidation refuse arbitrarily late. */ +import { cancelAgentWorkflowRunReferenceMaintenance } from "@/node/services/agentWorkflowRunReferences"; import crypto from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; @@ -217,6 +218,10 @@ export async function removeSessionDirUnderMemoryLocks(args: { // deleted directory cannot be recreated by a late mutation or // journal append. await publishTombstone(); + // Detached sidecar maintenance (provenance record retries) would survive removal and + // recreate the deleted session directory when its timer fires; cancel and drain it + // like the other session writers before the directory goes away. + await cancelAgentWorkflowRunReferenceMaintenance(args.sessionDir); await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); } ); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 176b45d413d..4994dcc6169 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6302,12 +6302,18 @@ describe("WorkspaceService workflow invocation events", () => { }; const session = sessionAccessor.getOrCreateSession(workspaceId); const emitSpy = spyOn(session, "emitChatEvent"); - // A directory at the sidecar path makes retirement fail. Retirement runs BEFORE the - // truncation, so the failure must abort the whole clear: the transcript survives, the - // renderer sees no deletion, and no crash window exists in which the transcript is gone - // while the sidecar lives on. - const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); - await fsPromises.mkdir(sidecarPath); + // A read-only session directory makes retirement fail (a directory at the sidecar path + // now self-heals instead). Retirement runs BEFORE the truncation, so the failure must + // abort the whole clear: the transcript survives, the renderer sees no deletion, and no + // crash window exists in which the transcript is gone while the sidecar lives on. + const sessionDir = config.getSessionDir(workspaceId); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: sessionDir, + runId: "wfr_retirement_blocked", + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + await fsPromises.chmod(sessionDir, 0o555); try { const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); expect(clearResult.success).toBe(false); @@ -6324,7 +6330,7 @@ describe("WorkspaceService workflow invocation events", () => { } } finally { emitSpy.mockRestore(); - await fsPromises.rmdir(sidecarPath); + await fsPromises.chmod(sessionDir, 0o755); } workspaceService.disposeSession(workspaceId); } finally { @@ -6414,6 +6420,58 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a corrupt sidecar directory self-heals during a full clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-selfheal"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-selfheal", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "hello", { timestamp: 1_000 }) + ); + // A directory at the known sidecar path is corruption (reads fail with EISDIR). The + // full clear must remove it and succeed, not fail identically on every retry and leave + // the workspace impossible to clear without manual session-storage repair. + const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); + await fsPromises.mkdir(sidecarPath); + await fsPromises.writeFile(path.join(sidecarPath, "junk.txt"), "junk"); + + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(true); + expect( + await fsPromises.access(sidecarPath).then( + () => true, + () => false + ) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; From 6866c5bcb2ac06a61b0aa31f023c1bd2f462495a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:17:12 +0000 Subject: [PATCH 17/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20round-14=20provenan?= =?UTF-8?q?ce=20integrity:=20supersede-older=20retries,=20generation=20lat?= =?UTF-8?q?ch,=20repair=20ordering,=20sidecar-gated=20finalization,=20pers?= =?UTF-8?q?isted=20drain=20provenance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detached record retries skip only when a STRICTLY NEWER record exists, so a failed workflow_resume re-record can supersede the stale dispatch entry it was meant to replace while a newer successful record still wins. - Cancellation bumps a per-path lifecycle generation before clearing timers; registration refuses stale generations, so a retry rescheduled from a failing write's catch handler during the cancellation drain cannot re-arm and recreate retired state. - Boundary repair treats a missing reference as retryable (the record retry lands it later) instead of a satisfied no-op that would leave the eventual entry permanently boundary-less. - listAgentReferencedWorkflowRunIds propagates an unreadable sidecar; the three completion-gating callers defer (assume blockers, skip stream-end reconciliation, keep the task running) instead of finalizing while a kernel workflow may still run. - The drain persists deliveredWorkflowRunIds onto the accepted row (MuxMetadata) and currentness recognizes consumption from that provenance instead of row text, which user-controlled synthetic content (e.g. a heartbeat body) could spoof to suppress a real wake. --- src/common/types/message.ts | 7 ++ src/common/utils/workflowRunMessages.ts | 35 -------- src/node/services/agentSession.ts | 7 ++ .../agentWorkflowRunReferences.test.ts | 60 ++++++++++++++ .../services/agentWorkflowRunReferences.ts | 50 ++++++++++-- src/node/services/messageQueue.ts | 2 + src/node/services/taskService.test.ts | 31 +++++++- src/node/services/taskService.ts | 79 ++++++++++++++----- src/node/services/tools/toolUtils.test.ts | 54 +++++++++++++ src/node/services/tools/toolUtils.ts | 22 +++++- src/node/services/workspaceService.test.ts | 47 +++++------ src/node/services/workspaceService.ts | 24 +++--- 12 files changed, 318 insertions(+), 100 deletions(-) diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 9b16234a106..2ad5017f66a 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -868,6 +868,13 @@ export interface ModelFallbackRecord { // Our custom metadata type export interface MuxMetadata { + /** + * Workflow run IDs whose terminal results this synthetic user row delivered (set by the + * terminal-attention drain). Currentness checks treat the row as consumption for these runs; + * persisted provenance, not text recognition, so repo/user-controlled content quoting a run + * ID (e.g. a heartbeat body) cannot spoof consumption and suppress a real wake. + */ + deliveredWorkflowRunIds?: string[]; /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) diff --git a/src/common/utils/workflowRunMessages.ts b/src/common/utils/workflowRunMessages.ts index 66a2b3b5fa9..9fc393e9f7e 100644 --- a/src/common/utils/workflowRunMessages.ts +++ b/src/common/utils/workflowRunMessages.ts @@ -190,41 +190,6 @@ export function buildWorkflowResultContextMessage(input: { ].join("\n\n"); } -/** - * Recognize this run's result payload inside a coalesced terminal-attention prompt from the - * builder's own output format. The drain's synthetic user row can coalesce several runs into - * one message and carries no workflow-result metadata, so currentness checks must read the - * consumption evidence out of the text: each payload block is parsed back and matched on the - * exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted - * inside another run's report cannot count as consumption. - */ -export function textContainsWorkflowResultPayload(text: string, runId: string): boolean { - assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required"); - if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) { - return false; - } - const blockPattern = new RegExp( - `<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n`, - "g" - ); - for (const match of text.matchAll(blockPattern)) { - let payload: unknown; - try { - payload = JSON.parse(match[1] ?? ""); - } catch { - continue; - } - if (!isRecordValue(payload)) { - continue; - } - const workflow = payload.workflow; - if (isRecordValue(workflow) && workflow.runId === runId) { - return true; - } - } - return false; -} - export interface WorkflowRunCardInput { scriptPath?: string; scriptSource?: string; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4422a49a05e..aba92a18eeb 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -2950,6 +2950,8 @@ export class AgentSession { internal?: { synthetic?: boolean; agentInitiated?: boolean; + /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ + deliveredWorkflowRunIds?: string[]; goalContinuation?: boolean; goalKind?: GoalSyntheticMessageKind; /** Goal identity persisted alongside goalKind so chat-tail reconciliation can scope the row. */ @@ -3545,6 +3547,9 @@ export class AgentSession { ...(internal?.enqueuedAtMs != null ? { enqueuedAtMs: internal.enqueuedAtMs } : {}), // Auto-resume and other system-generated messages are synthetic + UI-visible ...(internal?.synthetic && { synthetic: true, uiVisible: true }), + ...(internal?.deliveredWorkflowRunIds != null && internal.deliveredWorkflowRunIds.length > 0 + ? { deliveredWorkflowRunIds: internal.deliveredWorkflowRunIds } + : {}), }, additionalParts ); @@ -6390,6 +6395,8 @@ export class AgentSession { internal?: { synthetic?: boolean; agentInitiated?: boolean; + /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ + deliveredWorkflowRunIds?: string[]; /** Request-entry authoring time captured before send preflight awaits (see MessageQueue). */ authoredAtMs?: number; /** True only for a report that continues an existing workspace turn. */ diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 7f5b66a0636..30769c481ac 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -6,6 +6,7 @@ import { describe, expect, test } from "bun:test"; import { clearAgentWorkflowRunReferences, + getSidecarLifecycleGeneration, readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, scheduleAgentWorkflowRunReferenceRecordRetry, @@ -222,6 +223,65 @@ describe("agent workflow run references", () => { } }); + test("a retry supersedes an older entry for the same run", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A workflow_resume re-record fails transiently while an OLDER dispatch entry exists. + // The retry must replace that stale provenance (its boundary predates the resume) or the + // resumed run's wake is classified not_current; only a strictly newer record wins. + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_supersede", + createdAtMs: 500, + afterBoundaryMessageId: "old-row", + }); + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir, + runId: "wfr_supersede", + createdAtMs: 2_000, + afterBoundaryMessageId: "resume-row", + retryDelaysMs: [50], + }); + const deadline = Date.now() + 5_000; + let boundary: string | null | undefined; + while (Date.now() < deadline) { + boundary = (await readAgentWorkflowRunReferences(workspaceSessionDir)).find( + (reference) => reference.runId === "wfr_supersede" + )?.afterBoundaryMessageId; + if (boundary === "resume-row") { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(boundary).toBe("resume-row"); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("a chain scheduled before cancellation cannot re-arm after it", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Simulates the reschedule window: a failing in-flight write's catch handler schedules + // the next retry DURING the cancellation drain, carrying the pre-cancel generation. + // Registration must refuse it, or the retry recreates the retired sidecar later. + const staleGeneration = getSidecarLifecycleGeneration(workspaceSessionDir); + await clearAgentWorkflowRunReferences(workspaceSessionDir); + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir, + runId: "wfr_stale_chain", + createdAtMs: 1_000, + afterBoundaryMessageId: null, + retryDelaysMs: [30], + lifecycleGeneration: staleGeneration, + }); + await new Promise((resolve) => setTimeout(resolve, 250)); + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("a full history clear cancels pending record retries", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 9ca4544cece..06a192ca60f 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -40,14 +40,31 @@ const pendingSidecarMaintenanceTimersByPath = new Map< Map> >(); const inFlightSidecarMaintenanceByPath = new Map>>(); +// Bumped by cancellation BEFORE timers are cleared: a maintenance chain captures the +// generation when it is first scheduled, and registration refuses a stale generation, so a +// retry rescheduled from a failing write's catch handler DURING the cancellation drain cannot +// re-arm and later recreate retired state. +const lifecycleGenerationByPath = new Map(); -/** Arm a maintenance timer; a newer schedule for the same key supersedes the older timer. */ +export function getSidecarLifecycleGeneration(workspaceSessionDir: string): number { + return lifecycleGenerationByPath.get(referencesPath(workspaceSessionDir)) ?? 0; +} + +/** + * Arm a maintenance timer; a newer schedule for the same key supersedes the older timer. A + * stale lifecycleGeneration (captured before a cancellation) is refused so the chain dies. + */ export function registerSidecarMaintenanceTimer( workspaceSessionDir: string, key: string, - timer: ReturnType + timer: ReturnType, + lifecycleGeneration: number ): void { const filePath = referencesPath(workspaceSessionDir); + if ((lifecycleGenerationByPath.get(filePath) ?? 0) !== lifecycleGeneration) { + clearTimeout(timer); + return; + } let byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); if (byKey == null) { byKey = new Map(); @@ -113,6 +130,7 @@ export async function cancelAgentWorkflowRunReferenceMaintenance( workspaceSessionDir: string ): Promise { const filePath = referencesPath(workspaceSessionDir); + lifecycleGenerationByPath.set(filePath, (lifecycleGenerationByPath.get(filePath) ?? 0) + 1); const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); if (byKey != null) { for (const timer of byKey.values()) { @@ -242,8 +260,12 @@ export async function recordAgentWorkflowRunReference(input: { runId: string; createdAtMs?: number; afterBoundaryMessageId?: string | null; - /** Fill-absence mode for detached retries: an existing entry (any newer record) wins. */ - onlyIfAbsent?: boolean; + /** + * Detached-retry mode: skip only when a strictly newer record already exists, so a retry + * can supersede the stale entry a failed re-record (e.g. workflow_resume over an old + * dispatch) was meant to replace, while a newer successful record still wins. + */ + skipIfNewerRecordExists?: boolean; /** * Boundary-repair mode: only patch an entry that exists and still lacks a boundary * snapshot. A missing entry means the reference was retired (clear/removal) and must not be @@ -265,7 +287,11 @@ export async function recordAgentWorkflowRunReference(input: { // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); - if (input.onlyIfAbsent === true && previous != null) { + if ( + input.skipIfNewerRecordExists === true && + previous != null && + previous.createdAtMs > createdAtMs + ) { return; } if ( @@ -312,6 +338,8 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { afterBoundaryMessageId?: string | null; retryDelaysMs?: readonly number[] | null; attempt?: number; + /** Chain state: the lifecycle generation captured when the chain was first scheduled. */ + lifecycleGeneration?: number; }): void { const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; const attempt = input.attempt ?? 0; @@ -324,6 +352,8 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { return; } const key = `record:${input.runId}`; + const lifecycleGeneration = + input.lifecycleGeneration ?? getSidecarLifecycleGeneration(input.workspaceSessionDir); const timer = setTimeout(() => { if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { return; @@ -334,7 +364,7 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { workspaceSessionDir: input.workspaceSessionDir, runId: input.runId, createdAtMs: input.createdAtMs, - onlyIfAbsent: true, + skipIfNewerRecordExists: true, ...(input.afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId: input.afterBoundaryMessageId } : {}), @@ -344,10 +374,14 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { attempt: attempt + 1, error, }); - scheduleAgentWorkflowRunReferenceRecordRetry({ ...input, attempt: attempt + 1 }); + scheduleAgentWorkflowRunReferenceRecordRetry({ + ...input, + attempt: attempt + 1, + lifecycleGeneration, + }); }); trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); }, delayMs); timer.unref?.(); - registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer); + registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer, lifecycleGeneration); } diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index dbf71169ca3..82a12421356 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -96,6 +96,8 @@ type QueueDispatchMode = NonNullable; interface QueuedMessageInternalOptions { synthetic?: boolean; agentInitiated?: boolean; + /** Persisted onto the dispatched user row; see MuxMetadata.deliveredWorkflowRunIds. */ + deliveredWorkflowRunIds?: string[]; /** * When the sender authored this message (request entry), before any send * preflight awaits (pricing gate, settings persistence). Goal safety diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index f652dd36f46..6225945a12f 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6403,7 +6403,12 @@ describe("TaskService", () => { await flushTerminalAttentionDrains(taskService); expect(sendMessage).toHaveBeenCalledTimes(1); - const internal = sendMessage.mock.calls[0]?.[3] as { admissionStale?: () => boolean }; + const internal = sendMessage.mock.calls[0]?.[3] as { + admissionStale?: () => boolean; + deliveredWorkflowRunIds?: string[]; + }; + // Consumption provenance persisted with the accepted row (crash-replay suppression). + expect(internal.deliveredWorkflowRunIds).toEqual([runId]); expect(typeof internal.admissionStale).toBe("function"); expect(internal.admissionStale?.()).toBe(false); epoch = 2; @@ -6852,6 +6857,30 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); }); + test("an unreadable sidecar defers workspace-turn blocker checks", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const { workspaceService } = createWorkspaceServiceMocks(); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + hasActiveWorkspaceTurnDeferredBlockers(record: { workspaceId: string }): Promise; + }; + + expect(await internal.hasActiveWorkspaceTurnDeferredBlockers({ workspaceId: parentId })).toBe( + false + ); + + // The sidecar is the only durable provenance for kernel-launched workflows, and this + // result gates completion decisions: an unreadable sidecar must report blockers (defer), + // not "none" and let the turn finalize while a workflow may still be running. + await fsPromises.mkdir(path.join(config.getSessionDir(parentId), "agent-workflow-runs.json"), { + recursive: true, + }); + expect(await internal.hasActiveWorkspaceTurnDeferredBlockers({ workspaceId: parentId })).toBe( + true + ); + }); + test("initialize recovers terminal notify workspace turns without pending notification", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 27773cfb84f..dec1d8e87dd 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1702,13 +1702,10 @@ export class TaskService { } const runIds = new Set(); - let references: Awaited> = []; - try { - references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error: unknown) { - // Rediscovery is non-destructive and re-runs on the next listing; skip this pass. - log.warn("Failed to read agent workflow run references", { workspaceId, error }); - } + // An unreadable sidecar PROPAGATES: callers gate destructive completion decisions + // (finalizing task reports, ending workspace turns) on this listing, and treating the + // failure as "no references" would let those finalize while a kernel workflow still runs. + const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); for (const reference of references) { // If the latest user/reset supersession has no durable timestamp, fail safe: only trust // workflow provenance re-established by current/post-supersession assistant output below. @@ -8301,6 +8298,10 @@ export class TaskService { const deliverableWorkflowNotificationIds = new Set(); const promptSections: string[] = []; + // Persisted onto the accepted row as consumption provenance (see + // MuxMetadata.deliveredWorkflowRunIds): after a crash between acceptance and the outbox + // delivery mark, restart recovery recognizes the row and does not replay these results. + const deliveredWorkflowRunIds: string[] = []; if (publicAwaitIds.length > 0) { promptSections.push(buildCompletedWorkspaceTurnPrompt(publicAwaitIds)); } @@ -8330,6 +8331,7 @@ export class TaskService { continue; } deliverableWorkflowNotificationIds.add(notification.id); + deliveredWorkflowRunIds.push(notification.sourceId); promptSections.push(workflowPrompt.prompt); } @@ -8407,6 +8409,7 @@ export class TaskService { agentInitiated: true, requireIdle: true, admissionStale: sendAdmissionStale, + deliveredWorkflowRunIds, } ); @@ -8430,6 +8433,7 @@ export class TaskService { synthetic: true, agentInitiated: true, admissionStale: sendAdmissionStale, + deliveredWorkflowRunIds, onCanceled: () => { this.scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId); }, @@ -12535,10 +12539,21 @@ export class TaskService { return true; } - const referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - record.workspaceId, - [] - ); + let referencedWorkflowRunIds: string[]; + try { + referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + record.workspaceId, + [] + ); + } catch (error: unknown) { + // Unreadable sidecar: assume blockers exist so the deferred turn is not finalized while + // a kernel workflow may still be running; the next evaluation retries. + log.warn("Deferring workspace-turn blocker check; sidecar unreadable", { + workspaceId: record.workspaceId, + error, + }); + return true; + } if ( (await this.listActiveBackgroundWorkflowRunIds(record.workspaceId, referencedWorkflowRunIds)) .length > 0 @@ -14155,11 +14170,22 @@ export class TaskService { taskIndex, workspaceId ); - const referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - workspaceId, - event.parts, - event.messageId - ); + let referencedWorkflowRunIds: string[]; + try { + referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + workspaceId, + event.parts, + event.messageId + ); + } catch (error: unknown) { + // Unreadable sidecar: neither finalize the turn nor nudge the model about unknown + // runs; leave the stream-end unhandled so deferred recovery re-evaluates later. + log.warn("Skipping parent stream-end workflow reconciliation; sidecar unreadable", { + workspaceId, + error, + }); + return; + } let activeWorkflowRunIds = await this.listActiveBackgroundWorkflowRunIds( workspaceId, referencedWorkflowRunIds @@ -14446,11 +14472,22 @@ export class TaskService { return; } - const taskReferencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - workspaceId, - event.parts, - event.messageId - ); + let taskReferencedWorkflowRunIds: string[]; + try { + taskReferencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + workspaceId, + event.parts, + event.messageId + ); + } catch (error: unknown) { + // Unreadable sidecar: defer report finalization like an active blocker instead of + // publishing while a kernel workflow may still be running. + log.warn("Deferring task finalization; sidecar unreadable", { workspaceId, error }); + if (status === "awaiting_report") { + await this.setTaskStatus(workspaceId, "running"); + } + return; + } const activeTaskWorkflowRunIds = await this.listActiveBackgroundWorkflowRunIds( workspaceId, taskReferencedWorkflowRunIds diff --git a/src/node/services/tools/toolUtils.test.ts b/src/node/services/tools/toolUtils.test.ts index 538c2c249a5..9785a1ce04b 100644 --- a/src/node/services/tools/toolUtils.test.ts +++ b/src/node/services/tools/toolUtils.test.ts @@ -93,6 +93,60 @@ describe("recordBackgroundWorkflowRunReference", () => { } }); + test("boundary repair waits for the record retry to land the entry", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); + try { + // Both the boundary read AND the initial sidecar write fail: the reference does not + // exist when the repair first fires. A missing entry must stay retryable, or the record + // retry that lands later creates a permanently boundary-less reference. + let calls = 0; + const taskService = { + getWorkflowInvocationBoundaryMessageId: (): Promise => { + calls += 1; + return calls === 1 + ? Promise.reject(new Error("history unavailable")) + : Promise.resolve(null); + }, + }; + const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_seed", + createdAtMs: 1_000, + }); + await fs.chmod(filePath, 0o000); + await recordBackgroundWorkflowRunReference( + { + workspaceSessionDir, + workspaceId: "ws-boundary-late", + taskService, + } as unknown as ToolConfiguration, + "wfr_boundary_late", + 2_000, + [40, 40, 40, 40, 40] + ); + // Storage recovers only after the repair has fired at least once against the missing + // entry; the record retry then lands it and a later repair attempt patches null. + await new Promise((resolve) => setTimeout(resolve, 60)); + await fs.chmod(filePath, 0o600); + + const deadline = Date.now() + 5_000; + let boundary: string | null | undefined; + while (Date.now() < deadline) { + boundary = (await readAgentWorkflowRunReferences(workspaceSessionDir)).find( + (reference) => reference.runId === "wfr_boundary_late" + )?.afterBoundaryMessageId; + if (boundary === null) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(boundary).toBe(null); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("keeps the entry boundary-less when a decision row exists at repair time", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); try { diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 95b3c8355af..89e095dce18 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -9,6 +9,8 @@ import type { WorkspaceChatMessage } from "@/common/orpc/types"; import type { ToolConfiguration } from "@/common/utils/tools/tools"; import { SIDECAR_MAINTENANCE_RETRY_DELAYS_MS, + getSidecarLifecycleGeneration, + readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, registerSidecarMaintenanceTimer, scheduleAgentWorkflowRunReferenceRecordRetry, @@ -103,6 +105,8 @@ function scheduleBoundarySnapshotRepair(input: { getBoundary: () => Promise; retryDelaysMs?: readonly number[] | null; attempt?: number; + /** Chain state: the lifecycle generation captured when the chain was first scheduled. */ + lifecycleGeneration?: number; }): void { const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; const attempt = input.attempt ?? 0; @@ -115,11 +119,25 @@ function scheduleBoundarySnapshotRepair(input: { return; } const key = `boundary:${input.runId}`; + const lifecycleGeneration = + input.lifecycleGeneration ?? getSidecarLifecycleGeneration(input.workspaceSessionDir); const timer = setTimeout(() => { if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { return; } const work = (async () => { + // The reference may not exist yet: when the initial write also failed, the independent + // record-retry chain lands it later. A missing entry is retryable, not a satisfied + // no-op, or that later record would create a permanently boundary-less reference; + // lifecycle cancellation kills this chain when the reference was retired instead. + const references = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + const entry = references.find((reference) => reference.runId === input.runId); + if (entry == null) { + throw new Error("reference not recorded yet"); + } + if (entry.afterBoundaryMessageId !== undefined) { + return; + } const boundary = await input.getBoundary(); if (boundary !== null) { return; @@ -137,12 +155,12 @@ function scheduleBoundarySnapshotRepair(input: { attempt: attempt + 1, error: getErrorMessage(error), }); - scheduleBoundarySnapshotRepair({ ...input, attempt: attempt + 1 }); + scheduleBoundarySnapshotRepair({ ...input, attempt: attempt + 1, lifecycleGeneration }); }); trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); }, delayMs); timer.unref?.(); - registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer); + registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer, lifecycleGeneration); } /** diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4994dcc6169..e92a56be1f6 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6374,44 +6374,47 @@ describe("WorkspaceService workflow invocation events", () => { }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - // Another run's payload quoting nothing about this run must not count as consumption. + // A synthetic row whose TEXT reproduces a result payload must not count: synthetic rows + // can carry user-controlled content (e.g. a heartbeat body), and a quoted run ID must + // not spoof consumption and suppress the real wake. await historyService.appendToHistory( workspaceId, createMuxMessage( - "coalesced-other", + "coalesced-spoof", "user", buildWorkflowResultContextMessage({ - rawCommand: "workflow_run other.js", - name: "other.js", - runId: "wfr_currentness_other", + rawCommand: "workflow_run research.js", + name: "research.js", + runId, status: "completed", - result: { reportMarkdown: "other done" }, + result: { reportMarkdown: "spoofed" }, run: null, }), - { timestamp: 1_250, synthetic: true } + { timestamp: 1_240, synthetic: true } ) ); + // Another run's persisted provenance must not count for this run either. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("coalesced-other", "user", "results delivered", { + timestamp: 1_250, + synthetic: true, + deliveredWorkflowRunIds: ["wfr_currentness_other"], + }) + ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a - // crash between durable acceptance and the outbox delivery mark, this row is the only + // The drain persists deliveredWorkflowRunIds onto the accepted row. After a crash + // between durable acceptance and the outbox delivery mark, this provenance is the only // evidence the result already reached history; it must read as consumption or restart // recovery injects the same terminal result again. await historyService.appendToHistory( workspaceId, - createMuxMessage( - "coalesced-result", - "user", - buildWorkflowResultContextMessage({ - rawCommand: "workflow_run research.js", - name: "research.js", - runId, - status: "completed", - result: { reportMarkdown: "done" }, - run: null, - }), - { timestamp: 1_300, synthetic: true } - ) + createMuxMessage("coalesced-result", "user", "results delivered", { + timestamp: 1_300, + synthetic: true, + deliveredWorkflowRunIds: [runId], + }) ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); workspaceService.disposeSession(workspaceId); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index da118d47116..3b6e9b037b6 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -204,7 +204,6 @@ import { } from "@/node/services/workflows/workflowArchiveAdmission"; import { WORKFLOW_RESULT_METADATA_TYPE, - textContainsWorkflowResultPayload, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowRunCardMessage, @@ -478,18 +477,16 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) /** * The terminal-attention drain delivers workflow results as one synthetic user prompt that may - * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands - * between the send's durable acceptance and the outbox delivery mark, restart recovery drains - * the notification again; recognizing the accepted row as consumption is what suppresses the - * replay. Only synthetic rows qualify: a manual user message is a supersession boundary and is - * classified before this check runs. + * coalesce several runs. If a crash lands between the send's durable acceptance and the outbox + * delivery mark, restart recovery drains the notification again; recognizing the accepted row + * as consumption is what suppresses the replay. Recognition uses the drain's persisted + * provenance (MuxMetadata.deliveredWorkflowRunIds), never the row's text: synthetic rows can + * carry user-controlled content (e.g. a heartbeat body), and a quoted run ID must not spoof + * consumption and suppress a real wake. */ function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { - if (message.role !== "user" || message.metadata?.synthetic !== true) { - return false; - } - return message.parts.some( - (part) => part.type === "text" && textContainsWorkflowResultPayload(part.text, runId) + return ( + message.role === "user" && message.metadata?.deliveredWorkflowRunIds?.includes(runId) === true ); } @@ -11254,6 +11251,8 @@ export class WorkspaceService extends EventEmitter { goalId?: string; /** Force Copilot billing classification to "agent" for internal sends. */ agentInitiated?: boolean; + /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ + deliveredWorkflowRunIds?: string[]; onAccepted?: () => Promise | void; onCanceled?: (reason: string) => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; @@ -11516,6 +11515,7 @@ export class WorkspaceService extends EventEmitter { return await session.sendMessage(message, normalizedOptions, { synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, + deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, goalKind: internal?.goalKind, goalId: internal?.goalId, cancelState: internal?.cancelState, @@ -11660,6 +11660,7 @@ export class WorkspaceService extends EventEmitter { { synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, + deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, authoredAtMs, workspaceTurnContinuation: internal?.workspaceTurnContinuation, dedupeKey: internal?.queueDedupeKey, @@ -11772,6 +11773,7 @@ export class WorkspaceService extends EventEmitter { onTurnAdmissionCommitted: () => sessionInvisiblePreflight.release(), synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, + deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, goalKind: internal?.goalKind, goalId: internal?.goalId, goalContinuation: internal?.goalContinuation, From e14574d8bf2628b68d991ccaeee6e9cc89842860 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:36:47 +0000 Subject: [PATCH 18/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20restore=20the=20cal?= =?UTF-8?q?ler=20tool=20policy=20on=20kernel=20workflow=20wakes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ordinary in-stream workflow continuation carries the live turn's effectiveToolPolicy; the terminal-attention drain's synthetic send starts a fresh turn and omitted it, so a workflow wake could regain tools the caller disabled, with attacker-influenced workflow output choosing the timing. The drain now restores the newest manual user row's persisted caller policy (synthetic rows without one are skipped), fails closed by deferring the wake when history is unreadable, and the agent-level policy recomposes from agentId at send resolution. --- src/node/services/taskService.test.ts | 78 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 48 +++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 6225945a12f..79f5369a5af 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6360,6 +6360,84 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("workflow wakes restore the caller tool policy from the newest manual row", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_policy_restore"); + await createRun("wfr_policy_lifted"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // The launch turn disabled bash; a later synthetic row (an earlier wake) defines no + // policy and must be skipped. Omitting the policy on the wake would let workflow output + // regain the disabled tool at a time the workflow chooses. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("earlier-wake", "user", "results delivered", { + timestamp: 1_100, + synthetic: true, + }) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_policy_restore", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + toolPolicy: restrictedPolicy, + }); + + // A newer manual row without a policy means the caller lifted it: no restoration. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-unrestricted", "user", "carry on", { timestamp: 1_200 }) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_policy_lifted", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + const liftedOptions = sendMessage.mock.calls[1]?.[2] as { toolPolicy?: unknown }; + expect(liftedOptions.toolPolicy).toBeUndefined(); + }); + test("drain sends carry a staleness probe that trips after a full clear", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index dec1d8e87dd..5f431201f42 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -188,6 +188,7 @@ import { isNonRetryableStreamError } from "@/common/utils/messages/retryEligibil import type { SendMessageError, StreamErrorType } from "@/common/types/errors"; import { hasCompletedAgentReport } from "@/common/utils/agentTaskCompletion"; import { isWorkspaceArchived } from "@/common/utils/archive"; +import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; import { DEFAULT_CODER_ARCHIVE_BEHAVIOR } from "@/common/config/coderArchiveBehavior"; import { isSSHRuntime, isWorktreeRuntime } from "@/common/types/runtime"; @@ -7913,6 +7914,34 @@ export class TaskService { this.terminalAttentionDeferRetryTimers.set(ownerWorkspaceId, timer); } + /** + * Caller tool policy to restore on a terminal-attention wake. The newest manual user row + * carries the conversation's persisted caller policy; synthetic rows without one (earlier + * wakes, heartbeat scaffolding) do not define policy and are skipped. Throws when history + * is unreadable so the caller can fail closed instead of waking with unrestricted tools. + */ + private async resolveTerminalWakeCallerToolPolicy( + ownerWorkspaceId: string + ): Promise { + const historyResult = await this.historyService.getLastMessages(ownerWorkspaceId, 50); + if (!historyResult.success) { + throw new Error(`history unavailable: ${historyResult.error}`); + } + for (let i = historyResult.data.length - 1; i >= 0; i--) { + const message = historyResult.data[i]; + if (message?.role !== "user") { + continue; + } + if (message.metadata?.toolPolicy != null) { + return message.metadata.toolPolicy; + } + if (message.metadata?.synthetic !== true) { + return undefined; + } + } + return undefined; + } + private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { const promise = this.workspaceService .waitForIdleAndNoQueuedMessages(ownerWorkspaceId) @@ -8369,11 +8398,30 @@ export class TaskService { const workspaceTurnMuxMetadata = await this.getActiveWorkspaceTurnMuxMetadataForWorkspace(ownerWorkspaceId); + // Security: restore the conversation's active caller tool policy on the wake. The ordinary + // in-stream workflow continuation carries the live turn's effectiveToolPolicy; this + // synthetic send starts a fresh turn, and omitting the policy would let a workflow wake + // regain tools the caller disabled (with attacker-influenced workflow output choosing the + // timing). The agent-level policy recomposes from agentId at send resolution. + let wakeToolPolicy: ToolPolicy | undefined; + try { + wakeToolPolicy = await this.resolveTerminalWakeCallerToolPolicy(ownerWorkspaceId); + } catch (error: unknown) { + // Fail closed: an unknown policy must not fall back to unrestricted tools. + log.warn("Deferring terminal wake; caller tool policy unavailable", { + ownerWorkspaceId, + error, + }); + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); + return; + } + const sendOptions = { model: resumeOptions.model, agentId: resumeOptions.agentId, thinkingLevel: resumeOptions.thinkingLevel, reasoningMode: resumeOptions.reasoningMode, + ...(wakeToolPolicy != null ? { toolPolicy: wakeToolPolicy } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { From 95ca943e9d8468970b3849244c331bdfe7eb5708 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 05:21:55 +0000 Subject: [PATCH 19/39] =?UTF-8?q?=F0=9F=A4=96=20refactor:=20strip=20rounds?= =?UTF-8?q?=2011-14=20retry/repair=20machinery;=20keep=20identity-based=20?= =?UTF-8?q?currentness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per maintainer decision on the review spiral: the detached record retries, generation latches, boundary repair timers, corrupt-sidecar self-heal, removal drains, admission staleness probes, and persisted drain provenance (deliveredWorkflowRunIds) added in rounds 11-14 are removed wholesale. The core design stays: kernel-launched background workflow runs persist a sidecar reference whose boundary-row identity snapshot decides wake currentness, workflow_resume persists terminal consumption, and the terminal drain restores the caller's send restrictions. Folded-in review fixes on the retained core: - The wake restriction walk is unbounded (iterateFullHistory backward), so a long assistant/synthetic tail cannot silently lift the caller's tool policy, and the newest manual turn's disableWorkspaceAgents flag is restored alongside it. - Foreground workflow_resume derives terminal consumption from the dispatch result itself, so a transient refresh read failure cannot leave the delivered result armed for re-injection. --- src/common/types/message.ts | 7 - src/common/utils/workflowRunMessages.ts | 35 +++ src/node/services/agentSession.ts | 7 - .../agentWorkflowRunReferences.test.ts | 112 --------- .../services/agentWorkflowRunReferences.ts | 215 +----------------- src/node/services/messageQueue.ts | 2 - src/node/services/taskService.test.ts | 123 +++------- src/node/services/taskService.ts | 171 ++++++-------- src/node/services/tools/toolUtils.test.ts | 183 --------------- src/node/services/tools/toolUtils.ts | 108 +-------- .../services/tools/workflow_resume.test.ts | 38 ++++ src/node/services/tools/workflow_resume.ts | 16 +- src/node/services/workspaceRemoval.test.ts | 34 --- src/node/services/workspaceRemoval.ts | 5 - src/node/services/workspaceService.test.ts | 171 ++------------ src/node/services/workspaceService.ts | 65 +++--- 16 files changed, 231 insertions(+), 1061 deletions(-) delete mode 100644 src/node/services/tools/toolUtils.test.ts diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 2ad5017f66a..9b16234a106 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -868,13 +868,6 @@ export interface ModelFallbackRecord { // Our custom metadata type export interface MuxMetadata { - /** - * Workflow run IDs whose terminal results this synthetic user row delivered (set by the - * terminal-attention drain). Currentness checks treat the row as consumption for these runs; - * persisted provenance, not text recognition, so repo/user-controlled content quoting a run - * ID (e.g. a heartbeat body) cannot spoof consumption and suppress a real wake. - */ - deliveredWorkflowRunIds?: string[]; /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) diff --git a/src/common/utils/workflowRunMessages.ts b/src/common/utils/workflowRunMessages.ts index 9fc393e9f7e..66a2b3b5fa9 100644 --- a/src/common/utils/workflowRunMessages.ts +++ b/src/common/utils/workflowRunMessages.ts @@ -190,6 +190,41 @@ export function buildWorkflowResultContextMessage(input: { ].join("\n\n"); } +/** + * Recognize this run's result payload inside a coalesced terminal-attention prompt from the + * builder's own output format. The drain's synthetic user row can coalesce several runs into + * one message and carries no workflow-result metadata, so currentness checks must read the + * consumption evidence out of the text: each payload block is parsed back and matched on the + * exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted + * inside another run's report cannot count as consumption. + */ +export function textContainsWorkflowResultPayload(text: string, runId: string): boolean { + assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required"); + if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) { + return false; + } + const blockPattern = new RegExp( + `<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n`, + "g" + ); + for (const match of text.matchAll(blockPattern)) { + let payload: unknown; + try { + payload = JSON.parse(match[1] ?? ""); + } catch { + continue; + } + if (!isRecordValue(payload)) { + continue; + } + const workflow = payload.workflow; + if (isRecordValue(workflow) && workflow.runId === runId) { + return true; + } + } + return false; +} + export interface WorkflowRunCardInput { scriptPath?: string; scriptSource?: string; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index aba92a18eeb..4422a49a05e 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -2950,8 +2950,6 @@ export class AgentSession { internal?: { synthetic?: boolean; agentInitiated?: boolean; - /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ - deliveredWorkflowRunIds?: string[]; goalContinuation?: boolean; goalKind?: GoalSyntheticMessageKind; /** Goal identity persisted alongside goalKind so chat-tail reconciliation can scope the row. */ @@ -3547,9 +3545,6 @@ export class AgentSession { ...(internal?.enqueuedAtMs != null ? { enqueuedAtMs: internal.enqueuedAtMs } : {}), // Auto-resume and other system-generated messages are synthetic + UI-visible ...(internal?.synthetic && { synthetic: true, uiVisible: true }), - ...(internal?.deliveredWorkflowRunIds != null && internal.deliveredWorkflowRunIds.length > 0 - ? { deliveredWorkflowRunIds: internal.deliveredWorkflowRunIds } - : {}), }, additionalParts ); @@ -6395,8 +6390,6 @@ export class AgentSession { internal?: { synthetic?: boolean; agentInitiated?: boolean; - /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ - deliveredWorkflowRunIds?: string[]; /** Request-entry authoring time captured before send preflight awaits (see MessageQueue). */ authoredAtMs?: number; /** True only for a report that continues an existing workspace turn. */ diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 30769c481ac..69171c081cf 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -5,11 +5,8 @@ import * as path from "node:path"; import { describe, expect, test } from "bun:test"; import { - clearAgentWorkflowRunReferences, - getSidecarLifecycleGeneration, readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, - scheduleAgentWorkflowRunReferenceRecordRetry, } from "@/node/services/agentWorkflowRunReferences"; describe("agent workflow run references", () => { @@ -193,115 +190,6 @@ describe("agent workflow run references", () => { } }); - test("a pending record retry never overwrites newer provenance", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // The retry carries the stale launch-time snapshot; a workflow_resume records newer - // provenance before the timer fires. Fill-absence semantics must let the newer record win. - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId: "wfr_lifecycle", - createdAtMs: 1_000, - afterBoundaryMessageId: "stale-row", - retryDelaysMs: [150], - }); - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_lifecycle", - createdAtMs: 2_000, - afterBoundaryMessageId: "resume-row", - }); - await new Promise((resolve) => setTimeout(resolve, 400)); - const references = await readAgentWorkflowRunReferences(workspaceSessionDir); - expect(references).toHaveLength(1); - expect(references[0]).toMatchObject({ - runId: "wfr_lifecycle", - afterBoundaryMessageId: "resume-row", - }); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("a retry supersedes an older entry for the same run", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // A workflow_resume re-record fails transiently while an OLDER dispatch entry exists. - // The retry must replace that stale provenance (its boundary predates the resume) or the - // resumed run's wake is classified not_current; only a strictly newer record wins. - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_supersede", - createdAtMs: 500, - afterBoundaryMessageId: "old-row", - }); - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId: "wfr_supersede", - createdAtMs: 2_000, - afterBoundaryMessageId: "resume-row", - retryDelaysMs: [50], - }); - const deadline = Date.now() + 5_000; - let boundary: string | null | undefined; - while (Date.now() < deadline) { - boundary = (await readAgentWorkflowRunReferences(workspaceSessionDir)).find( - (reference) => reference.runId === "wfr_supersede" - )?.afterBoundaryMessageId; - if (boundary === "resume-row") { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(boundary).toBe("resume-row"); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("a chain scheduled before cancellation cannot re-arm after it", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // Simulates the reschedule window: a failing in-flight write's catch handler schedules - // the next retry DURING the cancellation drain, carrying the pre-cancel generation. - // Registration must refuse it, or the retry recreates the retired sidecar later. - const staleGeneration = getSidecarLifecycleGeneration(workspaceSessionDir); - await clearAgentWorkflowRunReferences(workspaceSessionDir); - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId: "wfr_stale_chain", - createdAtMs: 1_000, - afterBoundaryMessageId: null, - retryDelaysMs: [30], - lifecycleGeneration: staleGeneration, - }); - await new Promise((resolve) => setTimeout(resolve, 250)); - expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("a full history clear cancels pending record retries", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // A stale detached retry must not resurrect a reference the clear retired; against the - // then decision-free history it would read current and inject the pre-clear result. - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId: "wfr_cleared", - createdAtMs: 1_000, - afterBoundaryMessageId: null, - retryDelaysMs: [100], - }); - await clearAgentWorkflowRunReferences(workspaceSessionDir); - await new Promise((resolve) => setTimeout(resolve, 350)); - expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - test("self-heals unparseable file contents to empty", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 06a192ca60f..43a9d23a778 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -5,7 +5,6 @@ import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; -import { log } from "@/node/services/log"; export interface AgentWorkflowRunReference { runId: string; @@ -29,121 +28,6 @@ const MAX_FUTURE_SKEW_MS = 60 * 60_000; const referenceFileLocks = new MutexMap(); -// Detached sidecar maintenance (record retries, boundary-snapshot repairs) keyed by sidecar -// path and a per-run key so lifecycle events can govern it: a full clear or workspace removal -// cancels the path's timers and drains in-flight writes, so stale maintenance can neither -// resurrect a retired reference nor recreate a deleted session directory, and maintenance -// writes only ever fill gaps (onlyIfAbsent / onlyIfBoundaryAbsent), so they cannot overwrite -// newer provenance recorded by a later dispatch or workflow_resume. -const pendingSidecarMaintenanceTimersByPath = new Map< - string, - Map> ->(); -const inFlightSidecarMaintenanceByPath = new Map>>(); -// Bumped by cancellation BEFORE timers are cleared: a maintenance chain captures the -// generation when it is first scheduled, and registration refuses a stale generation, so a -// retry rescheduled from a failing write's catch handler DURING the cancellation drain cannot -// re-arm and later recreate retired state. -const lifecycleGenerationByPath = new Map(); - -export function getSidecarLifecycleGeneration(workspaceSessionDir: string): number { - return lifecycleGenerationByPath.get(referencesPath(workspaceSessionDir)) ?? 0; -} - -/** - * Arm a maintenance timer; a newer schedule for the same key supersedes the older timer. A - * stale lifecycleGeneration (captured before a cancellation) is refused so the chain dies. - */ -export function registerSidecarMaintenanceTimer( - workspaceSessionDir: string, - key: string, - timer: ReturnType, - lifecycleGeneration: number -): void { - const filePath = referencesPath(workspaceSessionDir); - if ((lifecycleGenerationByPath.get(filePath) ?? 0) !== lifecycleGeneration) { - clearTimeout(timer); - return; - } - let byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); - if (byKey == null) { - byKey = new Map(); - pendingSidecarMaintenanceTimersByPath.set(filePath, byKey); - } - const previous = byKey.get(key); - if (previous != null) { - clearTimeout(previous); - } - byKey.set(key, timer); -} - -/** - * Consume a fired maintenance timer. clearTimeout cannot stop a callback Node already - * dequeued, so registry identity is the authoritative cancellation signal: a - * cancelled-but-raced callback sees a mismatch and must abort. - */ -export function takeSidecarMaintenanceTimer( - workspaceSessionDir: string, - key: string, - timer: ReturnType -): boolean { - const filePath = referencesPath(workspaceSessionDir); - const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); - if (byKey?.get(key) !== timer) { - return false; - } - byKey.delete(key); - if (byKey.size === 0) { - pendingSidecarMaintenanceTimersByPath.delete(filePath); - } - return true; -} - -/** Track a maintenance write so cancelAgentWorkflowRunReferenceMaintenance can drain it. */ -export function trackSidecarMaintenanceWrite( - workspaceSessionDir: string, - work: Promise -): void { - const filePath = referencesPath(workspaceSessionDir); - let inFlight = inFlightSidecarMaintenanceByPath.get(filePath); - if (inFlight == null) { - inFlight = new Set(); - inFlightSidecarMaintenanceByPath.set(filePath, inFlight); - } - inFlight.add(work); - const remove = () => { - inFlight.delete(work); - if (inFlight.size === 0) { - inFlightSidecarMaintenanceByPath.delete(filePath); - } - }; - work.then(remove, remove); -} - -/** - * Cancel this sidecar's pending maintenance timers and drain writes already past their - * identity check, so a full clear or workspace removal cannot race a recreation of the file - * or the session directory it is about to delete. Must run BEFORE taking the sidecar file - * lock: an in-flight write acquires that same lock, so draining inside it would deadlock. - */ -export async function cancelAgentWorkflowRunReferenceMaintenance( - workspaceSessionDir: string -): Promise { - const filePath = referencesPath(workspaceSessionDir); - lifecycleGenerationByPath.set(filePath, (lifecycleGenerationByPath.get(filePath) ?? 0) + 1); - const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); - if (byKey != null) { - for (const timer of byKey.values()) { - clearTimeout(timer); - } - pendingSidecarMaintenanceTimersByPath.delete(filePath); - } - const inFlight = inFlightSidecarMaintenanceByPath.get(filePath); - if (inFlight != null && inFlight.size > 0) { - await Promise.allSettled([...inFlight]); - } -} - function referencesPath(workspaceSessionDir: string): string { assert(workspaceSessionDir.length > 0, "agent workflow references require session dir"); return path.join(workspaceSessionDir, AGENT_WORKFLOW_RUN_REFERENCES_FILE); @@ -241,17 +125,12 @@ export async function readAgentWorkflowRunReferences( * appending a reset boundary, which makes a verified-empty (null) boundary snapshot recorded * before the clear indistinguishable from one recorded after it; retiring the references with * the transcript keeps pre-clear workflow results out of the fresh conversation. A post-clear - * workflow_resume re-records provenance. Pending record retries are cancelled first so a stale - * detached retry cannot recreate a retired reference. + * workflow_resume re-records provenance. */ export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { const filePath = referencesPath(workspaceSessionDir); - await cancelAgentWorkflowRunReferenceMaintenance(workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { - // recursive: a directory at this known sidecar path is corruption (it also fails reads - // with EISDIR), and force alone refuses to remove directories, which would fail every - // subsequent full clear identically. Removing it self-heals the workspace. - await fs.rm(filePath, { force: true, recursive: true }); + await fs.rm(filePath, { force: true }); }); } @@ -260,18 +139,6 @@ export async function recordAgentWorkflowRunReference(input: { runId: string; createdAtMs?: number; afterBoundaryMessageId?: string | null; - /** - * Detached-retry mode: skip only when a strictly newer record already exists, so a retry - * can supersede the stale entry a failed re-record (e.g. workflow_resume over an old - * dispatch) was meant to replace, while a newer successful record still wins. - */ - skipIfNewerRecordExists?: boolean; - /** - * Boundary-repair mode: only patch an entry that exists and still lacks a boundary - * snapshot. A missing entry means the reference was retired (clear/removal) and must not be - * resurrected; a present boundary means newer provenance already landed. - */ - onlyIfBoundaryAbsent?: boolean; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -287,19 +154,6 @@ export async function recordAgentWorkflowRunReference(input: { // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); - if ( - input.skipIfNewerRecordExists === true && - previous != null && - previous.createdAtMs > createdAtMs - ) { - return; - } - if ( - input.onlyIfBoundaryAbsent === true && - (previous == null || previous.afterBoundaryMessageId !== undefined) - ) { - return; - } byRunId.set(input.runId, { runId: input.runId, // Latest record wins: workflow_resume re-records the reference, and a resume issued after @@ -320,68 +174,3 @@ export async function recordAgentWorkflowRunReference(input: { ); }); } - -// Delays for detached sidecar maintenance (record retries, boundary-snapshot repairs). -export const SIDECAR_MAINTENANCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; - -/** - * Retry a failed provenance record in the background. The launching tool has already returned - * and an untouched active run never hits a natural re-record site, so a single failed write - * would permanently supersede the run's terminal wake once storage recovers. Retries reuse the - * launch-time boundary snapshot, only fill absence (a later successful record wins), and are - * cancelled by a full history clear. - */ -export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { - workspaceSessionDir: string; - runId: string; - createdAtMs: number; - afterBoundaryMessageId?: string | null; - retryDelaysMs?: readonly number[] | null; - attempt?: number; - /** Chain state: the lifecycle generation captured when the chain was first scheduled. */ - lifecycleGeneration?: number; -}): void { - const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; - const attempt = input.attempt ?? 0; - const delayMs = retryDelaysMs[attempt]; - if (delayMs == null) { - log.error("Giving up on agent workflow run reference record after retries", { - runId: input.runId, - attempts: attempt, - }); - return; - } - const key = `record:${input.runId}`; - const lifecycleGeneration = - input.lifecycleGeneration ?? getSidecarLifecycleGeneration(input.workspaceSessionDir); - const timer = setTimeout(() => { - if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { - return; - } - // Detached by design: the launching tool already returned, so only this chain can finish - // the write. Failures reschedule until the bounded delays are exhausted. - const work = recordAgentWorkflowRunReference({ - workspaceSessionDir: input.workspaceSessionDir, - runId: input.runId, - createdAtMs: input.createdAtMs, - skipIfNewerRecordExists: true, - ...(input.afterBoundaryMessageId !== undefined - ? { afterBoundaryMessageId: input.afterBoundaryMessageId } - : {}), - }).catch((error: unknown) => { - log.warn("Agent workflow run reference record retry failed", { - runId: input.runId, - attempt: attempt + 1, - error, - }); - scheduleAgentWorkflowRunReferenceRecordRetry({ - ...input, - attempt: attempt + 1, - lifecycleGeneration, - }); - }); - trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); - }, delayMs); - timer.unref?.(); - registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer, lifecycleGeneration); -} diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 82a12421356..dbf71169ca3 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -96,8 +96,6 @@ type QueueDispatchMode = NonNullable; interface QueuedMessageInternalOptions { synthetic?: boolean; agentInitiated?: boolean; - /** Persisted onto the dispatched user row; see MuxMetadata.deliveredWorkflowRunIds. */ - deliveredWorkflowRunIds?: string[]; /** * When the sender authored this message (request entry), before any send * preflight awaits (pricing gate, settings persistence). Goal safety diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 79f5369a5af..8367cc47dc3 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -554,7 +554,6 @@ function createWorkspaceServiceMocks( isExperimentEnabled: ReturnType; emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; - getContextMutationEpoch: ReturnType; create: ReturnType; countQueuedAgentPeerMessages: ReturnType; }> @@ -592,7 +591,6 @@ function createWorkspaceServiceMocks( isExperimentEnabled: ReturnType; emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; - getContextMutationEpoch: ReturnType; create: ReturnType; } { const sendMessage = @@ -662,7 +660,6 @@ function createWorkspaceServiceMocks( const updateAgentStatus = overrides?.updateAgentStatus ?? mock((): Promise => Promise.resolve()); const isExperimentEnabled = overrides?.isExperimentEnabled ?? mock(() => false); - const getContextMutationEpoch = overrides?.getContextMutationEpoch ?? mock(() => 0); const emitChatEvent = overrides?.emitChatEvent ?? mock((_workspaceId: string, _message: WorkspaceChatMessage) => undefined); @@ -739,7 +736,6 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, - getContextMutationEpoch, getWorkflowInvocationCurrentness, countQueuedAgentPeerMessages, } as unknown as WorkspaceService, @@ -776,7 +772,6 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, - getContextMutationEpoch, }; } @@ -6438,10 +6433,11 @@ describe("TaskService", () => { expect(liftedOptions.toolPolicy).toBeUndefined(); }); - test("drain sends carry a staleness probe that trips after a full clear", async () => { + test("wake restriction restore walks past a long synthetic tail and carries the agent disable flag", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_admission_stale"; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runId = "wfr_policy_long_tail"; const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); await runStore.createRun({ id: runId, @@ -6463,34 +6459,38 @@ describe("TaskService", () => { const sendMessage = mock( (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) ); - // The prompt is validated against history before the send is admitted; a full clear in - // that window advances the context-mutation epoch. The probe handed to sendMessage must - // observe the live epoch so admission can refuse the stale prompt. - let epoch = 1; - const { workspaceService } = createWorkspaceServiceMocks({ - sendMessage, - getContextMutationEpoch: mock(() => epoch), - }); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }) + ); + // A tail longer than any bounded history read: the launch turn's restrictions must still + // be found, not silently lifted once enough rows accumulate after the manual turn. + for (let i = 0; i < 60; i++) { + await historyService.appendToHistory( + parentId, + createMuxMessage(`assistant-${i}`, "assistant", `progress ${i}`, { timestamp: 1_001 + i }) + ); + } await taskService.enqueueWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", }); await flushTerminalAttentionDrains(taskService); - expect(sendMessage).toHaveBeenCalledTimes(1); - const internal = sendMessage.mock.calls[0]?.[3] as { - admissionStale?: () => boolean; - deliveredWorkflowRunIds?: string[]; - }; - // Consumption provenance persisted with the accepted row (crash-replay suppression). - expect(internal.deliveredWorkflowRunIds).toEqual([runId]); - expect(typeof internal.admissionStale).toBe("function"); - expect(internal.admissionStale?.()).toBe(false); - epoch = 2; - expect(internal.admissionStale?.()).toBe(true); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }); }); test("initialize replays and clears persisted pending task guidance", async () => { @@ -6890,75 +6890,6 @@ describe("TaskService", () => { }); }); - test("initialize recovery keeps indeterminate workflow runs enqueued for a later drain", async () => { - const config = await createTestConfig(rootDir); - const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_recovery_indeterminate"; - const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); - await runStore.createRun({ - id: runId, - workspaceId: parentId, - workflow: { - name: "research", - description: "Research workflow", - scope: "built-in", - executable: true, - }, - source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", - args: {}, - attentionPolicy: "notify_on_terminal", - now: "2026-06-19T00:00:00.000Z", - }); - await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); - await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); - - const terminalAttentionStore = new TerminalAttentionStore(config); - const sendMessage = mock( - (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) - ); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - // History/sidecar unreadable at startup. Recovery is the only reconstruction point for a - // wake that never reached the outbox, and no pending notification exists yet to arm the - // drain's defer retry, so skipping here would strand the run until another restart. The - // boolean wrapper collapses indeterminate to false, which is what recovery must NOT use. - (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = - mock(() => Promise.resolve("indeterminate")); - (workspaceService as unknown as Record).isWorkflowInvocationCurrent = mock( - () => Promise.resolve(false) - ); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); - - await taskService.initialize(); - await flushTerminalAttentionDrains(taskService); - - expect(sendMessage).not.toHaveBeenCalled(); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); - }); - - test("an unreadable sidecar defers workspace-turn blocker checks", async () => { - const config = await createTestConfig(rootDir); - const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const { workspaceService } = createWorkspaceServiceMocks(); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); - const internal = taskService as unknown as { - hasActiveWorkspaceTurnDeferredBlockers(record: { workspaceId: string }): Promise; - }; - - expect(await internal.hasActiveWorkspaceTurnDeferredBlockers({ workspaceId: parentId })).toBe( - false - ); - - // The sidecar is the only durable provenance for kernel-launched workflows, and this - // result gates completion decisions: an unreadable sidecar must report blockers (defer), - // not "none" and let the turn finalize while a workflow may still be running. - await fsPromises.mkdir(path.join(config.getSessionDir(parentId), "agent-workflow-runs.json"), { - recursive: true, - }); - expect(await internal.hasActiveWorkspaceTurnDeferredBlockers({ workspaceId: parentId })).toBe( - true - ); - }); - test("initialize recovers terminal notify workspace turns without pending notification", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 5f431201f42..f2177fde1b5 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1703,10 +1703,13 @@ export class TaskService { } const runIds = new Set(); - // An unreadable sidecar PROPAGATES: callers gate destructive completion decisions - // (finalizing task reports, ending workspace turns) on this listing, and treating the - // failure as "no references" would let those finalize while a kernel workflow still runs. - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + let references: Awaited> = []; + try { + references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error: unknown) { + // Rediscovery is non-destructive and re-runs on the next listing; skip this pass. + log.warn("Failed to read agent workflow run references", { workspaceId, error }); + } for (const reference of references) { // If the latest user/reset supersession has no durable timestamp, fail safe: only trust // workflow provenance re-established by current/post-supersession assistant output below. @@ -7594,15 +7597,7 @@ export class TaskService { ) { continue; } - const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( - workspace.id, - run.id - ); - // Indeterminate (unreadable history/sidecar) must still enqueue: startup recovery is - // the only reconstruction point for wakes that never reached the outbox, and no - // pending notification exists yet to arm the drain's defer retry. The drain - // re-evaluates currentness and defers or supersedes with full context. - if (currentness === "not_current") { + if (!(await this.workspaceService.isWorkflowInvocationCurrent(workspace.id, run.id))) { continue; } const created = await this.terminalAttentionStore.enqueueIfAbsent({ @@ -7915,31 +7910,50 @@ export class TaskService { } /** - * Caller tool policy to restore on a terminal-attention wake. The newest manual user row - * carries the conversation's persisted caller policy; synthetic rows without one (earlier - * wakes, heartbeat scaffolding) do not define policy and are skipped. Throws when history - * is unreadable so the caller can fail closed instead of waking with unrestricted tools. + * Caller send restrictions (tool policy, workspace-agent disable flag) to restore on a + * terminal-attention wake. The newest manual user row carries the conversation's persisted + * restrictions; synthetic rows without any (earlier wakes, heartbeat scaffolding) do not + * define them and are skipped. The walk is unbounded: a long assistant/synthetic tail after + * the launch turn must not push the defining row out of sight and silently lift the + * restrictions. Throws when history is unreadable so the caller can fail closed instead of + * waking with unrestricted tools. */ - private async resolveTerminalWakeCallerToolPolicy( + private async resolveTerminalWakeCallerSendRestrictions( ownerWorkspaceId: string - ): Promise { - const historyResult = await this.historyService.getLastMessages(ownerWorkspaceId, 50); - if (!historyResult.success) { - throw new Error(`history unavailable: ${historyResult.error}`); - } - for (let i = historyResult.data.length - 1; i >= 0; i--) { - const message = historyResult.data[i]; - if (message?.role !== "user") { - continue; - } - if (message.metadata?.toolPolicy != null) { - return message.metadata.toolPolicy; - } - if (message.metadata?.synthetic !== true) { + ): Promise<{ toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean }> { + const state: { + found: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean } | null; + } = { found: null }; + const historyResult = await this.historyService.iterateFullHistory( + ownerWorkspaceId, + "backward", + (messages) => { + for (const message of messages) { + if (message.role !== "user") { + continue; + } + const metadata = message.metadata; + if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { + state.found = { + ...(metadata.toolPolicy != null ? { toolPolicy: metadata.toolPolicy } : {}), + ...(metadata.disableWorkspaceAgents != null + ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } + : {}), + }; + return false; + } + if (metadata?.synthetic !== true) { + state.found = {}; + return false; + } + } return undefined; } + ); + if (!historyResult.success) { + throw new Error(`history unavailable: ${historyResult.error}`); } - return undefined; + return state.found ?? {}; } private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { @@ -8314,23 +8328,12 @@ export class TaskService { isPersistentChildContinuation ? record.workspaceId : notification.sourceId ); } - // Workflow prompts are validated against history well before the send is admitted; a full - // clear in that window truncates history and retires the sidecar, so the send must refuse - // (admissionStale) rather than inject the stale result into the freshly cleared - // conversation. A refused send leaves the notifications pending for the next drain. - const admissionEpoch = this.workspaceService.getContextMutationEpoch(ownerWorkspaceId); - const sendAdmissionStale = () => - this.workspaceService.getContextMutationEpoch(ownerWorkspaceId) !== admissionEpoch; const workflowNotifications = pending.filter( (notification) => notification.sourceKind === "workflow_run" ); const deliverableWorkflowNotificationIds = new Set(); const promptSections: string[] = []; - // Persisted onto the accepted row as consumption provenance (see - // MuxMetadata.deliveredWorkflowRunIds): after a crash between acceptance and the outbox - // delivery mark, restart recovery recognizes the row and does not replay these results. - const deliveredWorkflowRunIds: string[] = []; if (publicAwaitIds.length > 0) { promptSections.push(buildCompletedWorkspaceTurnPrompt(publicAwaitIds)); } @@ -8360,7 +8363,6 @@ export class TaskService { continue; } deliverableWorkflowNotificationIds.add(notification.id); - deliveredWorkflowRunIds.push(notification.sourceId); promptSections.push(workflowPrompt.prompt); } @@ -8403,9 +8405,9 @@ export class TaskService { // synthetic send starts a fresh turn, and omitting the policy would let a workflow wake // regain tools the caller disabled (with attacker-influenced workflow output choosing the // timing). The agent-level policy recomposes from agentId at send resolution. - let wakeToolPolicy: ToolPolicy | undefined; + let wakeRestrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean }; try { - wakeToolPolicy = await this.resolveTerminalWakeCallerToolPolicy(ownerWorkspaceId); + wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); } catch (error: unknown) { // Fail closed: an unknown policy must not fall back to unrestricted tools. log.warn("Deferring terminal wake; caller tool policy unavailable", { @@ -8421,7 +8423,8 @@ export class TaskService { agentId: resumeOptions.agentId, thinkingLevel: resumeOptions.thinkingLevel, reasoningMode: resumeOptions.reasoningMode, - ...(wakeToolPolicy != null ? { toolPolicy: wakeToolPolicy } : {}), + ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), + ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { @@ -8451,14 +8454,7 @@ export class TaskService { prompt, sendOptions, // Synthetic, idle-only auto-resume — same flags as the active-work auto-resume path. - { - skipAutoResumeReset: true, - synthetic: true, - agentInitiated: true, - requireIdle: true, - admissionStale: sendAdmissionStale, - deliveredWorkflowRunIds, - } + { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, requireIdle: true } ); if (!sendResult.success && isWorkspaceBusyIdleOnlySend(sendResult.error)) { @@ -8480,8 +8476,6 @@ export class TaskService { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, - admissionStale: sendAdmissionStale, - deliveredWorkflowRunIds, onCanceled: () => { this.scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId); }, @@ -12587,21 +12581,10 @@ export class TaskService { return true; } - let referencedWorkflowRunIds: string[]; - try { - referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - record.workspaceId, - [] - ); - } catch (error: unknown) { - // Unreadable sidecar: assume blockers exist so the deferred turn is not finalized while - // a kernel workflow may still be running; the next evaluation retries. - log.warn("Deferring workspace-turn blocker check; sidecar unreadable", { - workspaceId: record.workspaceId, - error, - }); - return true; - } + const referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + record.workspaceId, + [] + ); if ( (await this.listActiveBackgroundWorkflowRunIds(record.workspaceId, referencedWorkflowRunIds)) .length > 0 @@ -14218,22 +14201,11 @@ export class TaskService { taskIndex, workspaceId ); - let referencedWorkflowRunIds: string[]; - try { - referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - workspaceId, - event.parts, - event.messageId - ); - } catch (error: unknown) { - // Unreadable sidecar: neither finalize the turn nor nudge the model about unknown - // runs; leave the stream-end unhandled so deferred recovery re-evaluates later. - log.warn("Skipping parent stream-end workflow reconciliation; sidecar unreadable", { - workspaceId, - error, - }); - return; - } + const referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + workspaceId, + event.parts, + event.messageId + ); let activeWorkflowRunIds = await this.listActiveBackgroundWorkflowRunIds( workspaceId, referencedWorkflowRunIds @@ -14520,22 +14492,11 @@ export class TaskService { return; } - let taskReferencedWorkflowRunIds: string[]; - try { - taskReferencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - workspaceId, - event.parts, - event.messageId - ); - } catch (error: unknown) { - // Unreadable sidecar: defer report finalization like an active blocker instead of - // publishing while a kernel workflow may still be running. - log.warn("Deferring task finalization; sidecar unreadable", { workspaceId, error }); - if (status === "awaiting_report") { - await this.setTaskStatus(workspaceId, "running"); - } - return; - } + const taskReferencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + workspaceId, + event.parts, + event.messageId + ); const activeTaskWorkflowRunIds = await this.listActiveBackgroundWorkflowRunIds( workspaceId, taskReferencedWorkflowRunIds diff --git a/src/node/services/tools/toolUtils.test.ts b/src/node/services/tools/toolUtils.test.ts deleted file mode 100644 index 9785a1ce04b..00000000000 --- a/src/node/services/tools/toolUtils.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -import * as fs from "node:fs/promises"; -import * as os from "node:os"; -import * as path from "node:path"; - -import { describe, expect, test } from "bun:test"; - -import type { ToolConfiguration } from "@/common/utils/tools/tools"; -import { - readAgentWorkflowRunReferences, - recordAgentWorkflowRunReference, -} from "@/node/services/agentWorkflowRunReferences"; -import { recordBackgroundWorkflowRunReference } from "@/node/services/tools/toolUtils"; - -describe("recordBackgroundWorkflowRunReference", () => { - test("retries a failed provenance record until storage recovers", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-record-")); - try { - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_existing", - createdAtMs: 1_000, - }); - const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); - // Unreadable at record time: the tool has already returned by the time storage recovers, - // and an untouched active run never hits a natural re-record site, so only the bounded - // background retry can persist provenance for the terminal wake. - await fs.chmod(filePath, 0o000); - await recordBackgroundWorkflowRunReference( - { workspaceSessionDir } as unknown as ToolConfiguration, - "wfr_retry", - 2_000, - [25, 25, 25] - ); - await fs.chmod(filePath, 0o600); - - const deadline = Date.now() + 5_000; - let runIds: string[] = []; - while (Date.now() < deadline) { - runIds = (await readAgentWorkflowRunReferences(workspaceSessionDir)).map( - (reference) => reference.runId - ); - if (runIds.includes("wfr_retry")) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(new Set(runIds)).toEqual(new Set(["wfr_existing", "wfr_retry"])); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("repairs a verified-empty boundary snapshot after a transient read failure", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); - try { - // Launch from a decision-free history whose boundary read fails once: the rediscovery - // entry lands boundary-less, and only the repair can restore the verified-empty (null) - // snapshot the decision-free currentness branch requires. - let calls = 0; - const taskService = { - getWorkflowInvocationBoundaryMessageId: (): Promise => { - calls += 1; - return calls === 1 - ? Promise.reject(new Error("history unavailable")) - : Promise.resolve(null); - }, - }; - await recordBackgroundWorkflowRunReference( - { - workspaceSessionDir, - workspaceId: "ws-boundary-repair", - taskService, - } as unknown as ToolConfiguration, - "wfr_boundary_repair", - 2_000, - [25, 25, 25] - ); - let reference: { afterBoundaryMessageId?: string | null } | undefined; - const deadline = Date.now() + 5_000; - while (Date.now() < deadline) { - [reference] = await readAgentWorkflowRunReferences(workspaceSessionDir); - if (reference?.afterBoundaryMessageId === null) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(reference).toMatchObject({ - runId: "wfr_boundary_repair", - afterBoundaryMessageId: null, - }); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("boundary repair waits for the record retry to land the entry", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); - try { - // Both the boundary read AND the initial sidecar write fail: the reference does not - // exist when the repair first fires. A missing entry must stay retryable, or the record - // retry that lands later creates a permanently boundary-less reference. - let calls = 0; - const taskService = { - getWorkflowInvocationBoundaryMessageId: (): Promise => { - calls += 1; - return calls === 1 - ? Promise.reject(new Error("history unavailable")) - : Promise.resolve(null); - }, - }; - const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_seed", - createdAtMs: 1_000, - }); - await fs.chmod(filePath, 0o000); - await recordBackgroundWorkflowRunReference( - { - workspaceSessionDir, - workspaceId: "ws-boundary-late", - taskService, - } as unknown as ToolConfiguration, - "wfr_boundary_late", - 2_000, - [40, 40, 40, 40, 40] - ); - // Storage recovers only after the repair has fired at least once against the missing - // entry; the record retry then lands it and a later repair attempt patches null. - await new Promise((resolve) => setTimeout(resolve, 60)); - await fs.chmod(filePath, 0o600); - - const deadline = Date.now() + 5_000; - let boundary: string | null | undefined; - while (Date.now() < deadline) { - boundary = (await readAgentWorkflowRunReferences(workspaceSessionDir)).find( - (reference) => reference.runId === "wfr_boundary_late" - )?.afterBoundaryMessageId; - if (boundary === null) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(boundary).toBe(null); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("keeps the entry boundary-less when a decision row exists at repair time", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); - try { - // A decision row seen at repair time may postdate the launch; persisting it would - // overclaim currentness, so the entry must stay boundary-less and fail safe. - let calls = 0; - const taskService = { - getWorkflowInvocationBoundaryMessageId: (): Promise => { - calls += 1; - return calls === 1 - ? Promise.reject(new Error("history unavailable")) - : Promise.resolve("manual-user"); - }, - }; - await recordBackgroundWorkflowRunReference( - { - workspaceSessionDir, - workspaceId: "ws-boundary-unsafe", - taskService, - } as unknown as ToolConfiguration, - "wfr_boundary_unsafe", - 2_000, - [25] - ); - await new Promise((resolve) => setTimeout(resolve, 300)); - const references = await readAgentWorkflowRunReferences(workspaceSessionDir); - expect(references).toHaveLength(1); - expect(references[0]?.runId).toBe("wfr_boundary_unsafe"); - expect(references[0]?.afterBoundaryMessageId).toBeUndefined(); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); -}); diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 89e095dce18..a56f4574762 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -7,16 +7,7 @@ import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkspaceChatMessage } from "@/common/orpc/types"; import type { ToolConfiguration } from "@/common/utils/tools/tools"; -import { - SIDECAR_MAINTENANCE_RETRY_DELAYS_MS, - getSidecarLifecycleGeneration, - readAgentWorkflowRunReferences, - recordAgentWorkflowRunReference, - registerSidecarMaintenanceTimer, - scheduleAgentWorkflowRunReferenceRecordRetry, - takeSidecarMaintenanceTimer, - trackSidecarMaintenanceWrite, -} from "@/node/services/agentWorkflowRunReferences"; +import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import { log } from "@/node/services/log"; import type { TaskService } from "@/node/services/taskService"; @@ -88,81 +79,6 @@ export async function emitWorkflowRunAttachedEvent(input: { await input.config.emitChatEvent(event); } -/** - * Repair a missing boundary snapshot in the background. A kernel launch from a decision-free - * history whose record-time boundary read failed persists a boundary-less entry, but the - * decision-free currentness branch accepts only an explicit verified-empty (null) snapshot, - * so without repair the run's terminal wake is permanently superseded once storage recovers. - * Rows never disappear outside a full clear (which retires the sidecar), so a history still - * verified-empty at repair time was also empty at launch and null is faithful launch - * provenance; a decision row seen at repair time may postdate the launch, so persisting it - * would overclaim currentness and the entry stays boundary-less (fail safe). - */ -function scheduleBoundarySnapshotRepair(input: { - workspaceSessionDir: string; - runId: string; - createdAtMs: number; - getBoundary: () => Promise; - retryDelaysMs?: readonly number[] | null; - attempt?: number; - /** Chain state: the lifecycle generation captured when the chain was first scheduled. */ - lifecycleGeneration?: number; -}): void { - const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; - const attempt = input.attempt ?? 0; - const delayMs = retryDelaysMs[attempt]; - if (delayMs == null) { - log.error("Giving up on workflow boundary snapshot repair after retries", { - runId: input.runId, - attempts: attempt, - }); - return; - } - const key = `boundary:${input.runId}`; - const lifecycleGeneration = - input.lifecycleGeneration ?? getSidecarLifecycleGeneration(input.workspaceSessionDir); - const timer = setTimeout(() => { - if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { - return; - } - const work = (async () => { - // The reference may not exist yet: when the initial write also failed, the independent - // record-retry chain lands it later. A missing entry is retryable, not a satisfied - // no-op, or that later record would create a permanently boundary-less reference; - // lifecycle cancellation kills this chain when the reference was retired instead. - const references = await readAgentWorkflowRunReferences(input.workspaceSessionDir); - const entry = references.find((reference) => reference.runId === input.runId); - if (entry == null) { - throw new Error("reference not recorded yet"); - } - if (entry.afterBoundaryMessageId !== undefined) { - return; - } - const boundary = await input.getBoundary(); - if (boundary !== null) { - return; - } - await recordAgentWorkflowRunReference({ - workspaceSessionDir: input.workspaceSessionDir, - runId: input.runId, - createdAtMs: input.createdAtMs, - afterBoundaryMessageId: null, - onlyIfBoundaryAbsent: true, - }); - })().catch((error: unknown) => { - log.warn("Workflow boundary snapshot repair failed", { - runId: input.runId, - attempt: attempt + 1, - error: getErrorMessage(error), - }); - scheduleBoundarySnapshotRepair({ ...input, attempt: attempt + 1, lifecycleGeneration }); - }); - trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); - }, delayMs); - timer.unref?.(); - registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer, lifecycleGeneration); -} - /** * Persist agent provenance for a workflow run that outlives the current turn (background * start/resume, or a foreground run that backgrounded itself). TaskService reads these @@ -173,8 +89,7 @@ function scheduleBoundarySnapshotRepair(input: { export async function recordBackgroundWorkflowRunReference( config: ToolConfiguration, runId: string, - createdAtMs: number, - retryDelaysMs?: readonly number[] | null + createdAtMs: number ): Promise { const workspaceSessionDir = config.workspaceSessionDir; if (workspaceSessionDir == null || workspaceSessionDir.length === 0) { @@ -202,18 +117,6 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); - const workspaceId = config.workspaceId; - const getBoundaryMessageId = - taskService?.getWorkflowInvocationBoundaryMessageId?.bind(taskService); - if (workspaceId != null && getBoundaryMessageId != null) { - scheduleBoundarySnapshotRepair({ - workspaceSessionDir, - runId, - createdAtMs, - getBoundary: () => getBoundaryMessageId(workspaceId, runId), - retryDelaysMs, - }); - } } } @@ -229,12 +132,5 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId, - createdAtMs, - retryDelaysMs, - ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), - }); } } diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index 63df2868bc6..79d9826f211 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -329,6 +329,44 @@ describe("workflow_resume tool", () => { }); }); + test("consumes the foreground terminal result even when the refresh read fails", async () => { + using tempDir = new TestTempDir("test-workflow-resume-refresh-failure-consume"); + // WorkflowService.getRun collapses transient read failures to null. The terminal result is + // still returned to the model, so consumption must derive from the dispatch result or the + // pending terminal attention would re-inject it later. + let getRunCalls = 0; + const workflowService = buildWorkflowService({ + getRun: mock(async () => { + getRunCalls += 1; + return getRunCalls === 1 ? buildRun() : null; + }), + resumeRun: mock(async () => ({ + runId: "wfr_resume_me", + status: "completed" as const, + result: { reportMarkdown: "resumed" }, + })), + }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + const result = await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: null }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + expect(result).toMatchObject({ status: "completed", runId: "wfr_resume_me", mode: "resume" }); + }); + test("rejects default resume of a failed run with checkpoint retry guidance", async () => { using tempDir = new TestTempDir("test-workflow-resume-failed"); const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index 649c1ca90bf..57870d7d78d 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -4,7 +4,7 @@ import { getErrorMessage } from "@/common/utils/errors"; import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; import { isTerminalWorkflowRunStatus, type WorkflowRunRecord } from "@/common/types/workflow"; import { getWorkflowCheckpointRetryEligibility } from "@/common/utils/workflowRetryEligibility"; -import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; +import { WorkflowRunRecordSchema, WorkflowRunStatusSchema } from "@/common/orpc/schemas"; import { WorkflowResumeToolResultSchema, TOOL_DEFINITIONS, @@ -158,7 +158,9 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // workflow_resume part in history, so the history-walk consumption predicates cannot see // that this turn already received the terminal result. Persist consumption durably so the // terminal-attention drain never re-delivers it. - const markTerminalAttentionConsumed = async (terminalRun: WorkflowRunRecord) => { + const markTerminalAttentionConsumed = async ( + terminalRun: Pick + ) => { if (!isTerminalWorkflowRunStatus(terminalRun.status)) { return; } @@ -249,8 +251,14 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Foreground only: a background dispatch can still observe the stale pre-dispatch terminal // status, and consuming it would tombstone the retried run's future terminal wake. - if (!isBackgroundDispatch && refreshedRun != null) { - await markTerminalAttentionConsumed(refreshedRun); + // Consumption derives from the dispatch result itself, not the refreshed record: the + // terminal result is returned to the model below even when the refresh read fails, and + // skipping the tombstone would let the pending terminal attention re-inject it later. + if (!isBackgroundDispatch) { + const dispatchedStatus = WorkflowRunStatusSchema.safeParse(dispatched.status); + if (dispatchedStatus.success) { + await markTerminalAttentionConsumed({ id: runId, status: dispatchedStatus.data }); + } } return parseToolResult( diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index e440984e50b..970ccea6d11 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -8,7 +8,6 @@ import { targetMutationLockFilePath, withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; -import { scheduleAgentWorkflowRunReferenceRecordRetry } from "@/node/services/agentWorkflowRunReferences"; import { acquireProcessFileLock, getProcessBirth } from "@/node/utils/concurrency/fileLock"; import { healRemovalTombstonesForRegisteredWorkspaces, @@ -22,39 +21,6 @@ import { } from "./workspaceRemoval"; describe("workspaceRemoval", () => { - test("removal cancels pending sidecar record retries so they cannot recreate the session dir", async () => { - using tmp = new DisposableTempDir("workspace-removal-sidecar"); - const rootDir = path.join(tmp.path, "xum-home"); - const workspaceId = "ws-removal-sidecar"; - const sessionDir = path.join(rootDir, "sessions", workspaceId); - await fsPromises.mkdir(sessionDir, { recursive: true }); - - // A detached provenance retry armed before removal; on fire it would mkdir the session - // directory back into existence after the deletion below. - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir: sessionDir, - runId: "wfr_removal_race", - createdAtMs: 1_000, - afterBoundaryMessageId: null, - retryDelaysMs: [100], - }); - - await removeSessionDirUnderMemoryLocks({ - rootDir, - sessionDir, - workspaceId, - attemptId: "test-attempt-sidecar", - }); - await new Promise((resolve) => setTimeout(resolve, 350)); - - expect( - await fsPromises.access(sessionDir).then( - () => true, - () => false - ) - ).toBe(false); - }); - test("deletion waits for a live memory writer, then tombstones and deletes (r61)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index c2fe3b8c4d5..154c7311626 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -27,7 +27,6 @@ * foreign backend's still-running consolidation refuse arbitrarily late. */ -import { cancelAgentWorkflowRunReferenceMaintenance } from "@/node/services/agentWorkflowRunReferences"; import crypto from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; @@ -218,10 +217,6 @@ export async function removeSessionDirUnderMemoryLocks(args: { // deleted directory cannot be recreated by a late mutation or // journal append. await publishTombstone(); - // Detached sidecar maintenance (provenance record retries) would survive removal and - // recreate the deleted session directory when its timer fires; cancel and drain it - // like the other session writers before the directory goes away. - await cancelAgentWorkflowRunReferenceMaintenance(args.sessionDir); await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); } ); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index e92a56be1f6..daac7889e48 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6266,78 +6266,6 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("a failed reference retirement aborts the clear before deleting history", async () => { - const { config, historyService, cleanup } = await createTestHistoryService(); - const workspaceId = "workflow-currentness-retire-emit"; - const projectPath = path.join(config.rootDir, "project"); - try { - await config.addWorkspace(projectPath, { - id: workspaceId, - name: "workflow-currentness-retire-emit", - projectName: "project", - projectPath, - runtimeConfig: { type: "local" }, - }); - const workspaceService = createWorkspaceServiceForTest({ - config, - historyService, - aiService: createMockAIService({ - stopStream: mock(() => Promise.resolve(Ok(undefined))), - }), - extensionMetadata: new ExtensionMetadataService( - path.join(config.rootDir, "extensionMetadata.json") - ), - initStateManager: { - ...mockInitStateManager, - off: mock(() => undefined as unknown as InitStateManager), - } as unknown as InitStateManager, - }); - - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user", "user", "hello", { timestamp: 1_000 }) - ); - const sessionAccessor = workspaceService as unknown as { - getOrCreateSession(id: string): { emitChatEvent(message: unknown): void }; - }; - const session = sessionAccessor.getOrCreateSession(workspaceId); - const emitSpy = spyOn(session, "emitChatEvent"); - // A read-only session directory makes retirement fail (a directory at the sidecar path - // now self-heals instead). Retirement runs BEFORE the truncation, so the failure must - // abort the whole clear: the transcript survives, the renderer sees no deletion, and no - // crash window exists in which the transcript is gone while the sidecar lives on. - const sessionDir = config.getSessionDir(workspaceId); - await recordAgentWorkflowRunReference({ - workspaceSessionDir: sessionDir, - runId: "wfr_retirement_blocked", - createdAtMs: 1_150, - afterBoundaryMessageId: "manual-user", - }); - await fsPromises.chmod(sessionDir, 0o555); - try { - const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); - expect(clearResult.success).toBe(false); - if (!clearResult.success) { - expect(clearResult.error).toContain("workflow run references"); - } - expect( - emitSpy.mock.calls.some((call) => (call[0] as { type?: string }).type === "delete") - ).toBe(false); - const survivingHistory = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(survivingHistory.success).toBe(true); - if (survivingHistory.success) { - expect(survivingHistory.data).toHaveLength(1); - } - } finally { - emitSpy.mockRestore(); - await fsPromises.chmod(sessionDir, 0o755); - } - workspaceService.disposeSession(workspaceId); - } finally { - await cleanup(); - } - }); - test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-coalesced"; @@ -6374,47 +6302,44 @@ describe("WorkspaceService workflow invocation events", () => { }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - // A synthetic row whose TEXT reproduces a result payload must not count: synthetic rows - // can carry user-controlled content (e.g. a heartbeat body), and a quoted run ID must - // not spoof consumption and suppress the real wake. + // Another run's payload quoting nothing about this run must not count as consumption. await historyService.appendToHistory( workspaceId, createMuxMessage( - "coalesced-spoof", + "coalesced-other", "user", buildWorkflowResultContextMessage({ - rawCommand: "workflow_run research.js", - name: "research.js", - runId, + rawCommand: "workflow_run other.js", + name: "other.js", + runId: "wfr_currentness_other", status: "completed", - result: { reportMarkdown: "spoofed" }, + result: { reportMarkdown: "other done" }, run: null, }), - { timestamp: 1_240, synthetic: true } + { timestamp: 1_250, synthetic: true } ) ); - // Another run's persisted provenance must not count for this run either. - await historyService.appendToHistory( - workspaceId, - createMuxMessage("coalesced-other", "user", "results delivered", { - timestamp: 1_250, - synthetic: true, - deliveredWorkflowRunIds: ["wfr_currentness_other"], - }) - ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - // The drain persists deliveredWorkflowRunIds onto the accepted row. After a crash - // between durable acceptance and the outbox delivery mark, this provenance is the only + // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a + // crash between durable acceptance and the outbox delivery mark, this row is the only // evidence the result already reached history; it must read as consumption or restart // recovery injects the same terminal result again. await historyService.appendToHistory( workspaceId, - createMuxMessage("coalesced-result", "user", "results delivered", { - timestamp: 1_300, - synthetic: true, - deliveredWorkflowRunIds: [runId], - }) + createMuxMessage( + "coalesced-result", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run research.js", + name: "research.js", + runId, + status: "completed", + result: { reportMarkdown: "done" }, + run: null, + }), + { timestamp: 1_300, synthetic: true } + ) ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); workspaceService.disposeSession(workspaceId); @@ -6423,58 +6348,6 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("a corrupt sidecar directory self-heals during a full clear", async () => { - const { config, historyService, cleanup } = await createTestHistoryService(); - const workspaceId = "workflow-currentness-selfheal"; - const projectPath = path.join(config.rootDir, "project"); - try { - await config.addWorkspace(projectPath, { - id: workspaceId, - name: "workflow-currentness-selfheal", - projectName: "project", - projectPath, - runtimeConfig: { type: "local" }, - }); - const workspaceService = createWorkspaceServiceForTest({ - config, - historyService, - aiService: createMockAIService({ - stopStream: mock(() => Promise.resolve(Ok(undefined))), - }), - extensionMetadata: new ExtensionMetadataService( - path.join(config.rootDir, "extensionMetadata.json") - ), - initStateManager: { - ...mockInitStateManager, - off: mock(() => undefined as unknown as InitStateManager), - } as unknown as InitStateManager, - }); - - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user", "user", "hello", { timestamp: 1_000 }) - ); - // A directory at the known sidecar path is corruption (reads fail with EISDIR). The - // full clear must remove it and succeed, not fail identically on every retry and leave - // the workspace impossible to clear without manual session-storage repair. - const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); - await fsPromises.mkdir(sidecarPath); - await fsPromises.writeFile(path.join(sidecarPath, "junk.txt"), "junk"); - - const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); - expect(clearResult.success).toBe(true); - expect( - await fsPromises.access(sidecarPath).then( - () => true, - () => false - ) - ).toBe(false); - workspaceService.disposeSession(workspaceId); - } finally { - await cleanup(); - } - }); - test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 3b6e9b037b6..bb95cabd55c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -204,6 +204,7 @@ import { } from "@/node/services/workflows/workflowArchiveAdmission"; import { WORKFLOW_RESULT_METADATA_TYPE, + textContainsWorkflowResultPayload, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowRunCardMessage, @@ -477,16 +478,18 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) /** * The terminal-attention drain delivers workflow results as one synthetic user prompt that may - * coalesce several runs. If a crash lands between the send's durable acceptance and the outbox - * delivery mark, restart recovery drains the notification again; recognizing the accepted row - * as consumption is what suppresses the replay. Recognition uses the drain's persisted - * provenance (MuxMetadata.deliveredWorkflowRunIds), never the row's text: synthetic rows can - * carry user-controlled content (e.g. a heartbeat body), and a quoted run ID must not spoof - * consumption and suppress a real wake. + * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands + * between the send's durable acceptance and the outbox delivery mark, restart recovery drains + * the notification again; recognizing the accepted row as consumption is what suppresses the + * replay. Only synthetic rows qualify: a manual user message is a supersession boundary and is + * classified before this check runs. */ function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { - return ( - message.role === "user" && message.metadata?.deliveredWorkflowRunIds?.includes(runId) === true + if (message.role !== "user" || message.metadata?.synthetic !== true) { + return false; + } + return message.parts.some( + (part) => part.type === "text" && textContainsWorkflowResultPayload(part.text, runId) ); } @@ -3803,14 +3806,6 @@ export class WorkspaceService extends EventEmitter { } } - /** - * Current context-mutation epoch (see contextMutationEpochs). Lets the terminal-attention - * drain refuse a send whose workflow prompt was validated before a full clear committed. - */ - getContextMutationEpoch(workspaceId: string): number { - return this.contextMutationEpochs.get(workspaceId) ?? 0; - } - /** r41: mark a context-discarding mutation as durably committed (see contextMutationEpochs). */ private advanceContextMutationEpoch(workspaceId: string): void { this.contextMutationEpochs.set( @@ -11251,8 +11246,6 @@ export class WorkspaceService extends EventEmitter { goalId?: string; /** Force Copilot billing classification to "agent" for internal sends. */ agentInitiated?: boolean; - /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ - deliveredWorkflowRunIds?: string[]; onAccepted?: () => Promise | void; onCanceled?: (reason: string) => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; @@ -11515,7 +11508,6 @@ export class WorkspaceService extends EventEmitter { return await session.sendMessage(message, normalizedOptions, { synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, - deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, goalKind: internal?.goalKind, goalId: internal?.goalId, cancelState: internal?.cancelState, @@ -11660,7 +11652,6 @@ export class WorkspaceService extends EventEmitter { { synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, - deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, authoredAtMs, workspaceTurnContinuation: internal?.workspaceTurnContinuation, dedupeKey: internal?.queueDedupeKey, @@ -11773,7 +11764,6 @@ export class WorkspaceService extends EventEmitter { onTurnAdmissionCommitted: () => sessionInvisiblePreflight.release(), synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, - deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, goalKind: internal?.goalKind, goalId: internal?.goalId, goalContinuation: internal?.goalContinuation, @@ -12912,23 +12902,6 @@ export class WorkspaceService extends EventEmitter { ); } } - // Kernel workflow run references belong to the transcript being discarded: a verified-empty - // (null) boundary snapshot recorded before the clear is indistinguishable from one recorded - // after it. Retire them durably BEFORE the truncation (like the r41 retry discard above) so - // no crash window exists in which the transcript is gone but the sidecar survives; a crash - // here can only lose a wake for a still-intact conversation, never inject a pre-clear - // result into the cleared one. A post-clear resume re-records provenance, and pending - // record retries are cancelled with the sidecar. - if (isFullClear) { - try { - await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error) { - return Err( - `Cannot clear history: stale workflow run references could not be retired ` + - `(${getErrorMessage(error)}); retry once the session storage is writable.` - ); - } - } if (effectivePercentage > 0) { session?.clearUsageState(); } @@ -12947,6 +12920,22 @@ export class WorkspaceService extends EventEmitter { // admitted afterwards (their content references the discarded context). if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); + // Kernel workflow run references belong to the cleared conversation: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one + // recorded after it, so a surviving reference could inject a pre-clear workflow result + // into the fresh conversation. Retire them immediately after the truncation commits, + // before any later post-clear step that can fail and return early (goal acknowledgment, + // carryover discard), or the stale reference would survive the committed clear. A + // post-clear resume re-records provenance. + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `History was cleared, but stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + + `result into the cleared conversation; retry once the session storage is writable.` + ); + } } // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the From d35491c6c96253abbbd530011ba746372df34e9c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 05:47:24 +0000 Subject: [PATCH 20/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20resolve=20the=20wak?= =?UTF-8?q?e's=20agent=20identity=20with=20an=20unbounded=20history=20walk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveParentAutoResumeOptions read only the last 20 rows to find the newest agent-bearing assistant row; agent-less synthetic rows (the drain appends one per pending sub-agent report) could push that row out of the window and silently recompose terminal-wake sends from the exec fallback, lifting a restricted agent's tool policy. Walk full history backward instead, matching the round-16 caller-restriction fix; the exec fallback now applies only to histories that never had an agent turn. --- src/node/services/taskService.test.ts | 7 ++++- src/node/services/taskService.ts | 37 +++++++++++++++------------ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 8367cc47dc3..2df7479c391 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6433,7 +6433,7 @@ describe("TaskService", () => { expect(liftedOptions.toolPolicy).toBeUndefined(); }); - test("wake restriction restore walks past a long synthetic tail and carries the agent disable flag", async () => { + test("wake restoration walks a long agent-less tail: caller policy, agent identity, disable flag", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; @@ -6472,6 +6472,10 @@ describe("TaskService", () => { disableWorkspaceAgents: true, }) ); + await historyService.appendToHistory( + parentId, + createMuxMessage("agent-turn", "assistant", "on it", { timestamp: 1_000, agentId: "plan" }) + ); // A tail longer than any bounded history read: the launch turn's restrictions must still // be found, not silently lifted once enough rows accumulate after the manual turn. for (let i = 0; i < 60; i++) { @@ -6488,6 +6492,7 @@ describe("TaskService", () => { await flushTerminalAttentionDrains(taskService); expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index f2177fde1b5..6b9992f0d17 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2308,26 +2308,29 @@ export class TaskService { // Compaction is internal bookkeeping, not an identity for resuming user work. let agentId = hint?.agentId === "compact" ? undefined : hint?.agentId; - // Durable history preserves the parent identity across process restarts. + // Durable history preserves the parent identity across process restarts. The walk is + // unbounded: synthetic rows without an agent identity (drain-appended sub-agent reports, + // heartbeat scaffolding) can push the newest agent-bearing assistant row past any fixed + // tail, and a truncated read would silently recompose terminal-wake sends from the exec + // fallback, lifting a restricted agent's tool policy. if (!agentId) { - try { - const historyResult = await this.historyService.getLastMessages(parentWorkspaceId, 20); - if (historyResult.success) { - for (let i = historyResult.data.length - 1; i >= 0; i--) { - const msg = historyResult.data[i]; - if ( - msg?.role === "assistant" && - msg.metadata?.agentId && - msg.metadata.agentId !== "compact" - ) { - agentId = msg.metadata.agentId; - break; - } + const found: { agentId?: string } = {}; + await this.historyService.iterateFullHistory(parentWorkspaceId, "backward", (messages) => { + for (const msg of messages) { + if ( + msg.role === "assistant" && + msg.metadata?.agentId && + msg.metadata.agentId !== "compact" + ) { + found.agentId = msg.metadata.agentId; + return false; } } - } catch { - // Best-effort; fall through to defaults - } + return undefined; + }); + // A failed read falls through to defaults (best-effort); the terminal drain separately + // fails closed on unreadable history via resolveTerminalWakeCallerSendRestrictions. + agentId = found.agentId; } // 3) Default From 07923962c415697db7839fcbe25b043a8164e63d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:09:24 +0000 Subject: [PATCH 21/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20sanitize=20persiste?= =?UTF-8?q?d=20wake=20restrictions=20before=20restoring=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A malformed persisted toolPolicy (object, number) copied into the synthetic send throws during agent resolution before the wake is accepted, leaving the terminal notification pending and every later drain blocked on the same corrupt row. Parse the field with ToolPolicySchema and drop invalid values (the intact disable flag on the same row still applies), per the self-healing doctrine for corrupt history rows. The agent-identity walk gets the same-class guard: only non-empty string agentId metadata defines the resume identity. --- src/node/services/taskService.test.ts | 56 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 20 ++++++++-- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 2df7479c391..930dc2d4918 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6498,6 +6498,62 @@ describe("TaskService", () => { }); }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_policy_corrupt"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // Persisted metadata is untrusted disk state: a corrupt toolPolicy shape must be dropped + // (not copied into the send, where it would throw during resolution and permanently block + // the wake), while the intact disable flag on the same row still applies. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-corrupt", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: { bogus: true }, + disableWorkspaceAgents: true, + } as unknown as Parameters[3]) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const options = sendMessage.mock.calls[0]?.[2] as { + toolPolicy?: unknown; + disableWorkspaceAgents?: unknown; + }; + expect(options.toolPolicy).toBeUndefined(); + expect(options.disableWorkspaceAgents).toBe(true); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 6b9992f0d17..b283bb60934 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -118,6 +118,7 @@ import { import type { ProjectRef, WorkspaceMetadata } from "@/common/types/workspace"; import { getRuntimeType } from "@/node/runtime/initHook"; import { AgentIdSchema } from "@/common/orpc/schemas"; +import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import type { AgentDefinitionScope } from "@/common/types/agentDefinition"; import { normalizeAgentId, @@ -2319,7 +2320,8 @@ export class TaskService { for (const msg of messages) { if ( msg.role === "assistant" && - msg.metadata?.agentId && + typeof msg.metadata?.agentId === "string" && + msg.metadata.agentId.length > 0 && msg.metadata.agentId !== "compact" ) { found.agentId = msg.metadata.agentId; @@ -7937,9 +7939,21 @@ export class TaskService { } const metadata = message.metadata; if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { + // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep + // inside send resolution and leave the wake permanently blocked on the same corrupt + // row. Sanitize instead of trusting the JSON shape; an unparseable policy restores + // nothing while a valid disable flag still applies (self-healing doctrine). + const parsedPolicy = + metadata.toolPolicy != null ? ToolPolicySchema.safeParse(metadata.toolPolicy) : null; + if (parsedPolicy != null && !parsedPolicy.success) { + log.warn("Ignoring malformed persisted toolPolicy on terminal wake", { + ownerWorkspaceId, + messageId: message.id, + }); + } state.found = { - ...(metadata.toolPolicy != null ? { toolPolicy: metadata.toolPolicy } : {}), - ...(metadata.disableWorkspaceAgents != null + ...(parsedPolicy?.success ? { toolPolicy: parsedPolicy.data } : {}), + ...(typeof metadata.disableWorkspaceAgents === "boolean" ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } : {}), }; From 92806142997e6304d7ccd29722862671bfee68e0 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:36:22 +0000 Subject: [PATCH 22/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20persist=20kernel=20?= =?UTF-8?q?workflow=20provenance=20before=20the=20runner=20can=20reach=20t?= =?UTF-8?q?erminal=20state?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both workflow tools recorded the sidecar reference after the dispatch returned, and background execution starts at lease acquisition, so a fast run (or a process exit mid-dispatch) could reach terminal state with no top-level invocation part and no sidecar reference; the terminal drain would then permanently supersede its wake. Explicit background launches now record in the awaited onRunCreated hook (run durable, runner not started) and explicit background resumes record before the dispatch; the post-dispatch records remain only for foreground runs that backgrounded themselves, where the outcome is only knowable after dispatch. Test mocks now honor the onRunCreated contract and probe that the reference is durable at dispatch time. --- .../services/tools/workflow_resume.test.ts | 14 ++++++- src/node/services/tools/workflow_resume.ts | 13 +++++- src/node/services/tools/workflow_run.test.ts | 42 ++++++++++++++----- src/node/services/tools/workflow_run.ts | 13 +++++- 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index 79d9826f211..b0d90d1858f 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -169,7 +169,18 @@ describe("workflow_resume tool", () => { test("resumes in background and records an agent workflow run reference", async () => { using tempDir = new TestTempDir("test-workflow-resume-bg"); - const workflowService = buildWorkflowService(); + // The reference must be durable BEFORE the dispatch: background execution starts at lease + // acquisition, and a fast run could otherwise reach terminal state with no provenance. + let referenceDurableAtDispatch = false; + const workflowService = buildWorkflowService({ + resumeRunInBackground: mock(async () => { + const references = await readAgentWorkflowRunReferences(tempDir.path); + referenceDurableAtDispatch = references.some( + (reference) => reference.runId === "wfr_resume_me" + ); + return { runId: "wfr_resume_me", status: "running" as const, result: null }; + }), + }); const tool = createWorkflowResumeTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: false, @@ -187,6 +198,7 @@ describe("workflow_resume tool", () => { projectTrusted: false, }); expect(workflowService.resumeRun).not.toHaveBeenCalled(); + expect(referenceDurableAtDispatch).toBe(true); const references = await readAgentWorkflowRunReferences(tempDir.path); expect(references.map((reference) => reference.runId)).toContain("wfr_resume_me"); expect(result).toMatchObject({ status: "running", runId: "wfr_resume_me", mode: "resume" }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index 57870d7d78d..70042c94b32 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -204,6 +204,15 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) runId, projectTrusted: config.trusted === true, }; + // Provenance must be durable BEFORE the dispatch: background execution starts at lease + // acquisition, and a fast run (or a process exit mid-dispatch) can reach terminal state + // before any post-dispatch write, permanently superseding its wake. A failed dispatch + // leaves the re-recorded reference behind, which is benign: it mirrors what a + // successful resume would persist for a run this turn explicitly re-engaged with, and + // rediscovery filters against the run store. + if (args.run_in_background === true) { + await recordBackgroundWorkflowRunReference(config, runId, invocationStartedAtMs); + } let dispatched: { runId: string; status: string; result: unknown }; try { if (mode === "retry_from_checkpoint") { @@ -234,9 +243,11 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Background-style resumes outlive this turn; persist provenance so the run is // rediscoverable (task_await/task_list) and its terminal result re-engages the agent. + // Explicit background resumes already recorded it pre-dispatch; this covers a + // foreground resume that backgrounded itself. const isBackgroundDispatch = args.run_in_background === true || dispatched.status === "backgrounded"; - if (isBackgroundDispatch) { + if (isBackgroundDispatch && args.run_in_background !== true) { await recordBackgroundWorkflowRunReference(config, runId, invocationStartedAtMs); } diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index 05064896ffa..d49544850c6 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -14,9 +14,14 @@ import { resolveWorkflowScript } from "@/node/services/workflows/workflowScriptR import { TestTempDir, createTestToolConfig, writeProjectSkill } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; import type { TaskService } from "@/node/services/taskService"; +import type { ToolConfiguration } from "@/common/utils/tools/tools"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkflowRunRecord } from "@/common/types/workflow"; +type BackgroundStartInput = Parameters< + NonNullable["startWorkflowInBackground"]> +>[0]; + const mockToolCallOptions: ToolExecutionOptions = { toolCallId: "test-call-id", messages: [], @@ -645,11 +650,23 @@ describe("workflow_run tool", () => { const startWorkflow = mock(async () => { throw new Error("foreground start should not be used"); }); - const startWorkflowInBackground = mock(async () => ({ - runId: "wfr_background", - status: "running" as const, - result: null, - })); + // Faithful to WorkflowService: onRunCreated is awaited at run creation, before the runner + // starts executing. The probe checks the sidecar reference is already durable at that + // point, so a fast run (or a crash) after launch cannot lose its terminal wake. + let referenceDurableBeforeRunnerStart = false; + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_background", + status: "pending", + result: null, + run: null, + }); + const references = await readAgentWorkflowRunReferences(tempDir.path); + referenceDurableBeforeRunnerStart = references.some( + (reference) => reference.runId === "wfr_background" + ); + return { runId: "wfr_background", status: "running" as const, result: null }; + }); const getRun = mock(async () => null); const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); const tool = createWorkflowRunTool({ @@ -668,6 +685,7 @@ describe("workflow_run tool", () => { mockToolCallOptions ); + expect(referenceDurableBeforeRunnerStart).toBe(true); // The reference must persist the invocation-boundary snapshot: currentness compares row // identity, so a reference recorded without it fails safe and the wake is dropped. const references = await readAgentWorkflowRunReferences(tempDir.path); @@ -697,11 +715,15 @@ describe("workflow_run tool", () => { test("records a rediscovery-only reference when the boundary snapshot fails", async () => { using tempDir = new TestTempDir("test-workflow-run-tool-boundary-error"); const scriptPath = await writeWorkflowScript(tempDir.path); - const startWorkflowInBackground = mock(async () => ({ - runId: "wfr_boundary_error", - status: "running" as const, - result: null, - })); + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_boundary_error", + status: "pending", + result: null, + run: null, + }); + return { runId: "wfr_boundary_error", status: "running" as const, result: null }; + }); const getWorkflowInvocationBoundaryMessageId = mock(async () => { throw new Error("history read failed"); }); diff --git a/src/node/services/tools/workflow_run.ts b/src/node/services/tools/workflow_run.ts index 425b379eae8..075089795cc 100644 --- a/src/node/services/tools/workflow_run.ts +++ b/src/node/services/tools/workflow_run.ts @@ -262,6 +262,7 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => } } const createdRun: { id: string | null } = { id: null }; + const invocationStartedAtMs = Date.now(); const startInput = { script, workspaceId, @@ -271,6 +272,12 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => createdRun.id = event.runId; // The run record is durable now, so a concurrent duplicate launch will see it. releaseAdmission?.(); + // Provenance must be durable BEFORE the runner starts: a fast background run (or a + // process exit mid-dispatch) can reach terminal state before any post-dispatch + // write, and a terminal wake with no sidecar reference is permanently superseded. + if (args.run_in_background === true) { + await recordBackgroundWorkflowRunReference(config, event.runId, invocationStartedAtMs); + } await emitWorkflowRunAttachedEvent({ config, workspaceId, @@ -280,7 +287,6 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => }); }, }; - const invocationStartedAtMs = Date.now(); let result: { runId: string; status: string; result: unknown }; try { result = @@ -335,7 +341,10 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => releaseAdmission?.(); } - if (isBackgroundWorkflowResult(args, result.status)) { + // Explicit background launches already recorded provenance in onRunCreated; this covers + // a foreground dispatch that backgrounded itself, where the run ID outcome is only + // knowable post-dispatch. + if (args.run_in_background !== true && isBackgroundWorkflowResult(args, result.status)) { await recordBackgroundWorkflowRunReference(config, result.runId, invocationStartedAtMs); } From 42a596beba53933cb598c76273189d4fa4e0e1de Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:59:37 +0000 Subject: [PATCH 23/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20the=20ca?= =?UTF-8?q?ller=20tool=20policy=20through=20on-send=20compaction=20follow-?= =?UTF-8?q?ups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pickPreservedSendOptions preserved disableWorkspaceAgents but not toolPolicy, so a restricted send that triggered on-send auto-compaction redispatched its follow-up allow-all; with terminal wakes now restoring the caller policy on synthetic sends, that gap let a compacting wake resume unrestricted. Preserve the policy in the durable follow-up and restore it at redispatch behind ToolPolicySchema validation, since the follow-up crosses the same raw JSON persistence boundary as its other fields. --- src/common/types/message.ts | 4 ++++ .../agentSession.autoCompaction.test.ts | 11 +++++++++++ src/node/services/agentSession.ts | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 9b16234a106..db02c4d6f64 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -63,6 +63,7 @@ type PreservedSendOptions = Pick< | "providerOptions" | "experiments" | "disableWorkspaceAgents" + | "toolPolicy" | "strictAgentResolution" | "allowAgentSetGoal" | "skipAiSettingsPersistence" @@ -82,6 +83,9 @@ export function pickPreservedSendOptions(options: SendMessageOptions): Preserved // can persist across restarts and build versions. experiments: withLegacyPtcExclusiveMirror(options.experiments), disableWorkspaceAgents: options.disableWorkspaceAgents, + // Security: a restricted turn (including a terminal-wake send restoring the caller's + // policy) that triggers on-send compaction must not redispatch its follow-up allow-all. + toolPolicy: options.toolPolicy, // Delegated turns with explicit agent overrides must stay loud across the // compaction replay too — dropping this would let the follow-up silently // fall back to exec if the agent vanished in the meantime. diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 3ef773f3396..ca2a53e8281 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -95,10 +95,12 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { getThreshold: mock(() => 0.85), } as unknown as CompactionMonitor; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; const result = await session.sendMessage("please inspect @foo.ts", { model: "openai:gpt-4o", agentId: "exec", disableWorkspaceAgents: true, + toolPolicy: restrictedPolicy, }); expect(result.success).toBe(true); @@ -120,6 +122,15 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { ); expect(persistedCompactionMessage).toBeDefined(); expect(persistedCompactionMessage?.metadata?.disableWorkspaceAgents).toBe(true); + // The durable follow-up must keep the caller's restrictions: the redispatched turn + // reconstructs its options from this persisted request, and dropping the policy there + // would resume the conversation allow-all after compaction. + const followUpContent = + persistedCompactionMessage?.metadata?.muxMetadata?.type === "compaction-request" + ? persistedCompactionMessage.metadata.muxMetadata.parsed.followUpContent + : undefined; + expect(followUpContent?.toolPolicy).toEqual(restrictedPolicy); + expect(followUpContent?.disableWorkspaceAgents).toBe(true); const emittedSnapshot = events.some( (message) => diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4422a49a05e..0b560f406e8 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -44,6 +44,7 @@ import { SendMessageOptionsSchema, SkillNameSchema, } from "@/common/orpc/schemas"; +import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { buildStreamErrorEventData, @@ -7200,6 +7201,23 @@ export class AgentSession { experiments: aliasLegacyPtcExclusive(followUp.experiments), allowAgentSetGoal: followUp.allowAgentSetGoal, disableWorkspaceAgents: followUp.disableWorkspaceAgents, + // Same raw JSON boundary: a persisted follow-up may carry a malformed toolPolicy, and + // restoring it unvalidated would throw during resolution. Invalid values are dropped + // like any corrupt persisted policy (self-healing doctrine); a restricted turn's + // follow-up must otherwise keep its policy instead of redispatching allow-all. + ...(() => { + if (followUp.toolPolicy == null) { + return {}; + } + const parsed = ToolPolicySchema.safeParse(followUp.toolPolicy); + if (!parsed.success) { + log.warn("Ignoring malformed persisted toolPolicy on compaction follow-up", { + workspaceId: this.workspaceId, + }); + return {}; + } + return { toolPolicy: parsed.data }; + })(), // Explicit-agent turns stay loud on the resumed turn too: the requested agent // may have been removed/hidden/disabled while compaction ran. strictAgentResolution: followUp.strictAgentResolution, From c59964ff7f385fd1c239e6265e227530fc6cd48a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 07:24:07 +0000 Subject: [PATCH 24/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20record=20resume=20p?= =?UTF-8?q?rovenance=20only=20after=20the=20dispatch=20restarts=20the=20ru?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-19 pre-dispatch record was wrong for workflow_resume: a resumed run sits in an old failed/interrupted state until the dispatch durably restarts it, and a crash in that window left a fresh, current sidecar reference pointing at the stale terminal state, which startup recovery would deliver as this resume's wake. Record after the dispatch again: the crash window now loses the resume's wake instead of replaying a stale one (fail-safe), and with the process alive delivery always waits for the owner to go idle, by which point the record is durable. workflow_run keeps its onRunCreated record, where the freshly created run has no prior terminal state to replay. --- .../services/tools/workflow_resume.test.ts | 7 ++++--- src/node/services/tools/workflow_resume.ts | 19 +++++++------------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index b0d90d1858f..f6433a16512 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -169,8 +169,9 @@ describe("workflow_resume tool", () => { test("resumes in background and records an agent workflow run reference", async () => { using tempDir = new TestTempDir("test-workflow-resume-bg"); - // The reference must be durable BEFORE the dispatch: background execution starts at lease - // acquisition, and a fast run could otherwise reach terminal state with no provenance. + // The reference must NOT be durable before the dispatch: the run still sits in its old + // terminal state until the dispatch restarts it, and a pre-dispatch reference would let a + // crash in that window replay the stale failure/interruption as a current wake. let referenceDurableAtDispatch = false; const workflowService = buildWorkflowService({ resumeRunInBackground: mock(async () => { @@ -198,7 +199,7 @@ describe("workflow_resume tool", () => { projectTrusted: false, }); expect(workflowService.resumeRun).not.toHaveBeenCalled(); - expect(referenceDurableAtDispatch).toBe(true); + expect(referenceDurableAtDispatch).toBe(false); const references = await readAgentWorkflowRunReferences(tempDir.path); expect(references.map((reference) => reference.runId)).toContain("wfr_resume_me"); expect(result).toMatchObject({ status: "running", runId: "wfr_resume_me", mode: "resume" }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index 70042c94b32..0fac258ed3c 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -204,15 +204,6 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) runId, projectTrusted: config.trusted === true, }; - // Provenance must be durable BEFORE the dispatch: background execution starts at lease - // acquisition, and a fast run (or a process exit mid-dispatch) can reach terminal state - // before any post-dispatch write, permanently superseding its wake. A failed dispatch - // leaves the re-recorded reference behind, which is benign: it mirrors what a - // successful resume would persist for a run this turn explicitly re-engaged with, and - // rediscovery filters against the run store. - if (args.run_in_background === true) { - await recordBackgroundWorkflowRunReference(config, runId, invocationStartedAtMs); - } let dispatched: { runId: string; status: string; result: unknown }; try { if (mode === "retry_from_checkpoint") { @@ -243,11 +234,15 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Background-style resumes outlive this turn; persist provenance so the run is // rediscoverable (task_await/task_list) and its terminal result re-engages the agent. - // Explicit background resumes already recorded it pre-dispatch; this covers a - // foreground resume that backgrounded itself. + // Deliberately AFTER the dispatch, unlike workflow_run's onRunCreated record: the + // resumed run sits in an old terminal state until the dispatch durably restarts it, and + // a pre-dispatch reference would let a crash in that window make startup recovery + // deliver the stale failure/interruption as a current wake. Recording late fails safe + // instead (a crash loses this resume's wake); with the process alive, delivery always + // waits for the owner to go idle, by which point this record is durable. const isBackgroundDispatch = args.run_in_background === true || dispatched.status === "backgrounded"; - if (isBackgroundDispatch && args.run_in_background !== true) { + if (isBackgroundDispatch) { await recordBackgroundWorkflowRunReference(config, runId, invocationStartedAtMs); } From 2befd06b0d71ae4086c485087e94fb5100f22daf Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 07:34:21 +0000 Subject: [PATCH 25/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20restore=20the=20str?= =?UTF-8?q?ict-agent=20pin=20on=20terminal=20wakes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An explicit agent override persists strictAgentResolution in the manual row's retry snapshot, and startup retries and compaction follow-ups both restore it, but the terminal-wake send dropped it: if the pinned agent definition vanished or was corrupted while a background run executed, the wake would silently recompose from the exec fallback. Restore the pin from the same restriction walk (it also stops at a plain manual row that carries only the pin), keeping vanished-agent failures loud. --- src/node/services/taskService.test.ts | 2 ++ src/node/services/taskService.ts | 43 +++++++++++++++++++-------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 930dc2d4918..8df60ea9b94 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6470,6 +6470,7 @@ describe("TaskService", () => { timestamp: 1_000, toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, + retrySendOptions: { model: "openai:gpt-4o", agentId: "exec", strictAgentResolution: true }, }) ); await historyService.appendToHistory( @@ -6493,6 +6494,7 @@ describe("TaskService", () => { expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ agentId: "plan", + strictAgentResolution: true, toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index b283bb60934..ea4245f1444 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7915,19 +7915,25 @@ export class TaskService { } /** - * Caller send restrictions (tool policy, workspace-agent disable flag) to restore on a - * terminal-attention wake. The newest manual user row carries the conversation's persisted - * restrictions; synthetic rows without any (earlier wakes, heartbeat scaffolding) do not - * define them and are skipped. The walk is unbounded: a long assistant/synthetic tail after - * the launch turn must not push the defining row out of sight and silently lift the - * restrictions. Throws when history is unreadable so the caller can fail closed instead of - * waking with unrestricted tools. + * Caller send restrictions (tool policy, workspace-agent disable flag, strict-agent pin) to + * restore on a terminal-attention wake. The newest manual user row carries the + * conversation's persisted restrictions; synthetic rows without any (earlier wakes, + * heartbeat scaffolding) do not define them and are skipped. The walk is unbounded: a long + * assistant/synthetic tail after the launch turn must not push the defining row out of + * sight and silently lift the restrictions. Throws when history is unreadable so the caller + * can fail closed instead of waking with unrestricted tools. */ - private async resolveTerminalWakeCallerSendRestrictions( - ownerWorkspaceId: string - ): Promise<{ toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean }> { + private async resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId: string): Promise<{ + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: boolean; + }> { const state: { - found: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean } | null; + found: { + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: boolean; + } | null; } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( ownerWorkspaceId, @@ -7938,6 +7944,11 @@ export class TaskService { continue; } const metadata = message.metadata; + // The strict-agent pin lives in the row's retry snapshot: an explicit agent override + // must stay loud on the wake too, or a vanished/corrupted definition would silently + // recompose the send from the exec fallback (same rule as startup retry and + // compaction follow-ups). + const strictAgentResolution = metadata?.retrySendOptions?.strictAgentResolution === true; if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep // inside send resolution and leave the wake permanently blocked on the same corrupt @@ -7956,11 +7967,12 @@ export class TaskService { ...(typeof metadata.disableWorkspaceAgents === "boolean" ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } : {}), + ...(strictAgentResolution ? { strictAgentResolution: true } : {}), }; return false; } if (metadata?.synthetic !== true) { - state.found = {}; + state.found = strictAgentResolution ? { strictAgentResolution: true } : {}; return false; } } @@ -8422,7 +8434,11 @@ export class TaskService { // synthetic send starts a fresh turn, and omitting the policy would let a workflow wake // regain tools the caller disabled (with attacker-influenced workflow output choosing the // timing). The agent-level policy recomposes from agentId at send resolution. - let wakeRestrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean }; + let wakeRestrictions: { + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: boolean; + }; try { wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); } catch (error: unknown) { @@ -8442,6 +8458,7 @@ export class TaskService { reasoningMode: resumeOptions.reasoningMode, ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), + ...(wakeRestrictions.strictAgentResolution === true ? { strictAgentResolution: true } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { From 36bb23ca60f4039072b3ebf976047c1b40a502da Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 07:59:20 +0000 Subject: [PATCH 26/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20forward=20the=20obj?= =?UTF-8?q?ect-form=20strict-agent=20pin=20on=20terminal=20wakes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-20 restore accepted only literal true, dropping the object form ({expectedScope, expectedSource, expectedChain}) that explicit workspace-agent overrides persist; a same-ID replacement definition could then satisfy resolution silently. Validate the persisted value against the SendMessageOptions union schema and forward it verbatim, as the field's design note requires and startup retry already does. --- src/node/services/taskService.test.ts | 8 +++++-- src/node/services/taskService.ts | 34 ++++++++++++++++++++------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 8df60ea9b94..7ecf8798210 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6470,7 +6470,11 @@ describe("TaskService", () => { timestamp: 1_000, toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, - retrySendOptions: { model: "openai:gpt-4o", agentId: "exec", strictAgentResolution: true }, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "exec", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + }, }) ); await historyService.appendToHistory( @@ -6494,7 +6498,7 @@ describe("TaskService", () => { expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ agentId: "plan", - strictAgentResolution: true, + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index ea4245f1444..c98b7fdd652 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -118,7 +118,7 @@ import { import type { ProjectRef, WorkspaceMetadata } from "@/common/types/workspace"; import { getRuntimeType } from "@/node/runtime/initHook"; import { AgentIdSchema } from "@/common/orpc/schemas"; -import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; +import { SendMessageOptionsSchema, ToolPolicySchema } from "@/common/orpc/schemas/stream"; import type { AgentDefinitionScope } from "@/common/types/agentDefinition"; import { normalizeAgentId, @@ -7926,13 +7926,13 @@ export class TaskService { private async resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId: string): Promise<{ toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean; - strictAgentResolution?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; }> { const state: { found: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean; - strictAgentResolution?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; } | null; } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( @@ -7947,8 +7947,22 @@ export class TaskService { // The strict-agent pin lives in the row's retry snapshot: an explicit agent override // must stay loud on the wake too, or a vanished/corrupted definition would silently // recompose the send from the exec fallback (same rule as startup retry and - // compaction follow-ups). - const strictAgentResolution = metadata?.retrySendOptions?.strictAgentResolution === true; + // compaction follow-ups). Forwarded verbatim per the field's design note: the object + // form pins the validated definition's scope/source/chain provenance, not just + // loudness. Schema-validated like toolPolicy below, since it crosses the same + // persisted-row boundary; invalid shapes are dropped. + const rawStrictPin = metadata?.retrySendOptions?.strictAgentResolution; + const parsedStrictPin = + rawStrictPin != null && rawStrictPin !== false + ? SendMessageOptionsSchema.shape.strictAgentResolution.safeParse(rawStrictPin) + : null; + if (parsedStrictPin != null && !parsedStrictPin.success) { + log.warn("Ignoring malformed persisted strictAgentResolution on terminal wake", { + ownerWorkspaceId, + messageId: message.id, + }); + } + const strictAgentResolution = parsedStrictPin?.success ? parsedStrictPin.data : undefined; if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep // inside send resolution and leave the wake permanently blocked on the same corrupt @@ -7967,12 +7981,12 @@ export class TaskService { ...(typeof metadata.disableWorkspaceAgents === "boolean" ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } : {}), - ...(strictAgentResolution ? { strictAgentResolution: true } : {}), + ...(strictAgentResolution != null ? { strictAgentResolution } : {}), }; return false; } if (metadata?.synthetic !== true) { - state.found = strictAgentResolution ? { strictAgentResolution: true } : {}; + state.found = strictAgentResolution != null ? { strictAgentResolution } : {}; return false; } } @@ -8437,7 +8451,7 @@ export class TaskService { let wakeRestrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean; - strictAgentResolution?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; }; try { wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); @@ -8458,7 +8472,9 @@ export class TaskService { reasoningMode: resumeOptions.reasoningMode, ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), - ...(wakeRestrictions.strictAgentResolution === true ? { strictAgentResolution: true } : {}), + ...(wakeRestrictions.strictAgentResolution != null + ? { strictAgentResolution: wakeRestrictions.strictAgentResolution } + : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { From cf0323c577cf79a516fba86eb451ae5c300d195d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 08:47:54 +0000 Subject: [PATCH 27/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20stop=20compaction?= =?UTF-8?q?=20recovery=20from=20clobbering=20preserved=20follow-up=20field?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pickPreservedSendOptions emitted every preserved field as an explicit key, so compaction retry (which spreads the pick over the persisted follow-up with storage-derived options that never carry a caller toolPolicy) overwrote the original restriction with undefined and the recovered follow-up resumed with unrestricted caller tools. Omit unset fields from the pick instead; this also stops the same latent clobbering of disableWorkspaceAgents and the other preserved fields on retry. --- src/browser/utils/chatCommands.test.ts | 26 +++++++++++++++++++ src/common/types/message.ts | 35 ++++++++++++++++++-------- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/browser/utils/chatCommands.test.ts b/src/browser/utils/chatCommands.test.ts index ed157332b1f..fcd4f8a3034 100644 --- a/src/browser/utils/chatCommands.test.ts +++ b/src/browser/utils/chatCommands.test.ts @@ -1582,6 +1582,32 @@ describe("prepareCompactionMessage", () => { expect(metadata.parsed.followUpContent?.agentId).toBe("exec"); }); + test("compaction recovery keeps the persisted follow-up's restrictions when retry options lack them", () => { + // Retrying a failed compaction passes the already-persisted follow-up together with + // storage-derived send options, which never carry a caller toolPolicy. The preserved + // restrictions must survive that recomposition or the recovered follow-up resumes with + // unrestricted caller tools. + const recoveredFollowUp = { + text: "Keep building", + model: "openai:gpt-4o", + agentId: "code", + toolPolicy: [{ regex_match: "^bash$", action: "disable" as const }], + disableWorkspaceAgents: true, + }; + const { metadata } = prepareCompactionMessage({ + workspaceId: "ws-1", + followUpContent: recoveredFollowUp, + sendMessageOptions: { model: "anthropic:claude-sonnet-4-6", agentId: "exec" }, + }); + + expectCompactionMetadata(metadata); + expect(metadata.parsed.followUpContent?.toolPolicy).toEqual(recoveredFollowUp.toolPolicy); + expect(metadata.parsed.followUpContent?.disableWorkspaceAgents).toBe(true); + // Existing model/agentId still win over the retry-time options. + expect(metadata.parsed.followUpContent?.model).toBe("openai:gpt-4o"); + expect(metadata.parsed.followUpContent?.agentId).toBe("code"); + }); + test("does not create followUpContent when no text or images provided", () => { const sendMessageOptions = createBaseOptions(); const { metadata } = prepareCompactionMessage({ diff --git a/src/common/types/message.ts b/src/common/types/message.ts index db02c4d6f64..2fa770234c9 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -74,24 +74,39 @@ type PreservedSendOptions = Pick< * Use this helper to avoid duplicating the field list when building CompactionFollowUpRequest. */ export function pickPreservedSendOptions(options: SendMessageOptions): PreservedSendOptions { + // Unset fields are OMITTED, not emitted as explicit undefined: compaction recovery spreads + // this pick over an already-persisted follow-up, and an undefined key would clobber the + // original preserved value (e.g. a restricted turn's toolPolicy) instead of leaving it. return { - thinkingLevel: options.thinkingLevel, - reasoningMode: options.reasoningMode, - additionalSystemInstructions: options.additionalSystemInstructions, - providerOptions: options.providerOptions, + ...(options.thinkingLevel !== undefined ? { thinkingLevel: options.thinkingLevel } : {}), + ...(options.reasoningMode !== undefined ? { reasoningMode: options.reasoningMode } : {}), + ...(options.additionalSystemInstructions !== undefined + ? { additionalSystemInstructions: options.additionalSystemInstructions } + : {}), + ...(options.providerOptions !== undefined ? { providerOptions: options.providerOptions } : {}), // Downgrade-compat (see withLegacyPtcExclusiveMirror): preserved options // can persist across restarts and build versions. - experiments: withLegacyPtcExclusiveMirror(options.experiments), - disableWorkspaceAgents: options.disableWorkspaceAgents, + ...(options.experiments !== undefined + ? { experiments: withLegacyPtcExclusiveMirror(options.experiments) } + : {}), + ...(options.disableWorkspaceAgents !== undefined + ? { disableWorkspaceAgents: options.disableWorkspaceAgents } + : {}), // Security: a restricted turn (including a terminal-wake send restoring the caller's // policy) that triggers on-send compaction must not redispatch its follow-up allow-all. - toolPolicy: options.toolPolicy, + ...(options.toolPolicy !== undefined ? { toolPolicy: options.toolPolicy } : {}), // Delegated turns with explicit agent overrides must stay loud across the // compaction replay too — dropping this would let the follow-up silently // fall back to exec if the agent vanished in the meantime. - strictAgentResolution: options.strictAgentResolution, - allowAgentSetGoal: options.allowAgentSetGoal, - skipAiSettingsPersistence: options.skipAiSettingsPersistence, + ...(options.strictAgentResolution !== undefined + ? { strictAgentResolution: options.strictAgentResolution } + : {}), + ...(options.allowAgentSetGoal !== undefined + ? { allowAgentSetGoal: options.allowAgentSetGoal } + : {}), + ...(options.skipAiSettingsPersistence !== undefined + ? { skipAiSettingsPersistence: options.skipAiSettingsPersistence } + : {}), }; } From 9f57ada8516c938a2752f423a177ff7c3dbd462b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:07:17 +0000 Subject: [PATCH 28/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20bind=20workflow=20t?= =?UTF-8?q?erminal=20wakes=20to=20the=20initiating=20agent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wake's agent identity came from the newest agent-bearing assistant row, which a later synthetic heartbeat turn can own without superseding the run, pairing a different agent's tool surface with the launch turn's caller policy. Persist the launching agent with the run reference and prefer it at drain time; legacy references keep the history-walk fallback. --- src/common/utils/tools/tools.ts | 2 + .../agentWorkflowRunReferences.test.ts | 35 +++++++++ .../services/agentWorkflowRunReferences.ts | 14 ++++ src/node/services/aiService.ts | 1 + src/node/services/taskService.test.ts | 73 +++++++++++++++++++ src/node/services/taskService.ts | 38 +++++++++- src/node/services/tools/toolUtils.ts | 1 + src/node/services/tools/workflow_run.test.ts | 3 + 8 files changed, 165 insertions(+), 2 deletions(-) diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index fc286a097c6..5db7e05ba30 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -184,6 +184,8 @@ export interface ToolConfiguration { workspaceSessionDir?: string; /** Workspace ID for tracking background processes and plan storage */ workspaceId?: string; + /** Resolved agent identity of the turn executing the tools (workflow wake provenance). */ + agentId?: string; /** Pre-resolved mux-managed resource scope (global ~/.xum vs project root). */ xumScope?: XumToolScope; /** Memory service for the memory tool (present only when the memory experiment is enabled). */ diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 69171c081cf..955c3c71b7c 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -58,6 +58,41 @@ describe("agent workflow run references", () => { } }); + test("roundtrips the initiating agent and drops invalid persisted shapes", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_agent", + createdAtMs: 1_000, + agentId: "plan", + }); + let references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toContainEqual({ + runId: "wfr_agent", + createdAtMs: 1_000, + agentId: "plan", + }); + + // Identity is advisory: a malformed persisted agentId drops the field, not the entry, + // so the run keeps its wake and identity falls back to the history walk. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_agent_number", createdAtMs: 1_000, agentId: 7 }, + { runId: "wfr_agent_empty", createdAtMs: 1_000, agentId: "" }, + ], + }) + ); + references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toContainEqual({ runId: "wfr_agent_number", createdAtMs: 1_000 }); + expect(references).toContainEqual({ runId: "wfr_agent_empty", createdAtMs: 1_000 }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("drops persisted future-dated references and repairs them on the next record", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 43a9d23a778..f31ca53d72d 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -17,6 +17,13 @@ export interface AgentWorkflowRunReference { * Absent on legacy entries, which fail safe to not-current. */ afterBoundaryMessageId?: string | null; + /** + * Agent identity of the turn that launched/resumed the run. The terminal wake binds to this + * instead of the newest agent-bearing assistant row, which a later synthetic turn (e.g. a + * heartbeat) can own without superseding the run. Advisory: absent on legacy entries, which + * fall back to the history walk. + */ + agentId?: string; } const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; @@ -81,6 +88,10 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { ? boundaryRaw : null : undefined; + // Identity is advisory (the wake falls back to the history walk), so an invalid shape + // drops only the field, not the entry. + const agentId = + typeof record.agentId === "string" && record.agentId.length > 0 ? record.agentId : undefined; // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run // superseded. The chosen record is kept wholesale, including its boundary snapshot. @@ -90,6 +101,7 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { runId: record.runId, createdAtMs: record.createdAtMs, ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + ...(agentId !== undefined ? { agentId } : {}), }); } } @@ -139,6 +151,7 @@ export async function recordAgentWorkflowRunReference(input: { runId: string; createdAtMs?: number; afterBoundaryMessageId?: string | null; + agentId?: string; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -165,6 +178,7 @@ export async function recordAgentWorkflowRunReference(input: { ...(input.afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId: input.afterBoundaryMessageId } : {}), + ...(input.agentId != null && input.agentId.length > 0 ? { agentId: input.agentId } : {}), }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index d0d192fd51f..f43478fb251 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -2646,6 +2646,7 @@ export class AIService extends EventEmitter { planFilePath, ancestorPlanFilePaths, workspaceId, + agentId: effectiveAgentId, xumScope, timelineService: timelineExperimentEnabled ? this.timelineService : undefined, workspaceHeartbeatService: this.workspaceHeartbeatService, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 7ecf8798210..3586153cd8b 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6504,6 +6504,79 @@ describe("TaskService", () => { }); }); + test("workflow wakes bind to the initiating agent, not a later synthetic turn's agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_initiating_agent"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("launch-turn", "assistant", "starting", { + timestamp: 1_001, + agentId: "plan", + }) + ); + // A heartbeat is synthetic, not a manual supersession boundary: the run stays current, but + // its agent-bearing assistant row is now the newest one in history. The wake must use the + // launch turn's agent from the sidecar, not the heartbeat's. + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat", "user", "heartbeat", { timestamp: 1_002, synthetic: true }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat-turn", "assistant", "idle check", { + timestamp: 1_003, + agentId: "exec", + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "plan", + }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index c98b7fdd652..44476e15edb 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8021,7 +8021,13 @@ export class TaskService { ownerWorkspaceId: string, runId: string ): Promise< - { outcome: "deliver"; prompt: string } | { outcome: "superseded" } | { outcome: "defer" } + | { + outcome: "deliver"; + prompt: string; + initiatingAgent?: { agentId: string; createdAtMs: number }; + } + | { outcome: "superseded" } + | { outcome: "defer" } > { assert(ownerWorkspaceId.length > 0, "buildWorkflowTerminalPrompt requires ownerWorkspaceId"); assert(runId.length > 0, "buildWorkflowTerminalPrompt requires runId"); @@ -8058,9 +8064,27 @@ export class TaskService { if (currentness === "not_current") { return { outcome: "superseded" }; } + // Bind the wake to the agent recorded at launch: the newest agent-bearing assistant row + // can belong to an unrelated later synthetic turn (a heartbeat is not a supersession + // boundary), which would pair a different agent's tool surface with the launch turn's + // caller policy. Advisory: legacy references fall back to the history walk. + let initiatingAgent: { agentId: string; createdAtMs: number } | undefined; + try { + const references = await readAgentWorkflowRunReferences( + this.config.getSessionDir(ownerWorkspaceId) + ); + const reference = references.find((candidate) => candidate.runId === run.id); + if (reference?.agentId != null) { + initiatingAgent = { agentId: reference.agentId, createdAtMs: reference.createdAtMs }; + } + } catch { + // Identity is advisory; an unreadable sidecar already deferred delivery above whenever + // currentness itself depended on it. + } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; return { outcome: "deliver", + ...(initiatingAgent != null ? { initiatingAgent } : {}), prompt: buildWorkflowResultContextMessage({ rawCommand: `workflow_run ${scriptPath}`, name: scriptPath, @@ -8375,6 +8399,7 @@ export class TaskService { (notification) => notification.sourceKind === "workflow_run" ); const deliverableWorkflowNotificationIds = new Set(); + let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; const promptSections: string[] = []; if (publicAwaitIds.length > 0) { @@ -8406,6 +8431,14 @@ export class TaskService { continue; } deliverableWorkflowNotificationIds.add(notification.id); + // Newest launch wins when several current runs coalesce into one wake. + if ( + workflowPrompt.initiatingAgent != null && + (workflowInitiatingAgent == null || + workflowPrompt.initiatingAgent.createdAtMs > workflowInitiatingAgent.createdAtMs) + ) { + workflowInitiatingAgent = workflowPrompt.initiatingAgent; + } promptSections.push(workflowPrompt.prompt); } @@ -8438,7 +8471,8 @@ export class TaskService { const resumeOptions = await this.resolveParentAutoResumeOptions( ownerWorkspaceId, entry, - defaultModel + defaultModel, + workflowInitiatingAgent != null ? { agentId: workflowInitiatingAgent.agentId } : undefined ); const workspaceTurnMuxMetadata = await this.getActiveWorkspaceTurnMuxMetadataForWorkspace(ownerWorkspaceId); diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index a56f4574762..24b20398666 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -126,6 +126,7 @@ export async function recordBackgroundWorkflowRunReference( runId, createdAtMs, ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + ...(config.agentId != null && config.agentId.length > 0 ? { agentId: config.agentId } : {}), }); } catch (error: unknown) { log.warn("Failed to record agent workflow run reference", { diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index d49544850c6..143a1bcddb6 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -672,6 +672,7 @@ describe("workflow_run tool", () => { const tool = createWorkflowRunTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, + agentId: "plan", taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, workflowService: { startWorkflow, @@ -693,6 +694,8 @@ describe("workflow_run tool", () => { expect(references[0]).toMatchObject({ runId: "wfr_background", afterBoundaryMessageId: "boundary-row-1", + // The wake binds to the launching agent, so the reference must carry its identity. + agentId: "plan", }); expect(getWorkflowInvocationBoundaryMessageId).toHaveBeenCalledWith( "workspace-1", From 125aae5a56df7aabdf00ed7486b3fbcd6990b4c0 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:29:07 +0000 Subject: [PATCH 29/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20schema-validate=20p?= =?UTF-8?q?ersisted=20initiating=20agent=20IDs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A malformed but non-empty persisted agentId passed the string check, and stream resolution normalizes an unknown requested agent to exec, so a corrupt sidecar entry could silently swap a restricted agent's wake onto exec's tool surface. Parse the field with AgentIdSchema and drop invalid values, keeping the history-walk fallback. --- src/node/services/agentWorkflowRunReferences.test.ts | 4 ++++ src/node/services/agentWorkflowRunReferences.ts | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 955c3c71b7c..03cb142ed41 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -82,12 +82,16 @@ describe("agent workflow run references", () => { references: [ { runId: "wfr_agent_number", createdAtMs: 1_000, agentId: 7 }, { runId: "wfr_agent_empty", createdAtMs: 1_000, agentId: "" }, + // Non-empty but schema-invalid: stream resolution would normalize it to exec, + // silently swapping a restricted agent's wake onto exec's tool surface. + { runId: "wfr_agent_malformed", createdAtMs: 1_000, agentId: "bad id" }, ], }) ); references = await readAgentWorkflowRunReferences(workspaceSessionDir); expect(references).toContainEqual({ runId: "wfr_agent_number", createdAtMs: 1_000 }); expect(references).toContainEqual({ runId: "wfr_agent_empty", createdAtMs: 1_000 }); + expect(references).toContainEqual({ runId: "wfr_agent_malformed", createdAtMs: 1_000 }); } finally { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index f31ca53d72d..f6b52a98ee6 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -3,6 +3,7 @@ import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; +import { AgentIdSchema } from "@/common/schemas/ids"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -89,9 +90,11 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { : null : undefined; // Identity is advisory (the wake falls back to the history walk), so an invalid shape - // drops only the field, not the entry. - const agentId = - typeof record.agentId === "string" && record.agentId.length > 0 ? record.agentId : undefined; + // drops only the field, not the entry. Schema-validate rather than accepting any string: + // stream resolution normalizes an unknown requested agent to exec, so a corrupt persisted + // ID would silently swap a restricted agent's wake onto exec's tool surface. + const agentIdParse = AgentIdSchema.safeParse(record.agentId); + const agentId = agentIdParse.success ? agentIdParse.data : undefined; // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run // superseded. The chosen record is kept wholesale, including its boundary snapshot. From e1b19548088169c6c682c3bed1f0d08259bef974 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:36:50 +0000 Subject: [PATCH 30/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20split=20coalesced?= =?UTF-8?q?=20workflow=20wakes=20by=20initiating=20agent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole coalesced prompt is handled under the single agentId passed to sendMessage, so batching current runs from different initiating agents would hand a restricted agent's attacker-influenced output to another agent's tool grants. Deliver one initiating-agent group per drain (newest launch first) and keep the other groups pending on the re-armed retry drain. --- src/node/services/taskService.test.ts | 93 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 46 ++++++++++--- 2 files changed, 131 insertions(+), 8 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 3586153cd8b..3357c018c1f 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6577,6 +6577,99 @@ describe("TaskService", () => { }); }); + test("coalesced workflow wakes split by initiating agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_split_plan"); + await createRun("wfr_split_exec"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run both audits", { timestamp: 1_000 }) + ); + // Two current runs from different initiating agents: one coalesced wake would hand the + // older run's (attacker-influenced) output to the newer agent's tool grants. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_split_exec", + createdAtMs: 1_000, + agentId: "exec", + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_split_plan", + createdAtMs: 2_000, + agentId: "plan", + }); + + // Seed the store directly so ONE drain observes both pending notifications; per-enqueue + // drains would deliver them separately without exercising the coalescing path. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_split_plan", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_split_exec", + }); + await drain(parentId); + + // The newest launch's group delivers first, alone, under its own agent. + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wfr_split_plan"); + expect(firstPrompt).not.toContain("wfr_split_exec"); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + + // The deferred group delivers on a later drain under its own agent. + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain("wfr_split_exec"); + expect(secondPrompt).not.toContain("wfr_split_plan"); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 44476e15edb..19378948242 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8399,7 +8399,11 @@ export class TaskService { (notification) => notification.sourceKind === "workflow_run" ); const deliverableWorkflowNotificationIds = new Set(); - let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; + const deliverableWorkflowPrompts: Array<{ + notificationId: string; + prompt: string; + initiatingAgent?: { agentId: string; createdAtMs: number }; + }> = []; const promptSections: string[] = []; if (publicAwaitIds.length > 0) { @@ -8430,16 +8434,42 @@ export class TaskService { await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); continue; } - deliverableWorkflowNotificationIds.add(notification.id); - // Newest launch wins when several current runs coalesce into one wake. + deliverableWorkflowPrompts.push({ + notificationId: notification.id, + prompt: workflowPrompt.prompt, + ...(workflowPrompt.initiatingAgent != null + ? { initiatingAgent: workflowPrompt.initiatingAgent } + : {}), + }); + } + // Deliver one initiating-agent group per drain: the whole coalesced prompt is handled + // under the single agentId passed to sendMessage, so batching runs from different agents + // would hand a restricted agent's (attacker-influenced) output to another agent's tool + // grants. The newest launch's group goes first; runs bound to other agents, and the + // history-walk fallback group, stay pending and deliver on the re-armed retry drain. + let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; + for (const candidate of deliverableWorkflowPrompts) { + const agent = candidate.initiatingAgent; if ( - workflowPrompt.initiatingAgent != null && - (workflowInitiatingAgent == null || - workflowPrompt.initiatingAgent.createdAtMs > workflowInitiatingAgent.createdAtMs) + agent != null && + (workflowInitiatingAgent == null || agent.createdAtMs > workflowInitiatingAgent.createdAtMs) ) { - workflowInitiatingAgent = workflowPrompt.initiatingAgent; + workflowInitiatingAgent = agent; } - promptSections.push(workflowPrompt.prompt); + } + const selectedAgentId = workflowInitiatingAgent?.agentId; + const selectedWorkflowPrompts = + selectedAgentId == null + ? deliverableWorkflowPrompts + : deliverableWorkflowPrompts.filter( + (candidate) => candidate.initiatingAgent?.agentId === selectedAgentId + ); + if (selectedWorkflowPrompts.length < deliverableWorkflowPrompts.length) { + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); + } + for (const candidate of selectedWorkflowPrompts) { + deliverableWorkflowNotificationIds.add(candidate.notificationId); + promptSections.push(candidate.prompt); } // Sub-agent reports and failures are already durable user-context messages. Resume from history From cb313080a0a34041c5571e1674f4923a85100582 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:56:37 +0000 Subject: [PATCH 31/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20isolate=20wake=20id?= =?UTF-8?q?entity=20groups=20and=20honor=20synthetic=20launch=20pins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two round-24 findings: (1) mixed drains applied the workflow group's initiating agent to coalesced workspace-turn and sub-agent sections, so those now resume under the conversation's own identity while agent-bound workflow groups defer to their own wake; (2) the restriction walk ignored synthetic launch rows carrying only a strict-agent pin, so the pin and the policy now resolve independently: the newest pin-bearing or manual row defines the pin, the newest policy-bearing or manual row defines the policy. --- src/node/services/taskService.test.ts | 154 ++++++++++++++++++++++++++ src/node/services/taskService.ts | 75 ++++++++----- 2 files changed, 199 insertions(+), 30 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 3357c018c1f..b957f5f06c4 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6670,6 +6670,160 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("mixed drains keep workspace-turn attention off the workflow's agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_mixed"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("agent-turn", "assistant", "on it", { timestamp: 1_001, agentId: "plan" }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "exec", + }); + + // A workspace-turn result resumes under the conversation's own identity; sharing its wake + // with an agent-bound workflow group would process it under the workflow's agent instead. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_mixed_handle", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + }); + await drain(parentId); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wst_mixed_handle"); + expect(firstPrompt).not.toContain(runId); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain(runId); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("wake keeps a synthetic launch row's strict pin without lifting the manual policy", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runId = "wfr_synthetic_pin"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + }) + ); + // The kernel workflow launched from a pinned synthetic turn (preserved heartbeat or + // compaction follow-up): its pin must ride the wake without lifting the manual policy. + await historyService.appendToHistory( + parentId, + createMuxMessage("synthetic-launch", "user", "heartbeat", { + timestamp: 1_100, + synthetic: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }, + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "plan", + }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + toolPolicy: restrictedPolicy, + strictAgentResolution: { expectedScope: "built-in" }, + }); + }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 19378948242..5a045aefe7f 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7928,13 +7928,14 @@ export class TaskService { disableWorkspaceAgents?: boolean; strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; }> { + // The pin and the policy resolve independently: a synthetic launch row (preserved + // heartbeat, compaction follow-up) can carry only a strict pin, and the wake bound to that + // turn's agent must keep the pin loud without lifting an older manual row's policy. + // Manual rows still define both wholesale (absence means lifted). const state: { - found: { - toolPolicy?: ToolPolicy; - disableWorkspaceAgents?: boolean; - strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; - } | null; - } = { found: null }; + pin: { strictAgentResolution?: SendMessageOptions["strictAgentResolution"] } | null; + restrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean } | null; + } = { pin: null, restrictions: null }; const historyResult = await this.historyService.iterateFullHistory( ownerWorkspaceId, "backward", @@ -7963,30 +7964,38 @@ export class TaskService { }); } const strictAgentResolution = parsedStrictPin?.success ? parsedStrictPin.data : undefined; - if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { + if ( + state.pin == null && + (strictAgentResolution != null || metadata?.synthetic !== true) + ) { + state.pin = strictAgentResolution != null ? { strictAgentResolution } : {}; + } + if ( + state.restrictions == null && + (metadata?.toolPolicy != null || + metadata?.disableWorkspaceAgents != null || + metadata?.synthetic !== true) + ) { // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep // inside send resolution and leave the wake permanently blocked on the same corrupt // row. Sanitize instead of trusting the JSON shape; an unparseable policy restores // nothing while a valid disable flag still applies (self-healing doctrine). const parsedPolicy = - metadata.toolPolicy != null ? ToolPolicySchema.safeParse(metadata.toolPolicy) : null; + metadata?.toolPolicy != null ? ToolPolicySchema.safeParse(metadata.toolPolicy) : null; if (parsedPolicy != null && !parsedPolicy.success) { log.warn("Ignoring malformed persisted toolPolicy on terminal wake", { ownerWorkspaceId, messageId: message.id, }); } - state.found = { + state.restrictions = { ...(parsedPolicy?.success ? { toolPolicy: parsedPolicy.data } : {}), - ...(typeof metadata.disableWorkspaceAgents === "boolean" + ...(typeof metadata?.disableWorkspaceAgents === "boolean" ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } : {}), - ...(strictAgentResolution != null ? { strictAgentResolution } : {}), }; - return false; } - if (metadata?.synthetic !== true) { - state.found = strictAgentResolution != null ? { strictAgentResolution } : {}; + if (state.pin != null && state.restrictions != null) { return false; } } @@ -7996,7 +8005,7 @@ export class TaskService { if (!historyResult.success) { throw new Error(`history unavailable: ${historyResult.error}`); } - return state.found ?? {}; + return { ...(state.restrictions ?? {}), ...(state.pin ?? {}) }; } private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { @@ -8445,25 +8454,31 @@ export class TaskService { // Deliver one initiating-agent group per drain: the whole coalesced prompt is handled // under the single agentId passed to sendMessage, so batching runs from different agents // would hand a restricted agent's (attacker-influenced) output to another agent's tool - // grants. The newest launch's group goes first; runs bound to other agents, and the - // history-walk fallback group, stay pending and deliver on the re-armed retry drain. + // grants. The same applies to mixed batches: workspace-turn and sub-agent attention + // resumes under the conversation's own (history-walk) identity, so agent-bound workflow + // groups never share their send. Deferred groups stay pending and deliver on the re-armed + // retry drain; among agent-bound groups the newest launch goes first. + const hasNonWorkflowDeliverables = + deliverableAgentNotificationIds.size > 0 || deliverableWorkspaceTurnNotificationIds.size > 0; let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; - for (const candidate of deliverableWorkflowPrompts) { - const agent = candidate.initiatingAgent; - if ( - agent != null && - (workflowInitiatingAgent == null || agent.createdAtMs > workflowInitiatingAgent.createdAtMs) - ) { - workflowInitiatingAgent = agent; + if (!hasNonWorkflowDeliverables) { + for (const candidate of deliverableWorkflowPrompts) { + const agent = candidate.initiatingAgent; + if ( + agent != null && + (workflowInitiatingAgent == null || + agent.createdAtMs > workflowInitiatingAgent.createdAtMs) + ) { + workflowInitiatingAgent = agent; + } } } const selectedAgentId = workflowInitiatingAgent?.agentId; - const selectedWorkflowPrompts = - selectedAgentId == null - ? deliverableWorkflowPrompts - : deliverableWorkflowPrompts.filter( - (candidate) => candidate.initiatingAgent?.agentId === selectedAgentId - ); + const selectedWorkflowPrompts = deliverableWorkflowPrompts.filter((candidate) => + hasNonWorkflowDeliverables + ? candidate.initiatingAgent == null + : selectedAgentId == null || candidate.initiatingAgent?.agentId === selectedAgentId + ); if (selectedWorkflowPrompts.length < deliverableWorkflowPrompts.length) { this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); } From 8e1bcd65500cc4bfed27363a05048fad351eb64c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:04:48 +0000 Subject: [PATCH 32/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20harden=20wake=20pro?= =?UTF-8?q?venance=20writes,=20reads,=20and=20pin=20pairing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three round-25 findings: (1) an explicit background launch now fails loudly (run stays pending and resumable) when the pre-launch sidecar write fails, instead of starting a runner whose terminal wake would be permanently superseded; (2) transient run-store read failures (non-ENOENT fs errors) defer the terminal wake for retry instead of tombstoning it, while missing or unparseable runs stay superseded; (3) the launch turn's strict-agent pin is persisted with the run reference (null for verified-unpinned) and the wake re-pins the selected group's own provenance, because the newest pin-bearing history row can belong to a different group's wake and a mismatched pin would reject the wake on every retry. --- src/common/utils/tools/tools.ts | 4 +- .../agentWorkflowRunReferences.test.ts | 27 ++++ .../services/agentWorkflowRunReferences.ts | 44 +++++- src/node/services/aiService.ts | 1 + src/node/services/taskService.test.ts | 138 ++++++++++++++++++ src/node/services/taskService.ts | 56 +++++-- src/node/services/tools/toolUtils.ts | 33 ++++- src/node/services/tools/workflow_run.test.ts | 40 ++++- src/node/services/tools/workflow_run.ts | 4 +- 9 files changed, 329 insertions(+), 18 deletions(-) diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index 5db7e05ba30..a6b69a4d248 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -2,7 +2,7 @@ import { xai } from "@ai-sdk/xai"; import { type LanguageModel, type Tool } from "ai"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; -import type { ProvidersConfigMap } from "@/common/orpc/types"; +import type { ProvidersConfigMap, SendMessageOptions } from "@/common/orpc/types"; import { isGrokFrontierModel } from "@/common/types/thinking"; import type { BackgroundWorkAttentionPolicy } from "@/common/types/backgroundWorkAttention"; import { cloneToolPreservingDescriptors } from "@/common/utils/tools/cloneToolPreservingDescriptors"; @@ -186,6 +186,8 @@ export interface ToolConfiguration { workspaceId?: string; /** Resolved agent identity of the turn executing the tools (workflow wake provenance). */ agentId?: string; + /** The turn's strict-agent pin, persisted with workflow run provenance so wakes re-pin the launch agent. */ + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; /** Pre-resolved mux-managed resource scope (global ~/.xum vs project root). */ xumScope?: XumToolScope; /** Memory service for the memory tool (present only when the memory experiment is enabled). */ diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 03cb142ed41..1939e4f305a 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -66,12 +66,14 @@ describe("agent workflow run references", () => { runId: "wfr_agent", createdAtMs: 1_000, agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, }); let references = await readAgentWorkflowRunReferences(workspaceSessionDir); expect(references).toContainEqual({ runId: "wfr_agent", createdAtMs: 1_000, agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, }); // Identity is advisory: a malformed persisted agentId drops the field, not the entry, @@ -85,6 +87,20 @@ describe("agent workflow run references", () => { // Non-empty but schema-invalid: stream resolution would normalize it to exec, // silently swapping a restricted agent's wake onto exec's tool surface. { runId: "wfr_agent_malformed", createdAtMs: 1_000, agentId: "bad id" }, + // Invalid pin shapes degrade to the legacy walk fallback (field dropped); a + // persisted false means verified-unpinned (null), like absence at record time. + { + runId: "wfr_pin_invalid", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: 42 }, + }, + { + runId: "wfr_pin_false", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: false, + }, ], }) ); @@ -92,6 +108,17 @@ describe("agent workflow run references", () => { expect(references).toContainEqual({ runId: "wfr_agent_number", createdAtMs: 1_000 }); expect(references).toContainEqual({ runId: "wfr_agent_empty", createdAtMs: 1_000 }); expect(references).toContainEqual({ runId: "wfr_agent_malformed", createdAtMs: 1_000 }); + expect(references).toContainEqual({ + runId: "wfr_pin_invalid", + createdAtMs: 1_000, + agentId: "plan", + }); + expect(references).toContainEqual({ + runId: "wfr_pin_false", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: null, + }); } finally { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index f6b52a98ee6..2f66b634964 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -3,10 +3,18 @@ import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; +import { SendMessageOptionsSchema } from "@/common/orpc/schemas/stream"; +import type { SendMessageOptions } from "@/common/orpc/types"; import { AgentIdSchema } from "@/common/schemas/ids"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; +/** A meaningful strict-agent pin: `false` and absence both mean "not pinned" and persist as null. */ +export type AgentWorkflowRunStrictPin = Exclude< + NonNullable, + false +>; + export interface AgentWorkflowRunReference { runId: string; createdAtMs: number; @@ -25,8 +33,18 @@ export interface AgentWorkflowRunReference { * fall back to the history walk. */ agentId?: string; + /** + * The launch turn's strict-agent pin, paired with agentId: a wake that re-binds this agent + * must re-pin the launch turn's provenance, because the newest pin-bearing history row can + * belong to a different group's wake and a mismatched pin makes resolution reject every + * retry. null records a verified-unpinned launch; absent (legacy or invalid persisted + * shape) falls back to the history-walk pin. + */ + strictAgentResolution?: AgentWorkflowRunStrictPin | null; } +const StrictPinSchema = SendMessageOptionsSchema.shape.strictAgentResolution; + const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; // Backward clock corrections (e.g. an NTP step after booting with a fast clock) can make a @@ -95,6 +113,21 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { // ID would silently swap a restricted agent's wake onto exec's tool surface. const agentIdParse = AgentIdSchema.safeParse(record.agentId); const agentId = agentIdParse.success ? agentIdParse.data : undefined; + // The pin only means anything paired with a surviving identity; false and invalid shapes + // degrade differently (unpinned vs legacy walk fallback), matching the field doc above. + let strictAgentResolution: AgentWorkflowRunStrictPin | null | undefined; + if (agentId !== undefined && "strictAgentResolution" in record) { + const pinRaw = record.strictAgentResolution; + if (pinRaw === null || pinRaw === false) { + strictAgentResolution = null; + } else { + const pinParse = StrictPinSchema.safeParse(pinRaw); + strictAgentResolution = + pinParse.success && pinParse.data != null && pinParse.data !== false + ? pinParse.data + : undefined; + } + } // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run // superseded. The chosen record is kept wholesale, including its boundary snapshot. @@ -105,6 +138,7 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { createdAtMs: record.createdAtMs, ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), ...(agentId !== undefined ? { agentId } : {}), + ...(strictAgentResolution !== undefined ? { strictAgentResolution } : {}), }); } } @@ -155,6 +189,7 @@ export async function recordAgentWorkflowRunReference(input: { createdAtMs?: number; afterBoundaryMessageId?: string | null; agentId?: string; + strictAgentResolution?: AgentWorkflowRunStrictPin | null; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -181,7 +216,14 @@ export async function recordAgentWorkflowRunReference(input: { ...(input.afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId: input.afterBoundaryMessageId } : {}), - ...(input.agentId != null && input.agentId.length > 0 ? { agentId: input.agentId } : {}), + ...(input.agentId != null && input.agentId.length > 0 + ? { + agentId: input.agentId, + ...(input.strictAgentResolution !== undefined + ? { strictAgentResolution: input.strictAgentResolution } + : {}), + } + : {}), }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index f43478fb251..58169e0749b 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -2647,6 +2647,7 @@ export class AIService extends EventEmitter { ancestorPlanFilePaths, workspaceId, agentId: effectiveAgentId, + strictAgentResolution, xumScope, timelineService: timelineExperimentEnabled ? this.timelineService : undefined, workspaceHeartbeatService: this.workspaceHeartbeatService, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index b957f5f06c4..6532f19e836 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6824,6 +6824,144 @@ describe("TaskService", () => { }); }); + test("transient run-store read failures defer the wake instead of tombstoning it", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + const terminalAttentionStore = new TerminalAttentionStore(config); + + // run.json exists but is unreadable (EISDIR): potentially transient, so the wake must + // stay pending for a later drain instead of being durably tombstoned. + const unreadableRunId = "wfr_unreadable"; + await fsPromises.mkdir( + path.join(config.getSessionDir(parentId), "workflows", unreadableRunId, "run.json"), + { recursive: true } + ); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: unreadableRunId, + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + + // A definitively missing run (ENOENT) is still tombstoned, not deferred forever. + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_missing", + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.get(parentId, "workflow_run:wfr_missing")).toMatchObject({ + status: "superseded", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + + test("wake re-pins the selected group's recorded launch pin, not the newest row's", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_pin_unpinned"); + await createRun("wfr_pin_recorded"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audits", { timestamp: 1_000 }) + ); + // The newest pin-bearing row belongs to a DIFFERENT group's wake: pinning its provenance + // onto this group's agentId would make resolution reject the wake on every retry. + await historyService.appendToHistory( + parentId, + createMuxMessage("other-group-wake", "user", "earlier group results", { + timestamp: 1_100, + synthetic: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "plan", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + }, + }) + ); + + // A verified-unpinned launch (null) must suppress the walk pin entirely. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_unpinned", + agentId: "exec", + strictAgentResolution: null, + }); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_pin_unpinned", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const unpinnedOptions = sendMessage.mock.calls[0]?.[2] as { + agentId?: string; + strictAgentResolution?: unknown; + }; + expect(unpinnedOptions.agentId).toBe("exec"); + expect(unpinnedOptions.strictAgentResolution).toBeUndefined(); + + // A recorded launch pin overrides the walk pin exactly. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_recorded", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_pin_recorded", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 5a045aefe7f..a1a486d4547 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -210,6 +210,7 @@ import { type TerminalAttentionOutcome, } from "@/node/services/terminalAttentionStore"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import type { AgentWorkflowRunStrictPin } from "@/node/services/agentWorkflowRunReferences"; import { isWorkflowRunTaskId } from "@/node/services/tools/taskId"; import { normalizeWorkflowAgentReportPayloadForHostSchema } from "@/common/utils/tools/workflowReportPayload"; import { @@ -1051,6 +1052,13 @@ interface ParentAutoResumeHint { agentId?: string; } +/** Launch identity recorded with a workflow run reference; see AgentWorkflowRunReference. */ +interface WorkflowWakeInitiatingAgent { + agentId: string; + createdAtMs: number; + strictAgentResolution?: AgentWorkflowRunStrictPin | null; +} + function isTypedWorkspaceEvent(value: unknown, type: string): boolean { return ( typeof value === "object" && @@ -8030,11 +8038,7 @@ export class TaskService { ownerWorkspaceId: string, runId: string ): Promise< - | { - outcome: "deliver"; - prompt: string; - initiatingAgent?: { agentId: string; createdAtMs: number }; - } + | { outcome: "deliver"; prompt: string; initiatingAgent?: WorkflowWakeInitiatingAgent } | { outcome: "superseded" } | { outcome: "defer" } > { @@ -8047,6 +8051,22 @@ export class TaskService { try { run = await runStore.getRun(runId); } catch (error: unknown) { + // A missing run (ENOENT) or an unparseable record (no fs code; rereading cannot repair + // it) is definitively ineligible. Every other fs failure (EIO, EACCES, EISDIR...) is + // potentially transient, and tombstoning on it would permanently drop the wake over a + // recoverable fault: defer those like indeterminate currentness below. + const code = + error != null && typeof error === "object" && "code" in error + ? (error as { code?: unknown }).code + : undefined; + if (typeof code === "string" && code !== "ENOENT") { + log.warn("Deferring workflow terminal wake-up; run record unreadable", { + ownerWorkspaceId, + runId, + error: getErrorMessage(error), + }); + return { outcome: "defer" }; + } log.warn("Failed to load terminal workflow run for wake-up", { ownerWorkspaceId, runId, @@ -8077,14 +8097,20 @@ export class TaskService { // can belong to an unrelated later synthetic turn (a heartbeat is not a supersession // boundary), which would pair a different agent's tool surface with the launch turn's // caller policy. Advisory: legacy references fall back to the history walk. - let initiatingAgent: { agentId: string; createdAtMs: number } | undefined; + let initiatingAgent: WorkflowWakeInitiatingAgent | undefined; try { const references = await readAgentWorkflowRunReferences( this.config.getSessionDir(ownerWorkspaceId) ); const reference = references.find((candidate) => candidate.runId === run.id); if (reference?.agentId != null) { - initiatingAgent = { agentId: reference.agentId, createdAtMs: reference.createdAtMs }; + initiatingAgent = { + agentId: reference.agentId, + createdAtMs: reference.createdAtMs, + ...(reference.strictAgentResolution !== undefined + ? { strictAgentResolution: reference.strictAgentResolution } + : {}), + }; } } catch { // Identity is advisory; an unreadable sidecar already deferred delivery above whenever @@ -8411,7 +8437,7 @@ export class TaskService { const deliverableWorkflowPrompts: Array<{ notificationId: string; prompt: string; - initiatingAgent?: { agentId: string; createdAtMs: number }; + initiatingAgent?: WorkflowWakeInitiatingAgent; }> = []; const promptSections: string[] = []; @@ -8460,7 +8486,7 @@ export class TaskService { // retry drain; among agent-bound groups the newest launch goes first. const hasNonWorkflowDeliverables = deliverableAgentNotificationIds.size > 0 || deliverableWorkspaceTurnNotificationIds.size > 0; - let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; + let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; if (!hasNonWorkflowDeliverables) { for (const candidate of deliverableWorkflowPrompts) { const agent = candidate.initiatingAgent; @@ -8544,6 +8570,14 @@ export class TaskService { return; } + // Pair the pin with the selected group: the newest pin-bearing history row can belong to + // a different group's wake (each wake persists its own pin), and pinning another agent's + // provenance onto this group's agentId makes resolution reject the wake on every retry. A + // recorded pin (or a verified-unpinned null) overrides the walk; legacy references + // without the field keep the walk pin. + const groupPin = workflowInitiatingAgent?.strictAgentResolution; + const effectiveStrictPin = + groupPin !== undefined ? (groupPin ?? undefined) : wakeRestrictions.strictAgentResolution; const sendOptions = { model: resumeOptions.model, agentId: resumeOptions.agentId, @@ -8551,9 +8585,7 @@ export class TaskService { reasoningMode: resumeOptions.reasoningMode, ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), - ...(wakeRestrictions.strictAgentResolution != null - ? { strictAgentResolution: wakeRestrictions.strictAgentResolution } - : {}), + ...(effectiveStrictPin != null ? { strictAgentResolution: effectiveStrictPin } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 24b20398666..79f2c8ca758 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -89,10 +89,26 @@ export async function emitWorkflowRunAttachedEvent(input: { export async function recordBackgroundWorkflowRunReference( config: ToolConfiguration, runId: string, - createdAtMs: number + createdAtMs: number, + options?: { + /** + * Pre-launch records must not fail soft: the sidecar is the kernel invocation's only + * durable provenance, and starting the runner without it lets a fast terminal run have + * its wake permanently marked superseded. Throwing before dispatch leaves the run + * pending and resumable (workflow_resume re-records), so the caller surfaces a loud, + * recoverable launch failure instead. Post-dispatch records stay best-effort because the + * run already started and failing the tool would strand it. + */ + propagateWriteFailure?: boolean; + } ): Promise { const workspaceSessionDir = config.workspaceSessionDir; if (workspaceSessionDir == null || workspaceSessionDir.length === 0) { + if (options?.propagateWriteFailure === true) { + throw new Error( + `Cannot record workflow run provenance without a workspace session dir: ${runId}` + ); + } log.warn("Skipping agent workflow run reference without workspace session dir", { runId }); return; } @@ -126,9 +142,22 @@ export async function recordBackgroundWorkflowRunReference( runId, createdAtMs, ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), - ...(config.agentId != null && config.agentId.length > 0 ? { agentId: config.agentId } : {}), + ...(config.agentId != null && config.agentId.length > 0 + ? { + agentId: config.agentId, + // The pin pairs with the identity: null records a verified-unpinned launch so the + // wake never inherits another row's pin (see AgentWorkflowRunReference). + strictAgentResolution: + config.strictAgentResolution != null && config.strictAgentResolution !== false + ? config.strictAgentResolution + : null, + } + : {}), }); } catch (error: unknown) { + if (options?.propagateWriteFailure === true) { + throw error; + } log.warn("Failed to record agent workflow run reference", { runId, error: getErrorMessage(error), diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index 143a1bcddb6..3e394d5c35f 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -673,6 +673,7 @@ describe("workflow_run tool", () => { ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, workflowService: { startWorkflow, @@ -694,8 +695,10 @@ describe("workflow_run tool", () => { expect(references[0]).toMatchObject({ runId: "wfr_background", afterBoundaryMessageId: "boundary-row-1", - // The wake binds to the launching agent, so the reference must carry its identity. + // The wake binds to the launching agent, so the reference must carry its identity + // and the launch turn's provenance pin. agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, }); expect(getWorkflowInvocationBoundaryMessageId).toHaveBeenCalledWith( "workspace-1", @@ -757,6 +760,41 @@ describe("workflow_run tool", () => { expect(references[0] != null && "afterBoundaryMessageId" in references[0]).toBe(false); }); + test("a sidecar write failure aborts the background launch before the runner starts", async () => { + using tempDir = new TestTempDir("test-workflow-run-tool-sidecar-write-error"); + const scriptPath = await writeWorkflowScript(tempDir.path); + // The reference path exists as a directory, so every sidecar write fails. + await fs.mkdir(path.join(tempDir.path, "agent-workflow-runs.json")); + let runnerStarted = false; + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_sidecar_write_error", + status: "pending", + result: null, + run: null, + }); + runnerStarted = true; + return { runId: "wfr_sidecar_write_error", status: "running" as const, result: null }; + }); + const getRun = mock(async () => null); + const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); + const tool = createWorkflowRunTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + agentId: "plan", + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, + workflowService: { startWorkflowInBackground, getRun }, + }); + + // Starting the runner without durable provenance would let a fast terminal run have its + // wake permanently superseded; the launch must fail loudly instead, leaving the run + // pending and resumable (workflow_resume re-records provenance). + await expect( + tool.execute!({ script_path: scriptPath, run_in_background: true }, mockToolCallOptions) + ).rejects.toThrow(/created durable run wfr_sidecar_write_error/); + expect(runnerStarted).toBe(false); + }); + test("requires the workflow service", async () => { using tempDir = new TestTempDir("test-workflow-run-tool-missing"); const scriptPath = await writeWorkflowScript(tempDir.path); diff --git a/src/node/services/tools/workflow_run.ts b/src/node/services/tools/workflow_run.ts index 075089795cc..300da0f60fc 100644 --- a/src/node/services/tools/workflow_run.ts +++ b/src/node/services/tools/workflow_run.ts @@ -276,7 +276,9 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => // process exit mid-dispatch) can reach terminal state before any post-dispatch // write, and a terminal wake with no sidecar reference is permanently superseded. if (args.run_in_background === true) { - await recordBackgroundWorkflowRunReference(config, event.runId, invocationStartedAtMs); + await recordBackgroundWorkflowRunReference(config, event.runId, invocationStartedAtMs, { + propagateWriteFailure: true, + }); } await emitWorkflowRunAttachedEvent({ config, From d53ac1ad2efe70b6470c2f502371f014477149ee Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:41:01 +0000 Subject: [PATCH 33/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20defer=20boundaryles?= =?UTF-8?q?s=20workflow=20references=20instead=20of=20wall-clock=20orderin?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/agentWorkflowRunReferences.ts | 6 +-- src/node/services/tools/toolUtils.ts | 2 +- src/node/services/workspaceService.test.ts | 27 +++++------ src/node/services/workspaceService.ts | 46 ++++++++----------- 4 files changed, 36 insertions(+), 45 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 2f66b634964..6e2ff131194 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -92,9 +92,9 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { const hasBoundary = "afterBoundaryMessageId" in record; const boundaryRaw = record.afterBoundaryMessageId; // A present-but-invalid snapshot ("" or a non-string) is corruption, not a legacy record: - // migrating it into the wall-clock fallback could let a stale reference outrank a newer - // boundary within the tolerated clock skew. Reject the entry; absence stays reserved for - // records that genuinely predate the field. + // demoting it to a boundaryless entry would misclassify a recorded boundary as unknowable + // provenance (parking its wake as indeterminate). Reject the entry; absence stays reserved + // for records that genuinely predate the field. if ( hasBoundary && boundaryRaw !== null && diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 79f2c8ca758..d842b9ab224 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -119,7 +119,7 @@ export async function recordBackgroundWorkflowRunReference( // failure must not be persisted as a verified-empty boundary (null): record without the // field instead, so the run stays rediscoverable (listAgentReferencedWorkflowRunIds) and a // later workflow_resume re-record can repair provenance, while the unverifiable boundary - // fails safe for wake delivery. + // defers wake delivery (indeterminate) instead of guessing from wall-clock order. let afterBoundaryMessageId: string | null | undefined; const taskService = config.taskService; if (config.workspaceId != null && taskService?.getWorkflowInvocationBoundaryMessageId != null) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index daac7889e48..6ee594182cc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6406,7 +6406,7 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("migrates legacy sidecar references through the wall-clock fallback", async () => { + test("defers boundaryless sidecar references instead of trusting wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-legacy"; const runId = "wfr_currentness_legacy"; @@ -6438,30 +6438,31 @@ describe("WorkspaceService workflow invocation events", () => { workspaceId, createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) ); - // Entries written before boundary snapshots existed carry only a timestamp. An in-flight - // run recorded after the newest boundary must keep its wake across the upgrade. + // A reference without a boundary snapshot (pre-upgrade entry or record-time history read + // failure) cannot be ordered against the decision row by identity: the wake defers + // instead of delivering, and the boolean caller stays fail-safe. await recordAgentWorkflowRunReference({ workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, }); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); - // A newer boundary still supersedes a legacy entry. + // A backward clock correction gives the newer superseding turn an OLDER timestamp than + // the reference. Wall-clock ordering would resurrect the superseded reference as current + // and deliver its output under the newer turn's tool policy; it must stay deferred. await historyService.appendToHistory( workspaceId, createMuxMessage("manual-user-2", "user", "never mind, answer something else", { - timestamp: 1_200, + timestamp: 1_100, }) ); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); - - // An undatable boundary cannot be ordered against a legacy timestamp: fail safe. - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user-undated", "user", "another instruction", {}) + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" ); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); workspaceService.disposeSession(workspaceId); } finally { await cleanup(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index bb95cabd55c..43688444a46 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10959,11 +10959,11 @@ export class WorkspaceService extends EventEmitter { } /** - * Three-state currentness: "indeterminate" means history could not be read, so the answer is - * unknown rather than no. Callers that would permanently drop a terminal wake on a negative - * answer (the terminal-attention drain tombstones notifications) must retain and retry on - * "indeterminate" instead; boolean callers treat it as not-current, the pre-existing - * fail-safe for non-destructive decisions. + * Three-state currentness: "indeterminate" means history/provenance could not be read or + * ordered, so the answer is unknown rather than no. Callers that would permanently drop a + * terminal wake on a negative answer (the terminal-attention drain tombstones notifications) + * must retain and retry on "indeterminate" instead; boolean callers treat it as not-current, + * the pre-existing fail-safe for non-destructive decisions. */ async getWorkflowInvocationCurrentness( workspaceId: string, @@ -10990,7 +10990,8 @@ export class WorkspaceService extends EventEmitter { // a legitimate wake nor let a pre-supersession reference outrank a newer boundary. For a // consumed boundary, equality means a background resume/retry was recorded after the prior // result was delivered. References without a boundary snapshot (pre-upgrade entries, - // record-time read failures) take a wall-clock migration fallback below. + // record-time read failures) cannot be ordered against the decision row at all and defer + // as indeterminate below. let references: AgentWorkflowRunReference[]; try { references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); @@ -11021,15 +11022,13 @@ export class WorkspaceService extends EventEmitter { return "not_current"; } if (reference.afterBoundaryMessageId === undefined) { - // Migration fallback: entries written before boundary snapshots existed (or after a - // record-time history read failure) carry only a timestamp, and an in-flight run must not - // lose its wake across the upgrade. Fall back to wall-clock ordering against a datable - // boundary; an undatable boundary fails safe. All new records take the identity path - // above, so clock-correction edge cases are confined to this shrinking population. - if (decision.timestampMs == null) { - return "not_current"; - } - return reference.createdAtMs > decision.timestampMs ? "current" : "not_current"; + // No boundary snapshot (pre-upgrade entry or record-time history read failure): row + // identity cannot be verified, and wall-clock ordering is the exact hole the identity + // path exists to close (a backward clock correction would let a pre-supersession + // reference outrank a newer manual turn and deliver its output under that turn's tool + // policy). Defer like an unreadable history: the wake stays pending, a workflow_resume + // re-record repairs provenance, and an explicit resume/await still consumes the run. + return "indeterminate"; } if (reference.afterBoundaryMessageId === null) { // Verified-empty snapshot: a decision row now exists, so it appeared after the record. @@ -11052,7 +11051,6 @@ export class WorkspaceService extends EventEmitter { status: "found"; outcome: "invocation" | "consumed" | "superseded"; messageId: string; - timestampMs: number | null; } | { status: "none" } | { status: "error" } @@ -11061,7 +11059,6 @@ export class WorkspaceService extends EventEmitter { found: { outcome: "invocation" | "consumed" | "superseded"; messageId: string; - timestampMs: number | null; } | null; } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( @@ -11069,10 +11066,8 @@ export class WorkspaceService extends EventEmitter { "backward", (messages) => { for (const message of messages) { - const timestamp = message.metadata?.timestamp; - const timestampMs = typeof timestamp === "number" ? timestamp : null; if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { - state.found = { outcome: "superseded", messageId: message.id, timestampMs }; + state.found = { outcome: "superseded", messageId: message.id }; return false; } if ( @@ -11081,11 +11076,11 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - state.found = { outcome: "consumed", messageId: message.id, timestampMs }; + state.found = { outcome: "consumed", messageId: message.id }; return false; } if (isWorkflowInvocationMessage(message, runId)) { - state.found = { outcome: "invocation", messageId: message.id, timestampMs }; + state.found = { outcome: "invocation", messageId: message.id }; return false; } } @@ -11101,12 +11096,7 @@ export class WorkspaceService extends EventEmitter { return { status: "error" }; } return state.found != null - ? { - status: "found", - outcome: state.found.outcome, - messageId: state.found.messageId, - timestampMs: state.found.timestampMs, - } + ? { status: "found", outcome: state.found.outcome, messageId: state.found.messageId } : { status: "none" }; } From cd1b200807b857cc4fe7a2c69c40c301bd0ca7a6 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:15:17 +0000 Subject: [PATCH 34/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20split=20wakes=20by?= =?UTF-8?q?=20launch=20pin=20and=20repair=20downgrade-stripped=20provenanc?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/orpc/router.ts | 2 + src/node/services/taskService.test.ts | 95 +++++++++++++++++++ src/node/services/taskService.ts | 32 +++++-- .../workflows/WorkflowService.test.ts | 56 +++++++++++ .../services/workflows/WorkflowService.ts | 21 ++++ src/node/services/workspaceService.test.ts | 90 +++++++++++++++++- src/node/services/workspaceService.ts | 40 ++++++++ 7 files changed, 326 insertions(+), 10 deletions(-) diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index c425ea2be02..327db8336f9 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -584,6 +584,8 @@ export async function resolveWorkflowContext( skillStorageContext, }), onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), + onRunCrashResumed: (event) => + context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId), ...(options.onBackgroundRunTerminal != null ? { onBackgroundRunTerminal: options.onBackgroundRunTerminal } : {}), diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 6532f19e836..0316e4089f1 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6670,6 +6670,101 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("coalesced workflow wakes split by strict pin within one agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_pin_split_pinned"); + await createRun("wfr_pin_split_unpinned"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run both audits", { timestamp: 1_000 }) + ); + // Same agentId, different launch pins (an agent definition replaced between synthetic + // launches): one coalesced wake would process the pinned run's output under the newer + // verified-unpinned launch. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_split_pinned", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_split_unpinned", + createdAtMs: 2_000, + agentId: "plan", + strictAgentResolution: null, + }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_pin_split_pinned", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_pin_split_unpinned", + }); + await drain(parentId); + + // The newest launch delivers first, alone, without the other launch's pin. + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wfr_pin_split_unpinned"); + expect(firstPrompt).not.toContain("wfr_pin_split_pinned"); + const firstOptions = sendMessage.mock.calls[0]?.[2] as Record; + expect(firstOptions.agentId).toBe("plan"); + expect(firstOptions.strictAgentResolution).toBeUndefined(); + + // The pinned launch delivers on the retry drain under its own recorded pin. + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain("wfr_pin_split_pinned"); + expect(secondPrompt).not.toContain("wfr_pin_split_unpinned"); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("mixed drains keep workspace-turn attention off the workflow's agent", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index a1a486d4547..89bafb64a7a 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1059,6 +1059,15 @@ interface WorkflowWakeInitiatingAgent { strictAgentResolution?: AgentWorkflowRunStrictPin | null; } +// Coalescing key for terminal workflow wakes: the pin is part of the launch identity, so an +// agentId alone must not merge a pinned launch with an unpinned (or differently pinned) one. +// undefined (legacy walk fallback), null (verified unpinned), and each concrete pin are +// distinct groups; over-splitting structurally equal pins is safe, merging them is not. +function workflowWakeGroupKey(agent: WorkflowWakeInitiatingAgent): string { + const pin = agent.strictAgentResolution; + return `${agent.agentId}\u0000${pin === undefined ? "walk" : JSON.stringify(pin)}`; +} + function isTypedWorkspaceEvent(value: unknown, type: string): boolean { return ( typeof value === "object" && @@ -8477,13 +8486,15 @@ export class TaskService { : {}), }); } - // Deliver one initiating-agent group per drain: the whole coalesced prompt is handled - // under the single agentId passed to sendMessage, so batching runs from different agents - // would hand a restricted agent's (attacker-influenced) output to another agent's tool - // grants. The same applies to mixed batches: workspace-turn and sub-agent attention - // resumes under the conversation's own (history-walk) identity, so agent-bound workflow - // groups never share their send. Deferred groups stay pending and deliver on the re-armed - // retry drain; among agent-bound groups the newest launch goes first. + // Deliver one launch-identity group per drain, keyed by agentId AND recorded strict pin: + // the whole coalesced prompt is handled under the single agentId/pin passed to + // sendMessage, so batching runs from different agents (or runs sharing an agentId but + // launched under different pins, e.g. an agent definition replaced between synthetic + // turns) would hand a restricted launch's (attacker-influenced) output to another + // launch's tool grants. The same applies to mixed batches: workspace-turn and sub-agent + // attention resumes under the conversation's own (history-walk) identity, so agent-bound + // workflow groups never share their send. Deferred groups stay pending and deliver on the + // re-armed retry drain; among agent-bound groups the newest launch goes first. const hasNonWorkflowDeliverables = deliverableAgentNotificationIds.size > 0 || deliverableWorkspaceTurnNotificationIds.size > 0; let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; @@ -8499,11 +8510,14 @@ export class TaskService { } } } - const selectedAgentId = workflowInitiatingAgent?.agentId; + const selectedGroupKey = + workflowInitiatingAgent != null ? workflowWakeGroupKey(workflowInitiatingAgent) : undefined; const selectedWorkflowPrompts = deliverableWorkflowPrompts.filter((candidate) => hasNonWorkflowDeliverables ? candidate.initiatingAgent == null - : selectedAgentId == null || candidate.initiatingAgent?.agentId === selectedAgentId + : selectedGroupKey == null || + (candidate.initiatingAgent != null && + workflowWakeGroupKey(candidate.initiatingAgent) === selectedGroupKey) ); if (selectedWorkflowPrompts.length < deliverableWorkflowPrompts.length) { this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index 008acd2a5ce..03ccf20442f 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -1007,3 +1007,59 @@ describe("WorkflowRunStore.listActiveRunSummaries", () => { expect(summaries.map((summary) => summary.runId)).toEqual(["wfr_healthy"]); }); }); + +describe("WorkflowService crash recovery", () => { + test("crash resume fires provenance repair before the run reaches terminal", async () => { + using tmp = new DisposableTempDir("workflow-service-crash-repair"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_crash", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo workflow", scope: "built-in", executable: true }, + source: 'export default function workflow() { return { reportMarkdown: "done" }; }\n', + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + // Orphaned by a crash: durable status says running, but no runner holds the lease. + await runStore.appendStatus("wfr_crash", "running", "2026-05-29T00:00:01.000Z"); + + const events: string[] = []; + let resolveCompleted: (() => void) | undefined; + const completed = new Promise((resolve) => { + resolveCompleted = resolve; + }); + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + generateRunId: () => "wfr_unused", + runnerId: "runner-a", + onRunCrashResumed: (event) => { + events.push(`repair:${event.workspaceId}:${event.runId}`); + }, + onRunStatusChanged: (event) => { + events.push(`status:${event.status}`); + if (event.status === "completed") { + resolveCompleted?.(); + } + }, + }); + + const resumed = await service.resumeCrashedRuns({ + workspaceId: "workspace-1", + projectTrusted: true, + }); + expect(resumed).toEqual(["wfr_crash"]); + await completed; + // The repair hook is awaited before the runner restarts, so even an instantly completing + // run cannot reach terminal with unrepaired provenance. + expect(events[0]).toBe("repair:workspace-1:wfr_crash"); + expect(events).toContain("status:completed"); + await expect(runStore.getRun("wfr_crash")).resolves.toMatchObject({ status: "completed" }); + }); +}); diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 70b8fb35f2a..94c90f8e528 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -54,6 +54,12 @@ export interface WorkflowServiceOptions { resolveWorkflowScript?: (scriptPath: string) => Promise; onBackgroundRunTerminal?: (event: WorkflowBackgroundRunTerminalEvent) => Promise | void; onRunStatusChanged?: (event: WorkflowRunStatusChangedEvent) => Promise | void; + /** + * Fired when crash recovery is about to resume an orphaned run, before the runner restarts. + * Used to repair wake provenance a pre-boundary build stripped from the sidecar; awaited so + * a fast run cannot reach terminal before the repair lands. + */ + onRunCrashResumed?: (event: { workspaceId: string; runId: string }) => Promise | void; /** When true, background terminal notifications also fire for interrupted runs. */ notifyInterruptedBackgroundRunTerminal?: boolean; generateRunId?: () => string; @@ -127,6 +133,10 @@ export class WorkflowService { private readonly onBackgroundRunTerminal?: ( event: WorkflowBackgroundRunTerminalEvent ) => Promise | void; + private readonly onRunCrashResumed?: (event: { + workspaceId: string; + runId: string; + }) => Promise | void; private readonly onRunStatusChanged?: ( event: WorkflowRunStatusChangedEvent ) => Promise | void; @@ -150,6 +160,7 @@ export class WorkflowService { this.taskAdapterFactory = options.taskAdapterFactory; this.resolveWorkflowScript = options.resolveWorkflowScript; this.onBackgroundRunTerminal = options.onBackgroundRunTerminal; + this.onRunCrashResumed = options.onRunCrashResumed; this.onRunStatusChanged = options.onRunStatusChanged; this.notifyInterruptedBackgroundRunTerminal = options.notifyInterruptedBackgroundRunTerminal === true; @@ -573,6 +584,16 @@ export class WorkflowService { return false; } + if (this.onRunCrashResumed != null) { + try { + await this.onRunCrashResumed({ workspaceId: run.workspaceId, runId: run.id }); + } catch (error) { + // Best-effort: an unrepaired reference defers its wake as indeterminate rather than + // losing it, so a failed repair must not block the resume itself. + console.error("Workflow crash-resume provenance repair failed:", error); + } + } + const retryDelayMs = await this.runStore.getLeaseRetryDelayMs( input.runId, this.clock?.nowMs() ?? Date.now() diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 6ee594182cc..72bb510eaf4 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -59,7 +59,10 @@ import { WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowResultContextMessage, } from "@/common/utils/workflowRunMessages"; -import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; +import { + readAgentWorkflowRunReferences, + recordAgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; import * as todoStorageModule from "@/node/services/todos/todoStorage"; import * as runtimeFactory from "@/node/runtime/runtimeFactory"; @@ -6469,6 +6472,91 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("crash-resume repair re-snapshots only boundaryless references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-crash-repair"; + const strippedRunId = "wfr_crash_repair_stripped"; + const anchoredRunId = "wfr_crash_repair_anchored"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-crash-repair", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // A downgrade rewrote the sidecar without boundary fields (or a record-time read failure + // omitted them): crash-resume repair re-snapshots the boundary, keeping the recorded + // launch identity, and the deferred wake becomes deliverable again. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: strippedRunId, + createdAtMs: 1_150, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, strippedRunId); + const repaired = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect(repaired.find((reference) => reference.runId === strippedRunId)).toMatchObject({ + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, strippedRunId) + ).toBe("current"); + + // A reference that still carries its boundary may record a pre-supersession launch: + // repair must not refresh it into the current context. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: anchoredRunId, + createdAtMs: 1_050, + afterBoundaryMessageId: "older-row", + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, anchoredRunId); + const untouched = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect( + untouched.find((reference) => reference.runId === anchoredRunId)?.afterBoundaryMessageId + ).toBe("older-row"); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, anchoredRunId) + ).toBe("not_current"); + + // Unknown run: nothing to repair, nothing recorded. + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, "wfr_unknown"); + const after = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect(after.map((reference) => reference.runId).sort()).toEqual([ + anchoredRunId, + strippedRunId, + ]); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("treats an unreadable history as indeterminate, not superseded", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-io-error"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 43688444a46..faaecf2b352 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6,6 +6,7 @@ import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; import { clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, + recordAgentWorkflowRunReference, type AgentWorkflowRunReference, } from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; @@ -11122,6 +11123,45 @@ export class WorkspaceService extends EventEmitter { return decision.status === "found" ? decision.messageId : null; } + /** + * Re-snapshot the boundary for a run reference that lost it: a pre-boundary build rewrites + * the sidecar with only runId/createdAtMs on any record (upgrade -> downgrade -> upgrade + * strips the field), and a boundaryless reference defers its terminal wake as indeterminate + * until provenance is re-established. Crash recovery calls this before restarting an + * orphaned run: the run is verifiably non-terminal there, so a resume-time snapshot is + * legitimate launch provenance for the continued execution, mirroring the workflow_resume + * tool's re-record. References that still carry a boundary (including verified-empty null) + * are left untouched: refreshing them would forgive manual supersessions on every restart. + */ + async repairWorkflowRunReferenceBoundary(workspaceId: string, runId: string): Promise { + assert(workspaceId.length > 0, "repairWorkflowRunReferenceBoundary requires workspaceId"); + assert(runId.length > 0, "repairWorkflowRunReferenceBoundary requires runId"); + const sessionDir = this.config.getSessionDir(workspaceId); + const references = await readAgentWorkflowRunReferences(sessionDir); + const reference = references.find((candidate) => candidate.runId === runId); + if (reference == null || reference.afterBoundaryMessageId !== undefined) { + return; + } + const afterBoundaryMessageId = await this.getWorkflowInvocationBoundaryMessageId( + workspaceId, + runId + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: sessionDir, + runId, + createdAtMs: reference.createdAtMs, + afterBoundaryMessageId, + ...(reference.agentId != null + ? { + agentId: reference.agentId, + ...(reference.strictAgentResolution !== undefined + ? { strictAgentResolution: reference.strictAgentResolution } + : {}), + } + : {}), + }); + } + /** * Increment a preflight admission counter in the caller's synchronous entry block and * return a disposable releasing it. Pairs renderer-initiated workspace activity with the From 578d72bf2b2385a91d7c02a6ee6347bfe034421c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:32:50 +0000 Subject: [PATCH 35/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20gate=20crash-resume?= =?UTF-8?q?=20provenance=20repair=20on=20supersession-free=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/workspaceService.test.ts | 39 +++++++++++++++++----- src/node/services/workspaceService.ts | 29 +++++++++++----- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 72bb510eaf4..9efd4e1d6cc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6472,10 +6472,11 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("crash-resume repair re-snapshots only boundaryless references", async () => { + test("crash-resume repair restores provenance only on supersession-free evidence", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-crash-repair"; const strippedRunId = "wfr_crash_repair_stripped"; + const supersededRunId = "wfr_crash_repair_superseded"; const anchoredRunId = "wfr_crash_repair_anchored"; const projectPath = path.join(config.rootDir, "project"); try { @@ -6501,13 +6502,9 @@ describe("WorkspaceService workflow invocation events", () => { } as unknown as InitStateManager, }); - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) - ); - // A downgrade rewrote the sidecar without boundary fields (or a record-time read failure - // omitted them): crash-resume repair re-snapshots the boundary, keeping the recorded - // launch identity, and the deferred wake becomes deliverable again. + // A downgrade rewrote the sidecar without boundary fields. With a decision-free history + // the repair has supersession-free evidence: it records a verified-empty boundary, + // keeps the recorded launch identity, and the deferred wake becomes deliverable. await recordAgentWorkflowRunReference({ workspaceSessionDir: config.getSessionDir(workspaceId), runId: strippedRunId, @@ -6519,7 +6516,7 @@ describe("WorkspaceService workflow invocation events", () => { const repaired = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); expect(repaired.find((reference) => reference.runId === strippedRunId)).toMatchObject({ createdAtMs: 1_150, - afterBoundaryMessageId: "manual-user", + afterBoundaryMessageId: null, agentId: "plan", strictAgentResolution: { expectedScope: "built-in" }, }); @@ -6527,6 +6524,29 @@ describe("WorkspaceService workflow invocation events", () => { await workspaceService.getWorkflowInvocationCurrentness(workspaceId, strippedRunId) ).toBe("current"); + // Once a manual row is the newest decision row, a stripped launch cannot be ordered + // against it: the run may predate the supersession, so repair must refuse and the wake + // must stay deferred rather than resurrect a possibly superseded result. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, do something else", { + timestamp: 1_200, + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: supersededRunId, + createdAtMs: 1_100, + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, supersededRunId); + const refused = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + const refusedReference = refused.find((reference) => reference.runId === supersededRunId); + expect(refusedReference).toBeDefined(); + expect(refusedReference != null && "afterBoundaryMessageId" in refusedReference).toBe(false); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, supersededRunId) + ).toBe("indeterminate"); + // A reference that still carries its boundary may record a pre-supersession launch: // repair must not refresh it into the current context. await recordAgentWorkflowRunReference({ @@ -6550,6 +6570,7 @@ describe("WorkspaceService workflow invocation events", () => { expect(after.map((reference) => reference.runId).sort()).toEqual([ anchoredRunId, strippedRunId, + supersededRunId, ]); workspaceService.disposeSession(workspaceId); } finally { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index faaecf2b352..f5c7c552492 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -11128,10 +11128,15 @@ export class WorkspaceService extends EventEmitter { * the sidecar with only runId/createdAtMs on any record (upgrade -> downgrade -> upgrade * strips the field), and a boundaryless reference defers its terminal wake as indeterminate * until provenance is re-established. Crash recovery calls this before restarting an - * orphaned run: the run is verifiably non-terminal there, so a resume-time snapshot is - * legitimate launch provenance for the continued execution, mirroring the workflow_resume - * tool's re-record. References that still carry a boundary (including verified-empty null) - * are left untouched: refreshing them would forgive manual supersessions on every restart. + * orphaned run, but the repair proceeds only on supersession-free evidence: a decision-free + * history (recorded as a verified-empty boundary) or a newest decision row that belongs to + * this run. A newest manual/reset row is refused: the stripped launch cannot be ordered + * against it by identity, and snapshotting it would resurrect a possibly pre-supersession + * result into the newer conversation (the same reference with a surviving boundary would + * stay not_current). Those wakes stay deferred until an explicit workflow_resume, which + * carries current-context intent. References that still carry a boundary (including + * verified-empty null) are left untouched: refreshing them would forgive manual + * supersessions on every restart. */ async repairWorkflowRunReferenceBoundary(workspaceId: string, runId: string): Promise { assert(workspaceId.length > 0, "repairWorkflowRunReferenceBoundary requires workspaceId"); @@ -11142,15 +11147,21 @@ export class WorkspaceService extends EventEmitter { if (reference == null || reference.afterBoundaryMessageId !== undefined) { return; } - const afterBoundaryMessageId = await this.getWorkflowInvocationBoundaryMessageId( - workspaceId, - runId - ); + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + if (decision.status === "error") { + throw new Error("workflow invocation boundary unavailable: history read failed"); + } + if (decision.status === "found" && decision.outcome === "superseded") { + return; + } + // Supersession-free evidence only: no decision row at all (verified-empty null), or the + // newest decision row is this run's own invocation/consumed row, which no manual row can + // postdate (the backward walk would have found that manual row first). await recordAgentWorkflowRunReference({ workspaceSessionDir: sessionDir, runId, createdAtMs: reference.createdAtMs, - afterBoundaryMessageId, + afterBoundaryMessageId: decision.status === "found" ? decision.messageId : null, ...(reference.agentId != null ? { agentId: reference.agentId, From 56171b714c4b5bffc59fdf81a8c7da97932d0ec1 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:54:32 +0000 Subject: [PATCH 36/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20make=20crash-resume?= =?UTF-8?q?=20boundary=20repair=20a=20compare-and-set=20under=20the=20side?= =?UTF-8?q?car=20lock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 77 +++++++++++++++++++ .../services/agentWorkflowRunReferences.ts | 34 ++++++++ src/node/services/workspaceService.ts | 18 ++--- 3 files changed, 117 insertions(+), 12 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 1939e4f305a..7ecab498c72 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -7,6 +7,7 @@ import { describe, expect, test } from "bun:test"; import { readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, + repairAgentWorkflowRunReferenceBoundary, } from "@/node/services/agentWorkflowRunReferences"; describe("agent workflow run references", () => { @@ -298,4 +299,80 @@ describe("agent workflow run references", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); + + test("boundary repair is a compare-and-set on a surviving boundaryless reference", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_repairable"; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_bystander", + createdAtMs: 1_100, + afterBoundaryMessageId: "row-1", + }); + + // Repairs in place, preserving the rest of the entry and its neighbors. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId, + afterBoundaryMessageId: null, + }) + ).toBe(true); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references.find((reference) => reference.runId === runId)).toEqual({ + runId, + createdAtMs: 1_000, + afterBoundaryMessageId: null, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(references.find((reference) => reference.runId === "wfr_bystander")).toEqual({ + runId: "wfr_bystander", + createdAtMs: 1_100, + afterBoundaryMessageId: "row-1", + }); + + // A reference that already carries a boundary (here the one just repaired) is never + // overwritten: a concurrent explicit re-record must win over a stale repair. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId, + afterBoundaryMessageId: "stale-row", + }) + ).toBe(false); + const unchanged = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(unchanged.find((reference) => reference.runId === runId)?.afterBoundaryMessageId).toBe( + null + ); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("boundary repair refuses to recreate a cleared sidecar", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A full-history clear deleted the sidecar between the repair's reads and its write: + // the stale repair must not resurrect the retired reference as verified-empty current. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId: "wfr_cleared", + afterBoundaryMessageId: null, + }) + ).toBe(false); + expect(await fs.readdir(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 6e2ff131194..01d5cba36ae 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -233,3 +233,37 @@ export async function recordAgentWorkflowRunReference(input: { ); }); } + +/** + * Compare-and-set boundary repair for a surviving boundaryless reference. The reference is + * re-validated under the sidecar file lock: a concurrent full-history clear deletes the file + * (retiring every reference), and an unconditional write would recreate it with a + * verified-empty boundary, resurrecting the retired pre-clear result as "current" in the + * freshly cleared conversation. A reference that concurrently gained a boundary (explicit + * workflow_resume re-record) is also left untouched. Returns false without writing when the + * reference is gone or already carries a boundary. + */ +export async function repairAgentWorkflowRunReferenceBoundary(input: { + workspaceSessionDir: string; + runId: string; + afterBoundaryMessageId: string | null; +}): Promise { + assert(input.runId.length > 0, "agent workflow reference repair requires runId"); + const filePath = referencesPath(input.workspaceSessionDir); + + return referenceFileLocks.withLock(filePath, async () => { + const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + const reference = existing.find((candidate) => candidate.runId === input.runId); + if (reference == null || reference.afterBoundaryMessageId !== undefined) { + return false; + } + const references = existing.map((candidate) => + candidate.runId === input.runId + ? { ...candidate, afterBoundaryMessageId: input.afterBoundaryMessageId } + : candidate + ); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await writeFileAtomic(filePath, JSON.stringify({ references }, null, 2)); + return true; + }); +} diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index f5c7c552492..40dbef1620e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6,7 +6,7 @@ import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; import { clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, - recordAgentWorkflowRunReference, + repairAgentWorkflowRunReferenceBoundary, type AgentWorkflowRunReference, } from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; @@ -11156,20 +11156,14 @@ export class WorkspaceService extends EventEmitter { } // Supersession-free evidence only: no decision row at all (verified-empty null), or the // newest decision row is this run's own invocation/consumed row, which no manual row can - // postdate (the backward walk would have found that manual row first). - await recordAgentWorkflowRunReference({ + // postdate (the backward walk would have found that manual row first). The write is a + // compare-and-set under the sidecar lock: a full clear landing after the reads above + // deletes the sidecar, and an unconditional record would recreate it with a + // verified-empty boundary, resurrecting the retired pre-clear result as "current". + await repairAgentWorkflowRunReferenceBoundary({ workspaceSessionDir: sessionDir, runId, - createdAtMs: reference.createdAtMs, afterBoundaryMessageId: decision.status === "found" ? decision.messageId : null, - ...(reference.agentId != null - ? { - agentId: reference.agentId, - ...(reference.strictAgentResolution !== undefined - ? { strictAgentResolution: reference.strictAgentResolution } - : {}), - } - : {}), }); } From 6776f483e89c00e35a192343e7969392dee28671 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:04:03 +0000 Subject: [PATCH 37/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20defer=20identity-le?= =?UTF-8?q?ss=20wakes=20and=20wire=20terminal=20attention=20into=20crash-r?= =?UTF-8?q?esumed=20services?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/orpc/router.test.ts | 39 ++++++++++++++++++++ src/node/orpc/router.ts | 21 +++++++++-- src/node/services/taskService.test.ts | 53 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 8 +++- 4 files changed, 116 insertions(+), 5 deletions(-) diff --git a/src/node/orpc/router.test.ts b/src/node/orpc/router.test.ts index 62db8bbac95..5d164ce80ee 100644 --- a/src/node/orpc/router.test.ts +++ b/src/node/orpc/router.test.ts @@ -951,6 +951,45 @@ export default function workflow() { return { reportMarkdown: "should not run" } expect(result.result).toBeNull(); await waitForRouterWorkflowStatus(client, "workspace-1", result.runId, "completed"); }); + + test("crash-resumed background runs enqueue terminal attention on settle", async () => { + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); + await runStore.createRun({ + id: "wfr_crash_wake", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + // Orphaned by a crash: durable status says running, but no live runner. + await runStore.appendStatus("wfr_crash_wake", "running", "2026-05-29T00:00:01.000Z"); + + const enqueueWorkflowRunTerminalAttention = mock(async () => undefined); + const context = createContext({ enabled: true }); + (context as unknown as Record).taskService = { + enqueueWorkflowRunTerminalAttention, + }; + ( + context.workspaceService as unknown as Record + ).repairWorkflowRunReferenceBoundary = mock(async () => undefined); + const client = createRouterClient(router(), { context }); + + // A read path triggers crash recovery; the resumed run's settle must land in the + // terminal-attention outbox instead of waiting for the next restart's sweep. + await client.workflows.listRuns({ workspaceId: "workspace-1" }); + await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_crash_wake", "completed"); + const deadline = Date.now() + 5_000; + while (enqueueWorkflowRunTerminalAttention.mock.calls.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(enqueueWorkflowRunTerminalAttention).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_crash_wake", + status: "completed", + }); + }); }); describe("router config.saveConfig", () => { diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index 327db8336f9..f809a63e7be 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -586,9 +586,24 @@ export async function resolveWorkflowContext( onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), onRunCrashResumed: (event) => context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId), - ...(options.onBackgroundRunTerminal != null - ? { onBackgroundRunTerminal: options.onBackgroundRunTerminal } - : {}), + // Read paths (listRuns / stream subscribe) create services purely to observe runs, but + // crash recovery can resume an orphaned background run on them: without a terminal + // callback the settled run would enqueue no terminal attention until the next restart's + // sweep. Default to the standard outbox enqueue (idempotent via enqueueIfAbsent); + // explicit callbacks (slash-command continuations, retry) keep their custom behavior. + onBackgroundRunTerminal: + options.onBackgroundRunTerminal ?? + (async (event) => { + // Nested runs surface through their parent workflow, not their own wake. + if (event.run.parentWorkflow != null) { + return; + } + await context.taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: workspaceId, + runId: event.runId, + status: event.status, + }); + }), getCurrentProjectTrusted: resolveWorkflowProjectTrusted, runnerId: `workflow-runner:${workspaceId}`, }), diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 0316e4089f1..8bb07ce4e29 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6964,6 +6964,59 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); }); + test("wake defers when the launch-identity read fails after currentness succeeds", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_identity_unreadable"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Currentness succeeds without the sidecar (e.g. a direct invocation row)... + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + const terminalAttentionStore = new TerminalAttentionStore(config); + + // ...but the launch-identity read fails transiently (EISDIR). Delivering without the + // recorded identity would bind the wake to the newest agent-bearing history row, so the + // wake must stay pending for the retry drain. + await fsPromises.mkdir(path.join(config.getSessionDir(parentId), "agent-workflow-runs.json"), { + recursive: true, + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + test("wake re-pins the selected group's recorded launch pin, not the newest row's", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 89bafb64a7a..3f399b09f42 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8122,8 +8122,12 @@ export class TaskService { }; } } catch { - // Identity is advisory; an unreadable sidecar already deferred delivery above whenever - // currentness itself depended on it. + // Currentness can succeed (e.g. a direct invocation row) and this identity read still + // fail transiently. Delivering without the recorded identity would bind the wake to the + // newest agent-bearing history row, handing the run's output to an unrelated later + // synthetic turn's agent; defer to the bounded retry instead, like an unreadable run + // record. + return { outcome: "defer" }; } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; return { From ce9085a89c48202e0c0349e898b109d08e73f75c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:00:35 +0000 Subject: [PATCH 38/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20retain=20and=20retr?= =?UTF-8?q?y=20failed=20workflow=20terminal=20attention=20enqueues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 129 ++++++++++++++++++++++++++ src/node/services/taskService.ts | 69 ++++++++++++-- 2 files changed, 192 insertions(+), 6 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 8bb07ce4e29..de36f151129 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6355,6 +6355,135 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("failed terminal attention enqueue is retained and retried on the bounded timer", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_enqueue_retry"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + ( + taskService as unknown as { terminalAttentionDeferRetryDelayMs: number } + ).terminalAttentionDeferRetryDelayMs = 10; + + // The workflow terminal callback driving this enqueue is single-attempt, so when the + // first outbox write fails only the retained in-process retry can persist the wake. + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( + new Error("EIO: outbox write failed") + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + + // Real timers: poll until the armed retry re-enqueues and the follow-up drain delivers. + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && sendMessage.mock.calls.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 10)); + await flushTerminalAttentionDrains(taskService); + } + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("reset drops a retained terminal attention enqueue", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_enqueue_reset"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( + new Error("EIO: outbox write failed") + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + // The run was reset (e.g. resumed) before the retry fired; the retained stale wake + // must be dropped instead of resurrected by the retry. + await taskService.resetWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId }); + + // Drive the armed retry directly so the outcome is deterministic under real timers. + await ( + taskService as unknown as { + retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise; + } + ).retryRetainedWorkflowTerminalEnqueues(parentId); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("workflow wakes restore the caller tool policy from the newest manual row", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 3f399b09f42..41d75da60c1 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1569,6 +1569,19 @@ export class TaskService { ReturnType >(); private terminalAttentionDeferRetryDelayMs = TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS; + // Workflow terminal callbacks are single-attempt (WorkflowService swallows callback + // rejections), so a transient outbox write failure would otherwise silence the wake for + // the life of the process. Retain failed enqueue params (owner -> runId -> status) and + // retry on the defer-retry cadence; reset drops the entry so a resumed run cannot + // resurrect its stale terminal wake. + private readonly retainedWorkflowTerminalEnqueues = new Map< + string, + Map + >(); + private readonly workflowTerminalEnqueueRetryTimers = new Map< + string, + ReturnType + >(); private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); // Tracks workspaces currently blocked in a foreground wait (e.g. a task tool call awaiting @@ -7748,12 +7761,55 @@ export class TaskService { if (!isTerminalWorkflowRunStatus(params.status)) { return; } - await this.enqueueTerminalAttention({ - ownerWorkspaceId: params.ownerWorkspaceId, - sourceKind: "workflow_run", - terminalOutcome: terminalAttentionOutcome(params.status), - sourceId: params.runId, - }); + try { + await this.enqueueTerminalAttention({ + ownerWorkspaceId: params.ownerWorkspaceId, + sourceKind: "workflow_run", + terminalOutcome: terminalAttentionOutcome(params.status), + sourceId: params.runId, + }); + } catch (error) { + log.error("Workflow terminal attention enqueue failed; retrying on bounded timer", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + error, + }); + this.retainWorkflowTerminalEnqueue(params); + } + } + + private retainWorkflowTerminalEnqueue(params: { + ownerWorkspaceId: string; + runId: string; + status: WorkflowRunStatus; + }): void { + let byRun = this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId); + if (byRun == null) { + byRun = new Map(); + this.retainedWorkflowTerminalEnqueues.set(params.ownerWorkspaceId, byRun); + } + byRun.set(params.runId, params.status); + if (this.workflowTerminalEnqueueRetryTimers.has(params.ownerWorkspaceId)) { + return; + } + const timer = setTimeout(() => { + this.workflowTerminalEnqueueRetryTimers.delete(params.ownerWorkspaceId); + void this.retryRetainedWorkflowTerminalEnqueues(params.ownerWorkspaceId); + }, this.terminalAttentionDeferRetryDelayMs); + timer.unref?.(); + this.workflowTerminalEnqueueRetryTimers.set(params.ownerWorkspaceId, timer); + } + + private async retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise { + const byRun = this.retainedWorkflowTerminalEnqueues.get(ownerWorkspaceId); + if (byRun == null) { + return; + } + this.retainedWorkflowTerminalEnqueues.delete(ownerWorkspaceId); + for (const [runId, status] of byRun) { + // A re-attempt that fails again re-retains the entry and re-arms the timer. + await this.enqueueWorkflowRunTerminalAttention({ ownerWorkspaceId, runId, status }); + } } async resetWorkflowRunTerminalAttention(params: { @@ -7765,6 +7821,7 @@ export class TaskService { "resetWorkflowRunTerminalAttention requires ownerWorkspaceId" ); assert(params.runId.length > 0, "resetWorkflowRunTerminalAttention requires runId"); + this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId)?.delete(params.runId); await this.terminalAttentionStore.delete( params.ownerWorkspaceId, TerminalAttentionStore.notificationId("workflow_run", params.runId) From 7edfe81c193e6e7b29f29f8a9465ac0a477cbf46 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:49:06 +0000 Subject: [PATCH 39/39] =?UTF-8?q?=F0=9F=A4=96=20fix:=20harden=20workflow?= =?UTF-8?q?=20wake=20recovery=20(resume=20reset,=20repair=20retry,=20reset?= =?UTF-8?q?-boundary=20stop)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/orpc/router.test.ts | 41 ++++++++++- src/node/orpc/router.ts | 15 +++- src/node/services/taskService.test.ts | 72 +++++++++++++++++++ src/node/services/taskService.ts | 12 +++- .../workflows/WorkflowService.test.ts | 58 +++++++++++++++ .../services/workflows/WorkflowService.ts | 40 ++++++++++- 6 files changed, 233 insertions(+), 5 deletions(-) diff --git a/src/node/orpc/router.test.ts b/src/node/orpc/router.test.ts index 5d164ce80ee..ccfc19616b7 100644 --- a/src/node/orpc/router.test.ts +++ b/src/node/orpc/router.test.ts @@ -362,7 +362,11 @@ describe("router workflow routes", () => { getWorkflowContinuationSendOptions: mock(() => null), sendMessage: mock(async () => ({ success: true, data: undefined })), }, - taskService: {}, + // Nonterminal run status changes reset any stale terminal notification; the stub keeps + // that call observable without wiring a full TaskService. + taskService: { + resetWorkflowRunTerminalAttention: mock(async () => undefined), + }, experimentsService: { isExperimentEnabled: mock(() => options.enabled), }, @@ -970,6 +974,7 @@ export default function workflow() { return { reportMarkdown: "should not run" } const context = createContext({ enabled: true }); (context as unknown as Record).taskService = { enqueueWorkflowRunTerminalAttention, + resetWorkflowRunTerminalAttention: mock(async () => undefined), }; ( context.workspaceService as unknown as Record @@ -990,6 +995,40 @@ export default function workflow() { return { reportMarkdown: "should not run" } status: "completed", }); }); + + test("router-managed resume resets a stale terminal notification before restart", async () => { + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); + await runStore.createRun({ + id: "wfr_resume_reset", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + + const context = createContext({ enabled: true }); + const resetWorkflowRunTerminalAttention = ( + context.taskService as unknown as { + resetWorkflowRunTerminalAttention: ReturnType; + } + ).resetWorkflowRunTerminalAttention; + const client = createRouterClient(router(), { context }); + + await client.workflows.interrupt({ workspaceId: "workspace-1", runId: "wfr_resume_reset" }); + resetWorkflowRunTerminalAttention.mockClear(); + + // The prior run's notification survives under the stable workflow_run: id as + // delivered/superseded; without a reset on restart, enqueueIfAbsent preserves that + // record and the resumed run's terminal wake is silently dropped. + await client.workflows.resume({ workspaceId: "workspace-1", runId: "wfr_resume_reset" }); + expect(resetWorkflowRunTerminalAttention).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_reset", + }); + await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_resume_reset", "completed"); + }); }); describe("router config.saveConfig", () => { diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index f809a63e7be..02feeed6332 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -116,6 +116,7 @@ import * as path from "node:path"; import type { DevToolsEvent } from "@/common/types/devtools"; import type { WorkflowRunStreamEvent } from "@/common/types/workflow"; +import { isTerminalWorkflowRunStatus } from "@/common/types/workflow"; import type { WorkflowRunLivenessEntry } from "@/common/orpc/schemas/api"; import type { MuxMessage } from "@/common/types/message"; import { coerceThinkingLevel } from "@/common/types/thinking"; @@ -583,7 +584,19 @@ export async function resolveWorkflowContext( includeAgentPlugins, skillStorageContext, }), - onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), + onRunStatusChanged: async (event) => { + // Router-managed restarts (resume / retry / crash recovery) must clear a prior + // delivered or superseded notification when the run leaves terminal state, or + // enqueueIfAbsent would preserve the stale record and silently drop the resumed + // run's next terminal wake (mirrors the AIService-owned service). + if (!isTerminalWorkflowRunStatus(event.status)) { + await context.taskService.resetWorkflowRunTerminalAttention({ + ownerWorkspaceId: event.workspaceId, + runId: event.runId, + }); + } + await context.workspaceService.emitWorkflowRunActivity(event); + }, onRunCrashResumed: (event) => context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId), // Read paths (listRuns / stream subscribe) create services purely to observe runs, but diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index de36f151129..6162817435f 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6484,6 +6484,78 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_policy_reset_boundary"; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // The pre-reset manual row disabled bash, but the context reset discarded that + // conversation. The workflow launched from a post-reset synthetic turn (heartbeat), so + // its wake must use fresh defaults instead of resurrecting the discarded restriction. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("reset-boundary", "assistant", "Context reset", { + timestamp: 2_000, + contextBoundaryKind: "reset", + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat-launch", "user", "[heartbeat] launch the workflow", { + timestamp: 3_000, + synthetic: true, + }) + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const options = sendMessage.mock.calls[0]?.[2] as { + toolPolicy?: unknown; + disableWorkspaceAgents?: unknown; + }; + expect(options.toolPolicy).toBeUndefined(); + expect(options.disableWorkspaceAgents).toBeUndefined(); + }); + test("workflow wakes restore the caller tool policy from the newest manual row", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 41d75da60c1..55f46a1245e 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7994,8 +7994,10 @@ export class TaskService { * conversation's persisted restrictions; synthetic rows without any (earlier wakes, * heartbeat scaffolding) do not define them and are skipped. The walk is unbounded: a long * assistant/synthetic tail after the launch turn must not push the defining row out of - * sight and silently lift the restrictions. Throws when history is unreadable so the caller - * can fail closed instead of waking with unrestricted tools. + * sight and silently lift the restrictions. It stops at the newest context reset boundary, + * since rows from the discarded context must not re-disable tools available to the reset + * context. Throws when history is unreadable so the caller can fail closed instead of + * waking with unrestricted tools. */ private async resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId: string): Promise<{ toolPolicy?: ToolPolicy; @@ -8015,6 +8017,12 @@ export class TaskService { "backward", (messages) => { for (const message of messages) { + // A context reset discards everything before it: pre-reset rows must not define + // the wake's restrictions or pin. Stopping here leaves undefined fields as fresh + // defaults, matching a manual send in the post-reset context. + if (isResetBoundaryMessage(message)) { + return false; + } if (message.role !== "user") { continue; } diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index 03ccf20442f..bfcb2ce0814 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -1062,4 +1062,62 @@ describe("WorkflowService crash recovery", () => { expect(events).toContain("status:completed"); await expect(runStore.getRun("wfr_crash")).resolves.toMatchObject({ status: "completed" }); }); + + test("failed crash-resume provenance repair retries on a bounded timer", async () => { + using tmp = new DisposableTempDir("workflow-service-crash-repair-retry"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_crash_repair_retry", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo workflow", scope: "built-in", executable: true }, + source: 'export default function workflow() { return { reportMarkdown: "done" }; }\n', + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_crash_repair_retry", "running", "2026-05-29T00:00:01.000Z"); + + // The repair hook only fires at resume time; if its transient failure were terminal, the + // reference would stay boundaryless after the run settles and every drain would defer the + // wake as indeterminate with nothing left to repair it. + const repairCalls: string[] = []; + let failFirstRepair = true; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + generateRunId: () => "wfr_unused", + runnerId: "runner-a", + onRunCrashResumed: (event) => { + repairCalls.push(`${event.workspaceId}:${event.runId}`); + if (failFirstRepair) { + failFirstRepair = false; + throw new Error("EIO: sidecar write failed"); + } + }, + }); + ( + service as unknown as { crashResumeRepairRetryDelayMs: number } + ).crashResumeRepairRetryDelayMs = 10; + + const resumed = await service.resumeCrashedRuns({ + workspaceId: "workspace-1", + projectTrusted: true, + }); + expect(resumed).toEqual(["wfr_crash_repair_retry"]); + expect(repairCalls).toEqual(["workspace-1:wfr_crash_repair_retry"]); + + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && repairCalls.length < 2) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(repairCalls).toEqual([ + "workspace-1:wfr_crash_repair_retry", + "workspace-1:wfr_crash_repair_retry", + ]); + }); }); diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 94c90f8e528..83fa80f363e 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -118,6 +118,13 @@ const WORKFLOW_BACKGROUND_CONTINUATION_STATUSES = new Set([ // oRPC creates a WorkflowService per request, so workflow lifecycle state that spans requests // needs process-wide registries. const pendingCrashResumeTimers = new Map>(); +// Crash-resume provenance repair only fires at resume time: once the run settles, nothing +// else re-records the boundary and its terminal wake stays indeterminate on every drain. +// Retry a failed repair on a bounded timer. Module-level like pendingCrashResumeTimers +// because WorkflowService instances are per-request. +const CRASH_RESUME_REPAIR_RETRY_DELAY_MS = 30_000; +const CRASH_RESUME_REPAIR_MAX_ATTEMPTS = 5; +const pendingCrashResumeRepairTimers = new Map>(); const activeWorkflowInterruptStatusWrites = new Map>(); const activeWorkflowRunnerAbortControllers = new Map(); @@ -137,6 +144,8 @@ export class WorkflowService { workspaceId: string; runId: string; }) => Promise | void; + // Field, not the constant, so tests can shrink the repair retry backoff. + private crashResumeRepairRetryDelayMs = CRASH_RESUME_REPAIR_RETRY_DELAY_MS; private readonly onRunStatusChanged?: ( event: WorkflowRunStatusChangedEvent ) => Promise | void; @@ -589,8 +598,11 @@ export class WorkflowService { await this.onRunCrashResumed({ workspaceId: run.workspaceId, runId: run.id }); } catch (error) { // Best-effort: an unrepaired reference defers its wake as indeterminate rather than - // losing it, so a failed repair must not block the resume itself. + // losing it, so a failed repair must not block the resume itself. Retry off-path: + // repair is CAS-guarded and refuses once a boundary exists, so late success (even + // after the run settles) only unblocks the deferred wake. console.error("Workflow crash-resume provenance repair failed:", error); + this.scheduleCrashResumeRepairRetry({ workspaceId: run.workspaceId, runId: run.id }, 1); } } @@ -651,6 +663,32 @@ export class WorkflowService { pendingCrashResumeTimers.set(input.runId, timer); } + private scheduleCrashResumeRepairRetry( + input: { workspaceId: string; runId: string }, + attempt: number + ): void { + const repairHook = this.onRunCrashResumed; + if (repairHook == null || attempt > CRASH_RESUME_REPAIR_MAX_ATTEMPTS) { + return; + } + if (pendingCrashResumeRepairTimers.has(input.runId)) { + return; + } + const timer = setTimeout(() => { + pendingCrashResumeRepairTimers.delete(input.runId); + void (async () => { + try { + await repairHook({ workspaceId: input.workspaceId, runId: input.runId }); + } catch (error) { + console.error("Workflow crash-resume provenance repair retry failed:", error); + this.scheduleCrashResumeRepairRetry(input, attempt + 1); + } + })(); + }, this.crashResumeRepairRetryDelayMs); + unrefTimer(timer); + pendingCrashResumeRepairTimers.set(input.runId, timer); + } + private registerActiveRunnerAbortController( runId: string, workspaceId: string,