diff --git a/src/browser/utils/chatCommands.test.ts b/src/browser/utils/chatCommands.test.ts index ed157332b1..fcd4f8a303 100644 --- a/src/browser/utils/chatCommands.test.ts +++ b/src/browser/utils/chatCommands.test.ts @@ -1582,6 +1582,32 @@ describe("prepareCompactionMessage", () => { expect(metadata.parsed.followUpContent?.agentId).toBe("exec"); }); + test("compaction recovery keeps the persisted follow-up's restrictions when retry options lack them", () => { + // Retrying a failed compaction passes the already-persisted follow-up together with + // storage-derived send options, which never carry a caller toolPolicy. The preserved + // restrictions must survive that recomposition or the recovered follow-up resumes with + // unrestricted caller tools. + const recoveredFollowUp = { + text: "Keep building", + model: "openai:gpt-4o", + agentId: "code", + toolPolicy: [{ regex_match: "^bash$", action: "disable" as const }], + disableWorkspaceAgents: true, + }; + const { metadata } = prepareCompactionMessage({ + workspaceId: "ws-1", + followUpContent: recoveredFollowUp, + sendMessageOptions: { model: "anthropic:claude-sonnet-4-6", agentId: "exec" }, + }); + + expectCompactionMetadata(metadata); + expect(metadata.parsed.followUpContent?.toolPolicy).toEqual(recoveredFollowUp.toolPolicy); + expect(metadata.parsed.followUpContent?.disableWorkspaceAgents).toBe(true); + // Existing model/agentId still win over the retry-time options. + expect(metadata.parsed.followUpContent?.model).toBe("openai:gpt-4o"); + expect(metadata.parsed.followUpContent?.agentId).toBe("code"); + }); + test("does not create followUpContent when no text or images provided", () => { const sendMessageOptions = createBaseOptions(); const { metadata } = prepareCompactionMessage({ diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 9b16234a10..2fa770234c 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -63,6 +63,7 @@ type PreservedSendOptions = Pick< | "providerOptions" | "experiments" | "disableWorkspaceAgents" + | "toolPolicy" | "strictAgentResolution" | "allowAgentSetGoal" | "skipAiSettingsPersistence" @@ -73,21 +74,39 @@ type PreservedSendOptions = Pick< * Use this helper to avoid duplicating the field list when building CompactionFollowUpRequest. */ export function pickPreservedSendOptions(options: SendMessageOptions): PreservedSendOptions { + // Unset fields are OMITTED, not emitted as explicit undefined: compaction recovery spreads + // this pick over an already-persisted follow-up, and an undefined key would clobber the + // original preserved value (e.g. a restricted turn's toolPolicy) instead of leaving it. return { - thinkingLevel: options.thinkingLevel, - reasoningMode: options.reasoningMode, - additionalSystemInstructions: options.additionalSystemInstructions, - providerOptions: options.providerOptions, + ...(options.thinkingLevel !== undefined ? { thinkingLevel: options.thinkingLevel } : {}), + ...(options.reasoningMode !== undefined ? { reasoningMode: options.reasoningMode } : {}), + ...(options.additionalSystemInstructions !== undefined + ? { additionalSystemInstructions: options.additionalSystemInstructions } + : {}), + ...(options.providerOptions !== undefined ? { providerOptions: options.providerOptions } : {}), // Downgrade-compat (see withLegacyPtcExclusiveMirror): preserved options // can persist across restarts and build versions. - experiments: withLegacyPtcExclusiveMirror(options.experiments), - disableWorkspaceAgents: options.disableWorkspaceAgents, + ...(options.experiments !== undefined + ? { experiments: withLegacyPtcExclusiveMirror(options.experiments) } + : {}), + ...(options.disableWorkspaceAgents !== undefined + ? { disableWorkspaceAgents: options.disableWorkspaceAgents } + : {}), + // Security: a restricted turn (including a terminal-wake send restoring the caller's + // policy) that triggers on-send compaction must not redispatch its follow-up allow-all. + ...(options.toolPolicy !== undefined ? { toolPolicy: options.toolPolicy } : {}), // Delegated turns with explicit agent overrides must stay loud across the // compaction replay too — dropping this would let the follow-up silently // fall back to exec if the agent vanished in the meantime. - strictAgentResolution: options.strictAgentResolution, - allowAgentSetGoal: options.allowAgentSetGoal, - skipAiSettingsPersistence: options.skipAiSettingsPersistence, + ...(options.strictAgentResolution !== undefined + ? { strictAgentResolution: options.strictAgentResolution } + : {}), + ...(options.allowAgentSetGoal !== undefined + ? { allowAgentSetGoal: options.allowAgentSetGoal } + : {}), + ...(options.skipAiSettingsPersistence !== undefined + ? { skipAiSettingsPersistence: options.skipAiSettingsPersistence } + : {}), }; } diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index fc286a097c..a6b69a4d24 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -2,7 +2,7 @@ import { xai } from "@ai-sdk/xai"; import { type LanguageModel, type Tool } from "ai"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; -import type { ProvidersConfigMap } from "@/common/orpc/types"; +import type { ProvidersConfigMap, SendMessageOptions } from "@/common/orpc/types"; import { isGrokFrontierModel } from "@/common/types/thinking"; import type { BackgroundWorkAttentionPolicy } from "@/common/types/backgroundWorkAttention"; import { cloneToolPreservingDescriptors } from "@/common/utils/tools/cloneToolPreservingDescriptors"; @@ -184,6 +184,10 @@ export interface ToolConfiguration { workspaceSessionDir?: string; /** Workspace ID for tracking background processes and plan storage */ workspaceId?: string; + /** Resolved agent identity of the turn executing the tools (workflow wake provenance). */ + agentId?: string; + /** The turn's strict-agent pin, persisted with workflow run provenance so wakes re-pin the launch agent. */ + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; /** Pre-resolved mux-managed resource scope (global ~/.xum vs project root). */ xumScope?: XumToolScope; /** Memory service for the memory tool (present only when the memory experiment is enabled). */ diff --git a/src/common/utils/workflowRunMessages.ts b/src/common/utils/workflowRunMessages.ts index 9fc393e9f7..66a2b3b5fa 100644 --- a/src/common/utils/workflowRunMessages.ts +++ b/src/common/utils/workflowRunMessages.ts @@ -190,6 +190,41 @@ export function buildWorkflowResultContextMessage(input: { ].join("\n\n"); } +/** + * Recognize this run's result payload inside a coalesced terminal-attention prompt from the + * builder's own output format. The drain's synthetic user row can coalesce several runs into + * one message and carries no workflow-result metadata, so currentness checks must read the + * consumption evidence out of the text: each payload block is parsed back and matched on the + * exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted + * inside another run's report cannot count as consumption. + */ +export function textContainsWorkflowResultPayload(text: string, runId: string): boolean { + assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required"); + if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) { + return false; + } + const blockPattern = new RegExp( + `<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n`, + "g" + ); + for (const match of text.matchAll(blockPattern)) { + let payload: unknown; + try { + payload = JSON.parse(match[1] ?? ""); + } catch { + continue; + } + if (!isRecordValue(payload)) { + continue; + } + const workflow = payload.workflow; + if (isRecordValue(workflow) && workflow.runId === runId) { + return true; + } + } + return false; +} + export interface WorkflowRunCardInput { scriptPath?: string; scriptSource?: string; diff --git a/src/node/orpc/router.test.ts b/src/node/orpc/router.test.ts index 62db8bbac9..ccfc19616b 100644 --- a/src/node/orpc/router.test.ts +++ b/src/node/orpc/router.test.ts @@ -362,7 +362,11 @@ describe("router workflow routes", () => { getWorkflowContinuationSendOptions: mock(() => null), sendMessage: mock(async () => ({ success: true, data: undefined })), }, - taskService: {}, + // Nonterminal run status changes reset any stale terminal notification; the stub keeps + // that call observable without wiring a full TaskService. + taskService: { + resetWorkflowRunTerminalAttention: mock(async () => undefined), + }, experimentsService: { isExperimentEnabled: mock(() => options.enabled), }, @@ -951,6 +955,80 @@ export default function workflow() { return { reportMarkdown: "should not run" } expect(result.result).toBeNull(); await waitForRouterWorkflowStatus(client, "workspace-1", result.runId, "completed"); }); + + test("crash-resumed background runs enqueue terminal attention on settle", async () => { + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); + await runStore.createRun({ + id: "wfr_crash_wake", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + // Orphaned by a crash: durable status says running, but no live runner. + await runStore.appendStatus("wfr_crash_wake", "running", "2026-05-29T00:00:01.000Z"); + + const enqueueWorkflowRunTerminalAttention = mock(async () => undefined); + const context = createContext({ enabled: true }); + (context as unknown as Record).taskService = { + enqueueWorkflowRunTerminalAttention, + resetWorkflowRunTerminalAttention: mock(async () => undefined), + }; + ( + context.workspaceService as unknown as Record + ).repairWorkflowRunReferenceBoundary = mock(async () => undefined); + const client = createRouterClient(router(), { context }); + + // A read path triggers crash recovery; the resumed run's settle must land in the + // terminal-attention outbox instead of waiting for the next restart's sweep. + await client.workflows.listRuns({ workspaceId: "workspace-1" }); + await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_crash_wake", "completed"); + const deadline = Date.now() + 5_000; + while (enqueueWorkflowRunTerminalAttention.mock.calls.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(enqueueWorkflowRunTerminalAttention).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_crash_wake", + status: "completed", + }); + }); + + test("router-managed resume resets a stale terminal notification before restart", async () => { + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); + await runStore.createRun({ + id: "wfr_resume_reset", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + + const context = createContext({ enabled: true }); + const resetWorkflowRunTerminalAttention = ( + context.taskService as unknown as { + resetWorkflowRunTerminalAttention: ReturnType; + } + ).resetWorkflowRunTerminalAttention; + const client = createRouterClient(router(), { context }); + + await client.workflows.interrupt({ workspaceId: "workspace-1", runId: "wfr_resume_reset" }); + resetWorkflowRunTerminalAttention.mockClear(); + + // The prior run's notification survives under the stable workflow_run: id as + // delivered/superseded; without a reset on restart, enqueueIfAbsent preserves that + // record and the resumed run's terminal wake is silently dropped. + await client.workflows.resume({ workspaceId: "workspace-1", runId: "wfr_resume_reset" }); + expect(resetWorkflowRunTerminalAttention).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_reset", + }); + await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_resume_reset", "completed"); + }); }); describe("router config.saveConfig", () => { diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index c425ea2be0..02feeed633 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -116,6 +116,7 @@ import * as path from "node:path"; import type { DevToolsEvent } from "@/common/types/devtools"; import type { WorkflowRunStreamEvent } from "@/common/types/workflow"; +import { isTerminalWorkflowRunStatus } from "@/common/types/workflow"; import type { WorkflowRunLivenessEntry } from "@/common/orpc/schemas/api"; import type { MuxMessage } from "@/common/types/message"; import { coerceThinkingLevel } from "@/common/types/thinking"; @@ -583,10 +584,39 @@ export async function resolveWorkflowContext( includeAgentPlugins, skillStorageContext, }), - onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), - ...(options.onBackgroundRunTerminal != null - ? { onBackgroundRunTerminal: options.onBackgroundRunTerminal } - : {}), + onRunStatusChanged: async (event) => { + // Router-managed restarts (resume / retry / crash recovery) must clear a prior + // delivered or superseded notification when the run leaves terminal state, or + // enqueueIfAbsent would preserve the stale record and silently drop the resumed + // run's next terminal wake (mirrors the AIService-owned service). + if (!isTerminalWorkflowRunStatus(event.status)) { + await context.taskService.resetWorkflowRunTerminalAttention({ + ownerWorkspaceId: event.workspaceId, + runId: event.runId, + }); + } + await context.workspaceService.emitWorkflowRunActivity(event); + }, + onRunCrashResumed: (event) => + context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId), + // Read paths (listRuns / stream subscribe) create services purely to observe runs, but + // crash recovery can resume an orphaned background run on them: without a terminal + // callback the settled run would enqueue no terminal attention until the next restart's + // sweep. Default to the standard outbox enqueue (idempotent via enqueueIfAbsent); + // explicit callbacks (slash-command continuations, retry) keep their custom behavior. + onBackgroundRunTerminal: + options.onBackgroundRunTerminal ?? + (async (event) => { + // Nested runs surface through their parent workflow, not their own wake. + if (event.run.parentWorkflow != null) { + return; + } + await context.taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: workspaceId, + runId: event.runId, + status: event.status, + }); + }), getCurrentProjectTrusted: resolveWorkflowProjectTrusted, runnerId: `workflow-runner:${workspaceId}`, }), diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 3ef773f339..ca2a53e828 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -95,10 +95,12 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { getThreshold: mock(() => 0.85), } as unknown as CompactionMonitor; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; const result = await session.sendMessage("please inspect @foo.ts", { model: "openai:gpt-4o", agentId: "exec", disableWorkspaceAgents: true, + toolPolicy: restrictedPolicy, }); expect(result.success).toBe(true); @@ -120,6 +122,15 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { ); expect(persistedCompactionMessage).toBeDefined(); expect(persistedCompactionMessage?.metadata?.disableWorkspaceAgents).toBe(true); + // The durable follow-up must keep the caller's restrictions: the redispatched turn + // reconstructs its options from this persisted request, and dropping the policy there + // would resume the conversation allow-all after compaction. + const followUpContent = + persistedCompactionMessage?.metadata?.muxMetadata?.type === "compaction-request" + ? persistedCompactionMessage.metadata.muxMetadata.parsed.followUpContent + : undefined; + expect(followUpContent?.toolPolicy).toEqual(restrictedPolicy); + expect(followUpContent?.disableWorkspaceAgents).toBe(true); const emittedSnapshot = events.some( (message) => diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4422a49a05..0b560f406e 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -44,6 +44,7 @@ import { SendMessageOptionsSchema, SkillNameSchema, } from "@/common/orpc/schemas"; +import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { buildStreamErrorEventData, @@ -7200,6 +7201,23 @@ export class AgentSession { experiments: aliasLegacyPtcExclusive(followUp.experiments), allowAgentSetGoal: followUp.allowAgentSetGoal, disableWorkspaceAgents: followUp.disableWorkspaceAgents, + // Same raw JSON boundary: a persisted follow-up may carry a malformed toolPolicy, and + // restoring it unvalidated would throw during resolution. Invalid values are dropped + // like any corrupt persisted policy (self-healing doctrine); a restricted turn's + // follow-up must otherwise keep its policy instead of redispatching allow-all. + ...(() => { + if (followUp.toolPolicy == null) { + return {}; + } + const parsed = ToolPolicySchema.safeParse(followUp.toolPolicy); + if (!parsed.success) { + log.warn("Ignoring malformed persisted toolPolicy on compaction follow-up", { + workspaceId: this.workspaceId, + }); + return {}; + } + return { toolPolicy: parsed.data }; + })(), // Explicit-agent turns stay loud on the resumed turn too: the requested agent // may have been removed/hidden/disabled while compaction ran. strictAgentResolution: followUp.strictAgentResolution, diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 88a6436139..7ecab498c7 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -7,6 +7,7 @@ import { describe, expect, test } from "bun:test"; import { readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, + repairAgentWorkflowRunReferenceBoundary, } from "@/node/services/agentWorkflowRunReferences"; describe("agent workflow run references", () => { @@ -31,4 +32,347 @@ describe("agent workflow run references", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); + + test("collapses persisted duplicate entries to the newest sane timestamp", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Corrupted files can carry duplicates in either order; order-sensitive consumers must + // never observe a stale duplicate ahead of a legitimate re-record. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_dup", createdAtMs: 1_000 }, + { runId: "wfr_dup", createdAtMs: 2_000 }, + { runId: "wfr_dup_reversed", createdAtMs: 2_000 }, + { runId: "wfr_dup_reversed", createdAtMs: 1_000 }, + ], + }) + ); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(2); + expect(references).toContainEqual({ runId: "wfr_dup", createdAtMs: 2_000 }); + expect(references).toContainEqual({ runId: "wfr_dup_reversed", createdAtMs: 2_000 }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("roundtrips the initiating agent and drops invalid persisted shapes", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_agent", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + let references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toContainEqual({ + runId: "wfr_agent", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + + // Identity is advisory: a malformed persisted agentId drops the field, not the entry, + // so the run keeps its wake and identity falls back to the history walk. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_agent_number", createdAtMs: 1_000, agentId: 7 }, + { runId: "wfr_agent_empty", createdAtMs: 1_000, agentId: "" }, + // Non-empty but schema-invalid: stream resolution would normalize it to exec, + // silently swapping a restricted agent's wake onto exec's tool surface. + { runId: "wfr_agent_malformed", createdAtMs: 1_000, agentId: "bad id" }, + // Invalid pin shapes degrade to the legacy walk fallback (field dropped); a + // persisted false means verified-unpinned (null), like absence at record time. + { + runId: "wfr_pin_invalid", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: 42 }, + }, + { + runId: "wfr_pin_false", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: false, + }, + ], + }) + ); + references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toContainEqual({ runId: "wfr_agent_number", createdAtMs: 1_000 }); + expect(references).toContainEqual({ runId: "wfr_agent_empty", createdAtMs: 1_000 }); + expect(references).toContainEqual({ runId: "wfr_agent_malformed", createdAtMs: 1_000 }); + expect(references).toContainEqual({ + runId: "wfr_pin_invalid", + createdAtMs: 1_000, + agentId: "plan", + }); + expect(references).toContainEqual({ + runId: "wfr_pin_false", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: null, + }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("drops persisted future-dated references and repairs them on the next record", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const futureMs = Date.now() + 86_400_000; + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_corrupt_future", createdAtMs: futureMs }, + { runId: "wfr_sane", createdAtMs: 1_000 }, + ], + }) + ); + + // A per-read clamp would re-evaluate to "now" on every read and outrank every later + // user/reset boundary; the corrupted entry must be dropped instead. + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([ + { runId: "wfr_sane", createdAtMs: 1_000 }, + ]); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_corrupt_future", + createdAtMs: 2_000, + }); + const repaired = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(repaired).toContainEqual({ runId: "wfr_corrupt_future", createdAtMs: 2_000 }); + const raw = await fs.readFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + "utf-8" + ); + expect(raw).not.toContain(String(futureMs)); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("keeps references within the backward-clock skew tolerance", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A backward clock correction makes a legitimately recorded reference look slightly + // future-dated; dropping it would strand the run's terminal wake. + const slightlyFutureMs = Date.now() + 5 * 60_000; + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ references: [{ runId: "wfr_clock_skew", createdAtMs: slightlyFutureMs }] }) + ); + + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([ + { runId: "wfr_clock_skew", createdAtMs: slightlyFutureMs }, + ]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("rejects entries with a present-but-invalid boundary snapshot", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // "" and non-string values are corruption, not legacy records: migrating them into the + // wall-clock fallback could outrank a newer boundary during tolerated clock skew. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_empty_boundary", createdAtMs: 1_000, afterBoundaryMessageId: "" }, + { runId: "wfr_numeric_boundary", createdAtMs: 1_000, afterBoundaryMessageId: 42 }, + { runId: "wfr_valid_boundary", createdAtMs: 1_000, afterBoundaryMessageId: "row-1" }, + { runId: "wfr_null_boundary", createdAtMs: 1_000, afterBoundaryMessageId: null }, + ], + }) + ); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(new Set(references.map((reference) => reference.runId))).toEqual( + new Set(["wfr_valid_boundary", "wfr_null_boundary"]) + ); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("propagates non-ENOENT read failures instead of flattening them to empty", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A directory at the file path fails reads with EISDIR. Callers deciding wake delivery + // must observe the failure rather than "no references". + await fs.mkdir(path.join(workspaceSessionDir, "agent-workflow-runs.json")); + let readError: unknown; + try { + await readAgentWorkflowRunReferences(workspaceSessionDir); + } catch (error: unknown) { + readError = error; + } + expect(String(readError)).toContain("EISDIR"); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("record propagates a sidecar read failure instead of clobbering existing references", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_existing", + createdAtMs: 1_000, + }); + const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); + // Unreadable file, writable directory: the atomic rewrite could replace contents it + // never saw, destroying every other run's only durable provenance. + await fs.chmod(filePath, 0o000); + let recordError: unknown; + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_new", + createdAtMs: 2_000, + }); + } catch (error: unknown) { + recordError = error; + } finally { + await fs.chmod(filePath, 0o600); + } + expect(String(recordError)).toContain("EACCES"); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references.map((reference) => reference.runId)).toEqual(["wfr_existing"]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("self-heals unparseable file contents to empty", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Unlike a failed read, corrupted contents cannot be repaired by rereading. + await fs.writeFile(path.join(workspaceSessionDir, "agent-workflow-runs.json"), "{not json"); + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("clamps future-dated createdAtMs to the current time", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_future"; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs: Date.now() + 86_400_000, + }); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(1); + expect(references[0]?.createdAtMs).toBeLessThanOrEqual(Date.now()); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("keeps the newest createdAtMs across re-records", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_re_recorded"; + await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs: 2_000 }); + await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs: 1_000 }); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toEqual([{ runId, createdAtMs: 2_000 }]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("boundary repair is a compare-and-set on a surviving boundaryless reference", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_repairable"; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_bystander", + createdAtMs: 1_100, + afterBoundaryMessageId: "row-1", + }); + + // Repairs in place, preserving the rest of the entry and its neighbors. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId, + afterBoundaryMessageId: null, + }) + ).toBe(true); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references.find((reference) => reference.runId === runId)).toEqual({ + runId, + createdAtMs: 1_000, + afterBoundaryMessageId: null, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(references.find((reference) => reference.runId === "wfr_bystander")).toEqual({ + runId: "wfr_bystander", + createdAtMs: 1_100, + afterBoundaryMessageId: "row-1", + }); + + // A reference that already carries a boundary (here the one just repaired) is never + // overwritten: a concurrent explicit re-record must win over a stale repair. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId, + afterBoundaryMessageId: "stale-row", + }) + ).toBe(false); + const unchanged = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(unchanged.find((reference) => reference.runId === runId)?.afterBoundaryMessageId).toBe( + null + ); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("boundary repair refuses to recreate a cleared sidecar", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A full-history clear deleted the sidecar between the repair's reads and its write: + // the stale repair must not resurrect the retired reference as verified-empty current. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId: "wfr_cleared", + afterBoundaryMessageId: null, + }) + ).toBe(false); + expect(await fs.readdir(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index b202f4a83d..01d5cba36a 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -3,16 +3,55 @@ import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; +import { SendMessageOptionsSchema } from "@/common/orpc/schemas/stream"; +import type { SendMessageOptions } from "@/common/orpc/types"; +import { AgentIdSchema } from "@/common/schemas/ids"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; +/** A meaningful strict-agent pin: `false` and absence both mean "not pinned" and persist as null. */ +export type AgentWorkflowRunStrictPin = Exclude< + NonNullable, + false +>; + export interface AgentWorkflowRunReference { runId: string; createdAtMs: number; + /** + * Message ID of the newest invocation-decision row (manual user/reset supersession, consumed + * terminal result for this run, or direct invocation part) at record time; null when history + * had none. Row identity, not wall clock: currentness compares this against the row the + * history walk stops at, so a backward clock correction cannot reorder the comparison. + * Absent on legacy entries, which fail safe to not-current. + */ + afterBoundaryMessageId?: string | null; + /** + * Agent identity of the turn that launched/resumed the run. The terminal wake binds to this + * instead of the newest agent-bearing assistant row, which a later synthetic turn (e.g. a + * heartbeat) can own without superseding the run. Advisory: absent on legacy entries, which + * fall back to the history walk. + */ + agentId?: string; + /** + * The launch turn's strict-agent pin, paired with agentId: a wake that re-binds this agent + * must re-pin the launch turn's provenance, because the newest pin-bearing history row can + * belong to a different group's wake and a mismatched pin makes resolution reject every + * retry. null records a verified-unpinned launch; absent (legacy or invalid persisted + * shape) falls back to the history-walk pin. + */ + strictAgentResolution?: AgentWorkflowRunStrictPin | null; } +const StrictPinSchema = SendMessageOptionsSchema.shape.strictAgentResolution; + const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; +// Backward clock corrections (e.g. an NTP step after booting with a fast clock) can make a +// legitimately recorded reference look future-dated. Tolerate that bounded skew so the run's +// terminal wake is not dropped; only implausibly future values are treated as corruption. +const MAX_FUTURE_SKEW_MS = 60 * 60_000; + const referenceFileLocks = new MutexMap(); function referencesPath(workspaceSessionDir: string): string { @@ -29,7 +68,8 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { return []; } - const parsed: AgentWorkflowRunReference[] = []; + const parsedByRunId = new Map(); + const now = Date.now(); for (const reference of references) { if (reference == null || typeof reference !== "object") { continue; @@ -41,41 +81,149 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (typeof record.createdAtMs !== "number" || !Number.isFinite(record.createdAtMs)) { continue; } - parsed.push({ runId: record.runId, createdAtMs: record.createdAtMs }); + // Reject implausibly future-dated references (corruption) instead of clamping at read + // time: a per-read clamp re-evaluates to "now" on every read, so the entry would outrank + // every later user/reset boundary until wall time catches up. Values within + // MAX_FUTURE_SKEW_MS are kept as-is (backward clock correction, not corruption). Rejected + // entries are replaced with a sane timestamp by the next legitimate record. + if (record.createdAtMs > now + MAX_FUTURE_SKEW_MS) { + continue; + } + const hasBoundary = "afterBoundaryMessageId" in record; + const boundaryRaw = record.afterBoundaryMessageId; + // A present-but-invalid snapshot ("" or a non-string) is corruption, not a legacy record: + // demoting it to a boundaryless entry would misclassify a recorded boundary as unknowable + // provenance (parking its wake as indeterminate). Reject the entry; absence stays reserved + // for records that genuinely predate the field. + if ( + hasBoundary && + boundaryRaw !== null && + (typeof boundaryRaw !== "string" || boundaryRaw.length === 0) + ) { + continue; + } + const afterBoundaryMessageId = hasBoundary + ? typeof boundaryRaw === "string" + ? boundaryRaw + : null + : undefined; + // Identity is advisory (the wake falls back to the history walk), so an invalid shape + // drops only the field, not the entry. Schema-validate rather than accepting any string: + // stream resolution normalizes an unknown requested agent to exec, so a corrupt persisted + // ID would silently swap a restricted agent's wake onto exec's tool surface. + const agentIdParse = AgentIdSchema.safeParse(record.agentId); + const agentId = agentIdParse.success ? agentIdParse.data : undefined; + // The pin only means anything paired with a surviving identity; false and invalid shapes + // degrade differently (unpinned vs legacy walk fallback), matching the field doc above. + let strictAgentResolution: AgentWorkflowRunStrictPin | null | undefined; + if (agentId !== undefined && "strictAgentResolution" in record) { + const pinRaw = record.strictAgentResolution; + if (pinRaw === null || pinRaw === false) { + strictAgentResolution = null; + } else { + const pinParse = StrictPinSchema.safeParse(pinRaw); + strictAgentResolution = + pinParse.success && pinParse.data != null && pinParse.data !== false + ? pinParse.data + : undefined; + } + } + // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive + // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run + // superseded. The chosen record is kept wholesale, including its boundary snapshot. + const existing = parsedByRunId.get(record.runId); + if (existing == null || record.createdAtMs > existing.createdAtMs) { + parsedByRunId.set(record.runId, { + runId: record.runId, + createdAtMs: record.createdAtMs, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + ...(agentId !== undefined ? { agentId } : {}), + ...(strictAgentResolution !== undefined ? { strictAgentResolution } : {}), + }); + } } - return parsed; + return Array.from(parsedByRunId.values()); } export async function readAgentWorkflowRunReferences( workspaceSessionDir: string ): Promise { + let raw: string; try { - const raw = await fs.readFile(referencesPath(workspaceSessionDir), "utf-8"); - return parseReferences(JSON.parse(raw) as unknown); + raw = await fs.readFile(referencesPath(workspaceSessionDir), "utf-8"); } catch (error: unknown) { if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { return []; } + // For kernel-launched runs this file is the only durable invocation evidence, and callers + // deciding wake delivery must distinguish "no reference" from "cannot know right now": + // flattening a transient read failure into [] would let the terminal drain tombstone the + // run's wake. Corrupted contents below stay self-healing because rereading cannot repair + // them, while a failed read can succeed later. + throw error; + } + try { + return parseReferences(JSON.parse(raw) as unknown); + } catch { return []; } } +/** + * Retire every reference for this workspace. A full history clear removes all rows without + * appending a reset boundary, which makes a verified-empty (null) boundary snapshot recorded + * before the clear indistinguishable from one recorded after it; retiring the references with + * the transcript keeps pre-clear workflow results out of the fresh conversation. A post-clear + * workflow_resume re-records provenance. + */ +export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { + const filePath = referencesPath(workspaceSessionDir); + await referenceFileLocks.withLock(filePath, async () => { + await fs.rm(filePath, { force: true }); + }); +} + export async function recordAgentWorkflowRunReference(input: { workspaceSessionDir: string; runId: string; createdAtMs?: number; + afterBoundaryMessageId?: string | null; + agentId?: string; + strictAgentResolution?: AgentWorkflowRunStrictPin | null; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { + // A read failure propagates instead of being treated as empty: the atomic rewrite below + // would otherwise replace valid-but-momentarily-unreadable contents with only this run, + // destroying every other active run's sole durable provenance. The failed record is + // retryable (workflow_resume re-records), while parse corruption still self-heals to + // empty inside readAgentWorkflowRunReferences because rereading cannot repair it. const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); const byRunId = new Map(existing.map((reference) => [reference.runId, reference])); - const createdAtMs = input.createdAtMs ?? Date.now(); + // Clamp like parseReferences: never persist a future-dated timestamp. + const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); byRunId.set(input.runId, { runId: input.runId, - createdAtMs: previous ? Math.min(previous.createdAtMs, createdAtMs) : createdAtMs, + // Latest record wins: workflow_resume re-records the reference, and a resume issued after + // a manual user message must re-establish provenance for supersession-timestamp + // comparisons (listAgentReferencedWorkflowRunIds). + createdAtMs: previous ? Math.max(previous.createdAtMs, createdAtMs) : createdAtMs, + // The new record event defines currentness provenance wholesale; a caller without + // boundary knowledge produces a legacy-style entry that fails safe to not-current. + ...(input.afterBoundaryMessageId !== undefined + ? { afterBoundaryMessageId: input.afterBoundaryMessageId } + : {}), + ...(input.agentId != null && input.agentId.length > 0 + ? { + agentId: input.agentId, + ...(input.strictAgentResolution !== undefined + ? { strictAgentResolution: input.strictAgentResolution } + : {}), + } + : {}), }); await fs.mkdir(path.dirname(filePath), { recursive: true }); @@ -85,3 +233,37 @@ export async function recordAgentWorkflowRunReference(input: { ); }); } + +/** + * Compare-and-set boundary repair for a surviving boundaryless reference. The reference is + * re-validated under the sidecar file lock: a concurrent full-history clear deletes the file + * (retiring every reference), and an unconditional write would recreate it with a + * verified-empty boundary, resurrecting the retired pre-clear result as "current" in the + * freshly cleared conversation. A reference that concurrently gained a boundary (explicit + * workflow_resume re-record) is also left untouched. Returns false without writing when the + * reference is gone or already carries a boundary. + */ +export async function repairAgentWorkflowRunReferenceBoundary(input: { + workspaceSessionDir: string; + runId: string; + afterBoundaryMessageId: string | null; +}): Promise { + assert(input.runId.length > 0, "agent workflow reference repair requires runId"); + const filePath = referencesPath(input.workspaceSessionDir); + + return referenceFileLocks.withLock(filePath, async () => { + const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + const reference = existing.find((candidate) => candidate.runId === input.runId); + if (reference == null || reference.afterBoundaryMessageId !== undefined) { + return false; + } + const references = existing.map((candidate) => + candidate.runId === input.runId + ? { ...candidate, afterBoundaryMessageId: input.afterBoundaryMessageId } + : candidate + ); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await writeFileAtomic(filePath, JSON.stringify({ references }, null, 2)); + return true; + }); +} diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index d0d192fd51..58169e0749 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -2646,6 +2646,8 @@ export class AIService extends EventEmitter { planFilePath, ancestorPlanFilePaths, workspaceId, + agentId: effectiveAgentId, + strictAgentResolution, xumScope, timelineService: timelineExperimentEnabled ? this.timelineService : undefined, workspaceHeartbeatService: this.workspaceHeartbeatService, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index b4d4131a4d..c0d96c1a73 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -665,6 +665,14 @@ function createWorkspaceServiceMocks( mock((_workspaceId: string, _message: WorkspaceChatMessage) => undefined); const isWorkflowInvocationCurrent = overrides?.isWorkflowInvocationCurrent ?? mock(() => Promise.resolve(true)); + // Derived from the boolean mock so tests that override isWorkflowInvocationCurrent keep + // steering the drain's three-state check. + const getWorkflowInvocationCurrentness = mock( + async (workspaceId: string, runId: string) => + ((await isWorkflowInvocationCurrent(workspaceId, runId)) === true + ? "current" + : "not_current") as "current" | "not_current" | "indeterminate" + ); const countQueuedAgentPeerMessages = overrides?.countQueuedAgentPeerMessages ?? mock(() => 0); // Granted by default (no live user activity): interrupt_active tests exercise the // interruption/archive flow; the hold's own refusal logic lives in workspaceService.test.ts. @@ -728,6 +736,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, + getWorkflowInvocationCurrentness, countQueuedAgentPeerMessages, } as unknown as WorkspaceService, create, @@ -6239,6 +6248,1189 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("terminal workflow wake-up defers when history is unreadable", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_defer"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // History unreadable at drain time: currentness is indeterminate, so the notification must + // stay pending for a later drain instead of being tombstoned as superseded. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + + test("deferred terminal wake-up retries on the bounded timer", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_defer_retry"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // The first drain sees a transient storage fault that clears before the retry fires. An + // already-idle owner produces no other drain trigger, so only the bounded retry delivers. + let currentnessCalls = 0; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => { + currentnessCalls += 1; + return Promise.resolve(currentnessCalls === 1 ? "indeterminate" : "current"); + }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + ( + taskService as unknown as { terminalAttentionDeferRetryDelayMs: number } + ).terminalAttentionDeferRetryDelayMs = 10; + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + + // Real timers: poll until the armed retry fires and the follow-up drain delivers. + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && sendMessage.mock.calls.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 10)); + await flushTerminalAttentionDrains(taskService); + } + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("failed terminal attention enqueue is retained and retried on the bounded timer", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_enqueue_retry"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + ( + taskService as unknown as { terminalAttentionDeferRetryDelayMs: number } + ).terminalAttentionDeferRetryDelayMs = 10; + + // The workflow terminal callback driving this enqueue is single-attempt, so when the + // first outbox write fails only the retained in-process retry can persist the wake. + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( + new Error("EIO: outbox write failed") + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + + // Real timers: poll until the armed retry re-enqueues and the follow-up drain delivers. + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && sendMessage.mock.calls.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 10)); + await flushTerminalAttentionDrains(taskService); + } + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("reset drops a retained terminal attention enqueue", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_enqueue_reset"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( + new Error("EIO: outbox write failed") + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + // The run was reset (e.g. resumed) before the retry fired; the retained stale wake + // must be dropped instead of resurrected by the retry. + await taskService.resetWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId }); + + // Drive the armed retry directly so the outcome is deterministic under real timers. + await ( + taskService as unknown as { + retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise; + } + ).retryRetainedWorkflowTerminalEnqueues(parentId); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("a failed reset is completed by the next terminal enqueue", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_reset_retained"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + // First settle delivers normally, leaving a delivered record under the stable id. + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + + // The restart's reset fails transiently; callers swallow the rejection, so only the + // retained pending reset can stop the delivered record from absorbing the next enqueue. + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "delete").mockRejectedValueOnce(new Error("EIO: outbox delete failed")); + await taskService.resetWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId }); + + // The resumed run settles again: the enqueue completes the reset and delivers fresh. + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("workflow wake restriction recovery stops at a context reset boundary", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_policy_reset_boundary"; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // The pre-reset manual row disabled bash, but the context reset discarded that + // conversation. The workflow launched from a post-reset synthetic turn (heartbeat), so + // its wake must use fresh defaults instead of resurrecting the discarded restriction. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("reset-boundary", "assistant", "Context reset", { + timestamp: 2_000, + contextBoundaryKind: "reset", + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat-launch", "user", "[heartbeat] launch the workflow", { + timestamp: 3_000, + synthetic: true, + }) + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const options = sendMessage.mock.calls[0]?.[2] as { + toolPolicy?: unknown; + disableWorkspaceAgents?: unknown; + }; + expect(options.toolPolicy).toBeUndefined(); + expect(options.disableWorkspaceAgents).toBeUndefined(); + }); + + test("workflow wakes restore the caller tool policy from the newest manual row", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_policy_restore"); + await createRun("wfr_policy_lifted"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // The launch turn disabled bash; a later synthetic row (an earlier wake) defines no + // policy and must be skipped. Omitting the policy on the wake would let workflow output + // regain the disabled tool at a time the workflow chooses. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("earlier-wake", "user", "results delivered", { + timestamp: 1_100, + synthetic: true, + }) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_policy_restore", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + toolPolicy: restrictedPolicy, + }); + + // A newer manual row without a policy means the caller lifted it: no restoration. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-unrestricted", "user", "carry on", { timestamp: 1_200 }) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_policy_lifted", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + const liftedOptions = sendMessage.mock.calls[1]?.[2] as { toolPolicy?: unknown }; + expect(liftedOptions.toolPolicy).toBeUndefined(); + }); + + test("wake restoration walks a long agent-less tail: caller policy, agent identity, disable flag", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runId = "wfr_policy_long_tail"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "exec", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + }, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("agent-turn", "assistant", "on it", { timestamp: 1_000, agentId: "plan" }) + ); + // A tail longer than any bounded history read: the launch turn's restrictions must still + // be found, not silently lifted once enough rows accumulate after the manual turn. + for (let i = 0; i < 60; i++) { + await historyService.appendToHistory( + parentId, + createMuxMessage(`assistant-${i}`, "assistant", `progress ${i}`, { timestamp: 1_001 + i }) + ); + } + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }); + }); + + test("workflow wakes bind to the initiating agent, not a later synthetic turn's agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_initiating_agent"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("launch-turn", "assistant", "starting", { + timestamp: 1_001, + agentId: "plan", + }) + ); + // A heartbeat is synthetic, not a manual supersession boundary: the run stays current, but + // its agent-bearing assistant row is now the newest one in history. The wake must use the + // launch turn's agent from the sidecar, not the heartbeat's. + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat", "user", "heartbeat", { timestamp: 1_002, synthetic: true }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat-turn", "assistant", "idle check", { + timestamp: 1_003, + agentId: "exec", + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "plan", + }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + }); + + test("coalesced workflow wakes split by initiating agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_split_plan"); + await createRun("wfr_split_exec"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run both audits", { timestamp: 1_000 }) + ); + // Two current runs from different initiating agents: one coalesced wake would hand the + // older run's (attacker-influenced) output to the newer agent's tool grants. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_split_exec", + createdAtMs: 1_000, + agentId: "exec", + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_split_plan", + createdAtMs: 2_000, + agentId: "plan", + }); + + // Seed the store directly so ONE drain observes both pending notifications; per-enqueue + // drains would deliver them separately without exercising the coalescing path. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_split_plan", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_split_exec", + }); + await drain(parentId); + + // The newest launch's group delivers first, alone, under its own agent. + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wfr_split_plan"); + expect(firstPrompt).not.toContain("wfr_split_exec"); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + + // The deferred group delivers on a later drain under its own agent. + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain("wfr_split_exec"); + expect(secondPrompt).not.toContain("wfr_split_plan"); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("coalesced workflow wakes split by strict pin within one agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_pin_split_pinned"); + await createRun("wfr_pin_split_unpinned"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run both audits", { timestamp: 1_000 }) + ); + // Same agentId, different launch pins (an agent definition replaced between synthetic + // launches): one coalesced wake would process the pinned run's output under the newer + // verified-unpinned launch. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_split_pinned", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_split_unpinned", + createdAtMs: 2_000, + agentId: "plan", + strictAgentResolution: null, + }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_pin_split_pinned", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_pin_split_unpinned", + }); + await drain(parentId); + + // The newest launch delivers first, alone, without the other launch's pin. + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wfr_pin_split_unpinned"); + expect(firstPrompt).not.toContain("wfr_pin_split_pinned"); + const firstOptions = sendMessage.mock.calls[0]?.[2] as Record; + expect(firstOptions.agentId).toBe("plan"); + expect(firstOptions.strictAgentResolution).toBeUndefined(); + + // The pinned launch delivers on the retry drain under its own recorded pin. + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain("wfr_pin_split_pinned"); + expect(secondPrompt).not.toContain("wfr_pin_split_unpinned"); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("mixed drains keep workspace-turn attention off the workflow's agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_mixed"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("agent-turn", "assistant", "on it", { timestamp: 1_001, agentId: "plan" }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "exec", + }); + + // A workspace-turn result resumes under the conversation's own identity; sharing its wake + // with an agent-bound workflow group would process it under the workflow's agent instead. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_mixed_handle", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + }); + await drain(parentId); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wst_mixed_handle"); + expect(firstPrompt).not.toContain(runId); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain(runId); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("wake keeps a synthetic launch row's strict pin without lifting the manual policy", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runId = "wfr_synthetic_pin"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + }) + ); + // The kernel workflow launched from a pinned synthetic turn (preserved heartbeat or + // compaction follow-up): its pin must ride the wake without lifting the manual policy. + await historyService.appendToHistory( + parentId, + createMuxMessage("synthetic-launch", "user", "heartbeat", { + timestamp: 1_100, + synthetic: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }, + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "plan", + }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + toolPolicy: restrictedPolicy, + strictAgentResolution: { expectedScope: "built-in" }, + }); + }); + + test("transient run-store read failures defer the wake instead of tombstoning it", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + const terminalAttentionStore = new TerminalAttentionStore(config); + + // run.json exists but is unreadable (EISDIR): potentially transient, so the wake must + // stay pending for a later drain instead of being durably tombstoned. + const unreadableRunId = "wfr_unreadable"; + await fsPromises.mkdir( + path.join(config.getSessionDir(parentId), "workflows", unreadableRunId, "run.json"), + { recursive: true } + ); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: unreadableRunId, + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + + // A definitively missing run (ENOENT) is still tombstoned, not deferred forever. + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_missing", + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.get(parentId, "workflow_run:wfr_missing")).toMatchObject({ + status: "superseded", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + + test("wake defers when the launch-identity read fails after currentness succeeds", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_identity_unreadable"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Currentness succeeds without the sidecar (e.g. a direct invocation row)... + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + const terminalAttentionStore = new TerminalAttentionStore(config); + + // ...but the launch-identity read fails transiently (EISDIR). Delivering without the + // recorded identity would bind the wake to the newest agent-bearing history row, so the + // wake must stay pending for the retry drain. + await fsPromises.mkdir(path.join(config.getSessionDir(parentId), "agent-workflow-runs.json"), { + recursive: true, + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + + test("wake re-pins the selected group's recorded launch pin, not the newest row's", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_pin_unpinned"); + await createRun("wfr_pin_recorded"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audits", { timestamp: 1_000 }) + ); + // The newest pin-bearing row belongs to a DIFFERENT group's wake: pinning its provenance + // onto this group's agentId would make resolution reject the wake on every retry. + await historyService.appendToHistory( + parentId, + createMuxMessage("other-group-wake", "user", "earlier group results", { + timestamp: 1_100, + synthetic: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "plan", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + }, + }) + ); + + // A verified-unpinned launch (null) must suppress the walk pin entirely. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_unpinned", + agentId: "exec", + strictAgentResolution: null, + }); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_pin_unpinned", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const unpinnedOptions = sendMessage.mock.calls[0]?.[2] as { + agentId?: string; + strictAgentResolution?: unknown; + }; + expect(unpinnedOptions.agentId).toBe("exec"); + expect(unpinnedOptions.strictAgentResolution).toBeUndefined(); + + // A recorded launch pin overrides the walk pin exactly. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_recorded", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_pin_recorded", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + }); + + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_policy_corrupt"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // Persisted metadata is untrusted disk state: a corrupt toolPolicy shape must be dropped + // (not copied into the send, where it would throw during resolution and permanently block + // the wake), while the intact disable flag on the same row still applies. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-corrupt", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: { bogus: true }, + disableWorkspaceAgents: true, + } as unknown as Parameters[3]) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const options = sendMessage.mock.calls[0]?.[2] as { + toolPolicy?: unknown; + disableWorkspaceAgents?: unknown; + }; + expect(options.toolPolicy).toBeUndefined(); + expect(options.disableWorkspaceAgents).toBe(true); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 7e43d21fde..523c44fa6b 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -118,6 +118,7 @@ import { import type { ProjectRef, WorkspaceMetadata } from "@/common/types/workspace"; import { getRuntimeType } from "@/node/runtime/initHook"; import { AgentIdSchema } from "@/common/orpc/schemas"; +import { SendMessageOptionsSchema, ToolPolicySchema } from "@/common/orpc/schemas/stream"; import type { AgentDefinitionScope } from "@/common/types/agentDefinition"; import { normalizeAgentId, @@ -188,6 +189,7 @@ import { isNonRetryableStreamError } from "@/common/utils/messages/retryEligibil import type { SendMessageError, StreamErrorType } from "@/common/types/errors"; import { hasCompletedAgentReport } from "@/common/utils/agentTaskCompletion"; import { isWorkspaceArchived } from "@/common/utils/archive"; +import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; import { DEFAULT_CODER_ARCHIVE_BEHAVIOR } from "@/common/config/coderArchiveBehavior"; import { isSSHRuntime, isWorktreeRuntime } from "@/common/types/runtime"; @@ -208,6 +210,7 @@ import { type TerminalAttentionOutcome, } from "@/node/services/terminalAttentionStore"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import type { AgentWorkflowRunStrictPin } from "@/node/services/agentWorkflowRunReferences"; import { isWorkflowRunTaskId } from "@/node/services/tools/taskId"; import { normalizeWorkflowAgentReportPayloadForHostSchema } from "@/common/utils/tools/workflowReportPayload"; import { @@ -862,6 +865,11 @@ function isWorkspaceBusyIdleOnlySend(error: unknown): boolean { const REMOVED_AGENT_TASKS_DIR = "removed-agent-tasks"; const COMPLETED_REPORT_CACHE_MAX_ENTRIES = 128; +// Retry cadence for terminal-attention drains deferred by indeterminate workflow currentness +// (unreadable history/sidecar). There is no deterministic "storage recovered" signal, so a +// bounded timer is the re-trigger; each retry that defers again arms the next one. +const TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS = 30_000; + /** Maximum consecutive auto-resumes before stopping. Prevents infinite loops when descendants are stuck. */ // Task-recovery paths must stay deterministic and editing-capable even when // workspace/default agent preferences evolve (e.g., auto router defaults). @@ -1044,6 +1052,22 @@ interface ParentAutoResumeHint { agentId?: string; } +/** Launch identity recorded with a workflow run reference; see AgentWorkflowRunReference. */ +interface WorkflowWakeInitiatingAgent { + agentId: string; + createdAtMs: number; + strictAgentResolution?: AgentWorkflowRunStrictPin | null; +} + +// Coalescing key for terminal workflow wakes: the pin is part of the launch identity, so an +// agentId alone must not merge a pinned launch with an unpinned (or differently pinned) one. +// undefined (legacy walk fallback), null (verified unpinned), and each concrete pin are +// distinct groups; over-splitting structurally equal pins is safe, merging them is not. +function workflowWakeGroupKey(agent: WorkflowWakeInitiatingAgent): string { + const pin = agent.strictAgentResolution; + return `${agent.agentId}\u0000${pin === undefined ? "walk" : JSON.stringify(pin)}`; +} + function isTypedWorkspaceEvent(value: unknown, type: string): boolean { return ( typeof value === "object" && @@ -1538,6 +1562,32 @@ export class TaskService { // tests and shutdown can await them; drains are idempotent and re-triggered on owner idle events. private readonly pendingTerminalAttentionDrainsByOwner = new Map>(); private readonly pendingTerminalAttentionDrains = new Set>(); + // One armed defer-retry timer per owner (see scheduleTerminalAttentionDeferRetry). The delay + // is a field, not a constant, so tests can shrink it without waiting out the real backoff. + private readonly terminalAttentionDeferRetryTimers = new Map< + string, + ReturnType + >(); + private terminalAttentionDeferRetryDelayMs = TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS; + // Workflow terminal callbacks are single-attempt (WorkflowService swallows callback + // rejections), so a transient outbox write failure would otherwise silence the wake for + // the life of the process. Retain failed enqueue params (owner -> runId -> status) and + // retry on the defer-retry cadence; reset drops the entry so a resumed run cannot + // resurrect its stale terminal wake. + private readonly retainedWorkflowTerminalEnqueues = new Map< + string, + Map + >(); + private readonly workflowTerminalEnqueueRetryTimers = new Map< + string, + ReturnType + >(); + // A reset whose store delete fails is retained here (owner -> runIds) instead of being + // swallowed: the stale delivered/superseded record only matters when the run settles + // again, so the next terminal enqueue completes the reset (delete before enqueue) and a + // failure there flows into the retained-enqueue retry above. No timer needed: with no + // later terminal transition the stale record is inert. + private readonly pendingWorkflowNotificationResets = new Map>(); private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); // Tracks workspaces currently blocked in a foreground wait (e.g. a task tool call awaiting @@ -1690,7 +1740,13 @@ export class TaskService { } const runIds = new Set(); - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + let references: Awaited> = []; + try { + references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error: unknown) { + // Rediscovery is non-destructive and re-runs on the next listing; skip this pass. + log.warn("Failed to read agent workflow run references", { workspaceId, error }); + } for (const reference of references) { // If the latest user/reset supersession has no durable timestamp, fail safe: only trust // workflow provenance re-established by current/post-supersession assistant output below. @@ -2289,26 +2345,30 @@ export class TaskService { // Compaction is internal bookkeeping, not an identity for resuming user work. let agentId = hint?.agentId === "compact" ? undefined : hint?.agentId; - // Durable history preserves the parent identity across process restarts. + // Durable history preserves the parent identity across process restarts. The walk is + // unbounded: synthetic rows without an agent identity (drain-appended sub-agent reports, + // heartbeat scaffolding) can push the newest agent-bearing assistant row past any fixed + // tail, and a truncated read would silently recompose terminal-wake sends from the exec + // fallback, lifting a restricted agent's tool policy. if (!agentId) { - try { - const historyResult = await this.historyService.getLastMessages(parentWorkspaceId, 20); - if (historyResult.success) { - for (let i = historyResult.data.length - 1; i >= 0; i--) { - const msg = historyResult.data[i]; - if ( - msg?.role === "assistant" && - msg.metadata?.agentId && - msg.metadata.agentId !== "compact" - ) { - agentId = msg.metadata.agentId; - break; - } + const found: { agentId?: string } = {}; + await this.historyService.iterateFullHistory(parentWorkspaceId, "backward", (messages) => { + for (const msg of messages) { + if ( + msg.role === "assistant" && + typeof msg.metadata?.agentId === "string" && + msg.metadata.agentId.length > 0 && + msg.metadata.agentId !== "compact" + ) { + found.agentId = msg.metadata.agentId; + return false; } } - } catch { - // Best-effort; fall through to defaults - } + return undefined; + }); + // A failed read falls through to defaults (best-effort); the terminal drain separately + // fails closed on unreadable history via resolveTerminalWakeCallerSendRestrictions. + agentId = found.agentId; } // 3) Default @@ -7707,12 +7767,65 @@ export class TaskService { if (!isTerminalWorkflowRunStatus(params.status)) { return; } - await this.enqueueTerminalAttention({ - ownerWorkspaceId: params.ownerWorkspaceId, - sourceKind: "workflow_run", - terminalOutcome: terminalAttentionOutcome(params.status), - sourceId: params.runId, - }); + try { + const pendingResets = this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId); + if (pendingResets?.has(params.runId)) { + // Marker cleared only after the delete succeeds so a failure retries the full + // reset-then-enqueue sequence instead of preserving the stale record. + await this.terminalAttentionStore.delete( + params.ownerWorkspaceId, + TerminalAttentionStore.notificationId("workflow_run", params.runId) + ); + pendingResets.delete(params.runId); + } + await this.enqueueTerminalAttention({ + ownerWorkspaceId: params.ownerWorkspaceId, + sourceKind: "workflow_run", + terminalOutcome: terminalAttentionOutcome(params.status), + sourceId: params.runId, + }); + } catch (error) { + log.error("Workflow terminal attention enqueue failed; retrying on bounded timer", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + error, + }); + this.retainWorkflowTerminalEnqueue(params); + } + } + + private retainWorkflowTerminalEnqueue(params: { + ownerWorkspaceId: string; + runId: string; + status: WorkflowRunStatus; + }): void { + let byRun = this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId); + if (byRun == null) { + byRun = new Map(); + this.retainedWorkflowTerminalEnqueues.set(params.ownerWorkspaceId, byRun); + } + byRun.set(params.runId, params.status); + if (this.workflowTerminalEnqueueRetryTimers.has(params.ownerWorkspaceId)) { + return; + } + const timer = setTimeout(() => { + this.workflowTerminalEnqueueRetryTimers.delete(params.ownerWorkspaceId); + void this.retryRetainedWorkflowTerminalEnqueues(params.ownerWorkspaceId); + }, this.terminalAttentionDeferRetryDelayMs); + timer.unref?.(); + this.workflowTerminalEnqueueRetryTimers.set(params.ownerWorkspaceId, timer); + } + + private async retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise { + const byRun = this.retainedWorkflowTerminalEnqueues.get(ownerWorkspaceId); + if (byRun == null) { + return; + } + this.retainedWorkflowTerminalEnqueues.delete(ownerWorkspaceId); + for (const [runId, status] of byRun) { + // A re-attempt that fails again re-retains the entry and re-arms the timer. + await this.enqueueWorkflowRunTerminalAttention({ ownerWorkspaceId, runId, status }); + } } async resetWorkflowRunTerminalAttention(params: { @@ -7724,10 +7837,40 @@ export class TaskService { "resetWorkflowRunTerminalAttention requires ownerWorkspaceId" ); assert(params.runId.length > 0, "resetWorkflowRunTerminalAttention requires runId"); - await this.terminalAttentionStore.delete( - params.ownerWorkspaceId, - TerminalAttentionStore.notificationId("workflow_run", params.runId) - ); + this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId)?.delete(params.runId); + try { + await this.terminalAttentionStore.delete( + params.ownerWorkspaceId, + TerminalAttentionStore.notificationId("workflow_run", params.runId) + ); + this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId)?.delete(params.runId); + } catch (error) { + // Restart callers swallow this rejection, so a transiently failed delete would leave + // the stale delivered/superseded record absorbing the resumed run's next terminal + // enqueue. Retain the reset; the next enqueue completes it before enqueuing fresh. + log.error("Workflow terminal attention reset failed; retained for the next enqueue", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + error, + }); + let pendingResets = this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId); + if (pendingResets == null) { + pendingResets = new Set(); + this.pendingWorkflowNotificationResets.set(params.ownerWorkspaceId, pendingResets); + } + pendingResets.add(params.runId); + } + } + + /** + * Tool-path access to the invocation-boundary snapshot recorded into the + * agent-workflow-runs sidecar (see recordBackgroundWorkflowRunReference). + */ + async getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise { + return this.workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); } async markWorkflowRunTerminalAttentionConsumed(params: { @@ -7861,6 +8004,126 @@ export class TaskService { this.pendingTerminalAttentionDrains.add(promise); } + /** + * A deferred (indeterminate) terminal wake has no deterministic "storage recovered" signal + * to re-trigger the drain, and an idle-wait would resolve immediately on an already-idle + * owner and busy-loop while the fault persists. Retry on a bounded timer instead, one armed + * timer per owner; each retry that defers again arms the next one. + */ + private scheduleTerminalAttentionDeferRetry(ownerWorkspaceId: string): void { + if (this.terminalAttentionDeferRetryTimers.has(ownerWorkspaceId)) { + return; + } + const timer = setTimeout(() => { + this.terminalAttentionDeferRetryTimers.delete(ownerWorkspaceId); + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + }, this.terminalAttentionDeferRetryDelayMs); + timer.unref?.(); + this.terminalAttentionDeferRetryTimers.set(ownerWorkspaceId, timer); + } + + /** + * Caller send restrictions (tool policy, workspace-agent disable flag, strict-agent pin) to + * restore on a terminal-attention wake. The newest manual user row carries the + * conversation's persisted restrictions; synthetic rows without any (earlier wakes, + * heartbeat scaffolding) do not define them and are skipped. The walk is unbounded: a long + * assistant/synthetic tail after the launch turn must not push the defining row out of + * sight and silently lift the restrictions. It stops at the newest context reset boundary, + * since rows from the discarded context must not re-disable tools available to the reset + * context. Throws when history is unreadable so the caller can fail closed instead of + * waking with unrestricted tools. + */ + private async resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId: string): Promise<{ + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; + }> { + // The pin and the policy resolve independently: a synthetic launch row (preserved + // heartbeat, compaction follow-up) can carry only a strict pin, and the wake bound to that + // turn's agent must keep the pin loud without lifting an older manual row's policy. + // Manual rows still define both wholesale (absence means lifted). + const state: { + pin: { strictAgentResolution?: SendMessageOptions["strictAgentResolution"] } | null; + restrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean } | null; + } = { pin: null, restrictions: null }; + const historyResult = await this.historyService.iterateFullHistory( + ownerWorkspaceId, + "backward", + (messages) => { + for (const message of messages) { + // A context reset discards everything before it: pre-reset rows must not define + // the wake's restrictions or pin. Stopping here leaves undefined fields as fresh + // defaults, matching a manual send in the post-reset context. + if (isResetBoundaryMessage(message)) { + return false; + } + if (message.role !== "user") { + continue; + } + const metadata = message.metadata; + // The strict-agent pin lives in the row's retry snapshot: an explicit agent override + // must stay loud on the wake too, or a vanished/corrupted definition would silently + // recompose the send from the exec fallback (same rule as startup retry and + // compaction follow-ups). Forwarded verbatim per the field's design note: the object + // form pins the validated definition's scope/source/chain provenance, not just + // loudness. Schema-validated like toolPolicy below, since it crosses the same + // persisted-row boundary; invalid shapes are dropped. + const rawStrictPin = metadata?.retrySendOptions?.strictAgentResolution; + const parsedStrictPin = + rawStrictPin != null && rawStrictPin !== false + ? SendMessageOptionsSchema.shape.strictAgentResolution.safeParse(rawStrictPin) + : null; + if (parsedStrictPin != null && !parsedStrictPin.success) { + log.warn("Ignoring malformed persisted strictAgentResolution on terminal wake", { + ownerWorkspaceId, + messageId: message.id, + }); + } + const strictAgentResolution = parsedStrictPin?.success ? parsedStrictPin.data : undefined; + if ( + state.pin == null && + (strictAgentResolution != null || metadata?.synthetic !== true) + ) { + state.pin = strictAgentResolution != null ? { strictAgentResolution } : {}; + } + if ( + state.restrictions == null && + (metadata?.toolPolicy != null || + metadata?.disableWorkspaceAgents != null || + metadata?.synthetic !== true) + ) { + // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep + // inside send resolution and leave the wake permanently blocked on the same corrupt + // row. Sanitize instead of trusting the JSON shape; an unparseable policy restores + // nothing while a valid disable flag still applies (self-healing doctrine). + const parsedPolicy = + metadata?.toolPolicy != null ? ToolPolicySchema.safeParse(metadata.toolPolicy) : null; + if (parsedPolicy != null && !parsedPolicy.success) { + log.warn("Ignoring malformed persisted toolPolicy on terminal wake", { + ownerWorkspaceId, + messageId: message.id, + }); + } + state.restrictions = { + ...(parsedPolicy?.success ? { toolPolicy: parsedPolicy.data } : {}), + ...(typeof metadata?.disableWorkspaceAgents === "boolean" + ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } + : {}), + }; + } + if (state.pin != null && state.restrictions != null) { + return false; + } + } + return undefined; + } + ); + if (!historyResult.success) { + throw new Error(`history unavailable: ${historyResult.error}`); + } + return { ...(state.restrictions ?? {}), ...(state.pin ?? {}) }; + } + private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { const promise = this.workspaceService .waitForIdleAndNoQueuedMessages(ownerWorkspaceId) @@ -7882,7 +8145,11 @@ export class TaskService { private async buildWorkflowTerminalPrompt( ownerWorkspaceId: string, runId: string - ): Promise { + ): Promise< + | { outcome: "deliver"; prompt: string; initiatingAgent?: WorkflowWakeInitiatingAgent } + | { outcome: "superseded" } + | { outcome: "defer" } + > { assert(ownerWorkspaceId.length > 0, "buildWorkflowTerminalPrompt requires ownerWorkspaceId"); assert(runId.length > 0, "buildWorkflowTerminalPrompt requires runId"); const runStore = new WorkflowRunStore({ @@ -7892,30 +8159,88 @@ export class TaskService { try { run = await runStore.getRun(runId); } catch (error: unknown) { + // A missing run (ENOENT) or an unparseable record (no fs code; rereading cannot repair + // it) is definitively ineligible. Every other fs failure (EIO, EACCES, EISDIR...) is + // potentially transient, and tombstoning on it would permanently drop the wake over a + // recoverable fault: defer those like indeterminate currentness below. + const code = + error != null && typeof error === "object" && "code" in error + ? (error as { code?: unknown }).code + : undefined; + if (typeof code === "string" && code !== "ENOENT") { + log.warn("Deferring workflow terminal wake-up; run record unreadable", { + ownerWorkspaceId, + runId, + error: getErrorMessage(error), + }); + return { outcome: "defer" }; + } log.warn("Failed to load terminal workflow run for wake-up", { ownerWorkspaceId, runId, error: getErrorMessage(error), }); - return null; + return { outcome: "superseded" }; } if ( run.workspaceId !== ownerWorkspaceId || run.parentWorkflow != null || - !isTerminalWorkflowRunStatus(run.status) || - !(await this.workspaceService.isWorkflowInvocationCurrent(ownerWorkspaceId, run.id)) + !isTerminalWorkflowRunStatus(run.status) ) { - return null; + return { outcome: "superseded" }; + } + const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( + ownerWorkspaceId, + run.id + ); + // Indeterminate means history was unreadable, not that the run was superseded: tombstoning + // now would permanently drop the wake over a transient fault, so defer and retry instead. + if (currentness === "indeterminate") { + return { outcome: "defer" }; + } + if (currentness === "not_current") { + return { outcome: "superseded" }; + } + // Bind the wake to the agent recorded at launch: the newest agent-bearing assistant row + // can belong to an unrelated later synthetic turn (a heartbeat is not a supersession + // boundary), which would pair a different agent's tool surface with the launch turn's + // caller policy. Advisory: legacy references fall back to the history walk. + let initiatingAgent: WorkflowWakeInitiatingAgent | undefined; + try { + const references = await readAgentWorkflowRunReferences( + this.config.getSessionDir(ownerWorkspaceId) + ); + const reference = references.find((candidate) => candidate.runId === run.id); + if (reference?.agentId != null) { + initiatingAgent = { + agentId: reference.agentId, + createdAtMs: reference.createdAtMs, + ...(reference.strictAgentResolution !== undefined + ? { strictAgentResolution: reference.strictAgentResolution } + : {}), + }; + } + } catch { + // Currentness can succeed (e.g. a direct invocation row) and this identity read still + // fail transiently. Delivering without the recorded identity would bind the wake to the + // newest agent-bearing history row, handing the run's output to an unrelated later + // synthetic turn's agent; defer to the bounded retry instead, like an unreadable run + // record. + return { outcome: "defer" }; } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; - return buildWorkflowResultContextMessage({ - rawCommand: `workflow_run ${scriptPath}`, - name: scriptPath, - runId: run.id, - status: run.status, - result: null, - run, - }); + return { + outcome: "deliver", + ...(initiatingAgent != null ? { initiatingAgent } : {}), + prompt: buildWorkflowResultContextMessage({ + rawCommand: `workflow_run ${scriptPath}`, + name: scriptPath, + runId: run.id, + status: run.status, + result: null, + run, + }), + }; } private async ensureAgentTerminalMessages( @@ -8221,6 +8546,11 @@ export class TaskService { (notification) => notification.sourceKind === "workflow_run" ); const deliverableWorkflowNotificationIds = new Set(); + const deliverableWorkflowPrompts: Array<{ + notificationId: string; + prompt: string; + initiatingAgent?: WorkflowWakeInitiatingAgent; + }> = []; const promptSections: string[] = []; if (publicAwaitIds.length > 0) { @@ -8231,12 +8561,73 @@ export class TaskService { ownerWorkspaceId, notification.sourceId ); - if (workflowPrompt == null) { + if (workflowPrompt.outcome === "defer") { + // Currentness was indeterminate (history or sidecar unreadable): keep the notification + // pending rather than permanently dropping the wake, and arm a bounded retry, because an + // already-idle owner produces no further drain trigger on its own. + log.warn("Deferring workflow terminal attention; history unavailable", { + ownerWorkspaceId, + runId: notification.sourceId, + }); + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); + continue; + } + if (workflowPrompt.outcome === "superseded") { + // Dropping a notify_on_terminal wake strands the run's owner; keep the drop diagnosable. + log.warn("Dropping superseded workflow terminal attention", { + ownerWorkspaceId, + runId: notification.sourceId, + }); await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); continue; } - deliverableWorkflowNotificationIds.add(notification.id); - promptSections.push(workflowPrompt); + deliverableWorkflowPrompts.push({ + notificationId: notification.id, + prompt: workflowPrompt.prompt, + ...(workflowPrompt.initiatingAgent != null + ? { initiatingAgent: workflowPrompt.initiatingAgent } + : {}), + }); + } + // Deliver one launch-identity group per drain, keyed by agentId AND recorded strict pin: + // the whole coalesced prompt is handled under the single agentId/pin passed to + // sendMessage, so batching runs from different agents (or runs sharing an agentId but + // launched under different pins, e.g. an agent definition replaced between synthetic + // turns) would hand a restricted launch's (attacker-influenced) output to another + // launch's tool grants. The same applies to mixed batches: workspace-turn and sub-agent + // attention resumes under the conversation's own (history-walk) identity, so agent-bound + // workflow groups never share their send. Deferred groups stay pending and deliver on the + // re-armed retry drain; among agent-bound groups the newest launch goes first. + const hasNonWorkflowDeliverables = + deliverableAgentNotificationIds.size > 0 || deliverableWorkspaceTurnNotificationIds.size > 0; + let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; + if (!hasNonWorkflowDeliverables) { + for (const candidate of deliverableWorkflowPrompts) { + const agent = candidate.initiatingAgent; + if ( + agent != null && + (workflowInitiatingAgent == null || + agent.createdAtMs > workflowInitiatingAgent.createdAtMs) + ) { + workflowInitiatingAgent = agent; + } + } + } + const selectedGroupKey = + workflowInitiatingAgent != null ? workflowWakeGroupKey(workflowInitiatingAgent) : undefined; + const selectedWorkflowPrompts = deliverableWorkflowPrompts.filter((candidate) => + hasNonWorkflowDeliverables + ? candidate.initiatingAgent == null + : selectedGroupKey == null || + (candidate.initiatingAgent != null && + workflowWakeGroupKey(candidate.initiatingAgent) === selectedGroupKey) + ); + if (selectedWorkflowPrompts.length < deliverableWorkflowPrompts.length) { + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); + } + for (const candidate of selectedWorkflowPrompts) { + deliverableWorkflowNotificationIds.add(candidate.notificationId); + promptSections.push(candidate.prompt); } // Sub-agent reports and failures are already durable user-context messages. Resume from history @@ -8268,16 +8659,50 @@ export class TaskService { const resumeOptions = await this.resolveParentAutoResumeOptions( ownerWorkspaceId, entry, - defaultModel + defaultModel, + workflowInitiatingAgent != null ? { agentId: workflowInitiatingAgent.agentId } : undefined ); const workspaceTurnMuxMetadata = await this.getActiveWorkspaceTurnMuxMetadataForWorkspace(ownerWorkspaceId); + // Security: restore the conversation's active caller tool policy on the wake. The ordinary + // in-stream workflow continuation carries the live turn's effectiveToolPolicy; this + // synthetic send starts a fresh turn, and omitting the policy would let a workflow wake + // regain tools the caller disabled (with attacker-influenced workflow output choosing the + // timing). The agent-level policy recomposes from agentId at send resolution. + let wakeRestrictions: { + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; + }; + try { + wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); + } catch (error: unknown) { + // Fail closed: an unknown policy must not fall back to unrestricted tools. + log.warn("Deferring terminal wake; caller tool policy unavailable", { + ownerWorkspaceId, + error, + }); + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); + return; + } + + // Pair the pin with the selected group: the newest pin-bearing history row can belong to + // a different group's wake (each wake persists its own pin), and pinning another agent's + // provenance onto this group's agentId makes resolution reject the wake on every retry. A + // recorded pin (or a verified-unpinned null) overrides the walk; legacy references + // without the field keep the walk pin. + const groupPin = workflowInitiatingAgent?.strictAgentResolution; + const effectiveStrictPin = + groupPin !== undefined ? (groupPin ?? undefined) : wakeRestrictions.strictAgentResolution; const sendOptions = { model: resumeOptions.model, agentId: resumeOptions.agentId, thinkingLevel: resumeOptions.thinkingLevel, reasoningMode: resumeOptions.reasoningMode, + ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), + ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), + ...(effectiveStrictPin != null ? { strictAgentResolution: effectiveStrictPin } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index eb60511d88..d842b9ab22 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -89,17 +89,75 @@ export async function emitWorkflowRunAttachedEvent(input: { export async function recordBackgroundWorkflowRunReference( config: ToolConfiguration, runId: string, - createdAtMs: number + createdAtMs: number, + options?: { + /** + * Pre-launch records must not fail soft: the sidecar is the kernel invocation's only + * durable provenance, and starting the runner without it lets a fast terminal run have + * its wake permanently marked superseded. Throwing before dispatch leaves the run + * pending and resumable (workflow_resume re-records), so the caller surfaces a loud, + * recoverable launch failure instead. Post-dispatch records stay best-effort because the + * run already started and failing the tool would strand it. + */ + propagateWriteFailure?: boolean; + } ): Promise { const workspaceSessionDir = config.workspaceSessionDir; if (workspaceSessionDir == null || workspaceSessionDir.length === 0) { + if (options?.propagateWriteFailure === true) { + throw new Error( + `Cannot record workflow run provenance without a workspace session dir: ${runId}` + ); + } log.warn("Skipping agent workflow run reference without workspace session dir", { runId }); return; } + // Snapshot which invocation-decision row is newest at launch so the terminal-wake + // currentness check compares row identity instead of wall-clock order, which clock + // corrections can reorder (see WorkspaceService.isWorkflowInvocationCurrent). A history read + // failure must not be persisted as a verified-empty boundary (null): record without the + // field instead, so the run stays rediscoverable (listAgentReferencedWorkflowRunIds) and a + // later workflow_resume re-record can repair provenance, while the unverifiable boundary + // defers wake delivery (indeterminate) instead of guessing from wall-clock order. + let afterBoundaryMessageId: string | null | undefined; + const taskService = config.taskService; + if (config.workspaceId != null && taskService?.getWorkflowInvocationBoundaryMessageId != null) { + try { + afterBoundaryMessageId = await taskService.getWorkflowInvocationBoundaryMessageId( + config.workspaceId, + runId + ); + } catch (error: unknown) { + log.error("Failed to snapshot workflow invocation boundary for run reference", { + runId, + error: getErrorMessage(error), + }); + } + } + try { - await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + ...(config.agentId != null && config.agentId.length > 0 + ? { + agentId: config.agentId, + // The pin pairs with the identity: null records a verified-unpinned launch so the + // wake never inherits another row's pin (see AgentWorkflowRunReference). + strictAgentResolution: + config.strictAgentResolution != null && config.strictAgentResolution !== false + ? config.strictAgentResolution + : null, + } + : {}), + }); } catch (error: unknown) { + if (options?.propagateWriteFailure === true) { + throw error; + } log.warn("Failed to record agent workflow run reference", { runId, error: getErrorMessage(error), diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index 1ae69b9402..f6433a1651 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -6,6 +6,7 @@ import { TestTempDir, createTestToolConfig } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; import { WORKFLOW_CHECKPOINT_RETRY_ERROR_MESSAGE } from "@/common/utils/workflowRetryEligibility"; import type { WorkflowRunRecord } from "@/common/types/workflow"; +import type { TaskService } from "@/node/services/taskService"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; const mockToolCallOptions: ToolExecutionOptions = { @@ -168,7 +169,19 @@ describe("workflow_resume tool", () => { test("resumes in background and records an agent workflow run reference", async () => { using tempDir = new TestTempDir("test-workflow-resume-bg"); - const workflowService = buildWorkflowService(); + // The reference must NOT be durable before the dispatch: the run still sits in its old + // terminal state until the dispatch restarts it, and a pre-dispatch reference would let a + // crash in that window replay the stale failure/interruption as a current wake. + let referenceDurableAtDispatch = false; + const workflowService = buildWorkflowService({ + resumeRunInBackground: mock(async () => { + const references = await readAgentWorkflowRunReferences(tempDir.path); + referenceDurableAtDispatch = references.some( + (reference) => reference.runId === "wfr_resume_me" + ); + return { runId: "wfr_resume_me", status: "running" as const, result: null }; + }), + }); const tool = createWorkflowResumeTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: false, @@ -186,6 +199,7 @@ describe("workflow_resume tool", () => { projectTrusted: false, }); expect(workflowService.resumeRun).not.toHaveBeenCalled(); + expect(referenceDurableAtDispatch).toBe(false); const references = await readAgentWorkflowRunReferences(tempDir.path); expect(references.map((reference) => reference.runId)).toContain("wfr_resume_me"); expect(result).toMatchObject({ status: "running", runId: "wfr_resume_me", mode: "resume" }); @@ -229,6 +243,143 @@ describe("workflow_resume tool", () => { }); }); + test("marks terminal attention consumed when returning an already-completed run's result", async () => { + using tempDir = new TestTempDir("test-workflow-resume-consumed"); + const completedRun = buildRun({ + status: "completed", + events: [ + { sequence: 1, type: "status", at: "2026-05-29T00:00:00.000Z", status: "running" }, + { + sequence: 2, + type: "result", + at: "2026-05-29T00:00:01.000Z", + result: { reportMarkdown: "already done" }, + }, + { sequence: 3, type: "status", at: "2026-05-29T00:00:01.000Z", status: "completed" }, + ], + }); + const workflowService = buildWorkflowService({ getRun: mock(async () => completedRun) }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: null }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + }); + + test("does not mark terminal attention consumed for background dispatches", async () => { + using tempDir = new TestTempDir("test-workflow-resume-background-no-consume"); + // The refresh after a background dispatch can still observe the stale pre-dispatch failed + // status; consuming it would tombstone the retried run's future terminal wake. + const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: true, mode: "retry_from_checkpoint" }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).not.toHaveBeenCalled(); + }); + + test("marks terminal attention consumed when a foreground retry finishes terminal", async () => { + using tempDir = new TestTempDir("test-workflow-resume-foreground-consume"); + const failedRun = buildFailedRun(); + const completedRun = buildRun({ + status: "completed", + events: [ + { sequence: 1, type: "status", at: "2026-05-29T00:00:00.000Z", status: "running" }, + { sequence: 2, type: "status", at: "2026-05-29T00:00:02.000Z", status: "completed" }, + ], + }); + let getRunCalls = 0; + const workflowService = buildWorkflowService({ + getRun: mock(async () => { + getRunCalls += 1; + return getRunCalls === 1 ? failedRun : completedRun; + }), + retryRunFromCheckpoint: mock(async () => ({ + runId: "wfr_resume_me", + status: "completed" as const, + result: { reportMarkdown: "retried" }, + })), + }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: "retry_from_checkpoint" }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + }); + + test("consumes the foreground terminal result even when the refresh read fails", async () => { + using tempDir = new TestTempDir("test-workflow-resume-refresh-failure-consume"); + // WorkflowService.getRun collapses transient read failures to null. The terminal result is + // still returned to the model, so consumption must derive from the dispatch result or the + // pending terminal attention would re-inject it later. + let getRunCalls = 0; + const workflowService = buildWorkflowService({ + getRun: mock(async () => { + getRunCalls += 1; + return getRunCalls === 1 ? buildRun() : null; + }), + resumeRun: mock(async () => ({ + runId: "wfr_resume_me", + status: "completed" as const, + result: { reportMarkdown: "resumed" }, + })), + }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + const result = await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: null }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + expect(result).toMatchObject({ status: "completed", runId: "wfr_resume_me", mode: "resume" }); + }); + test("rejects default resume of a failed run with checkpoint retry guidance", async () => { using tempDir = new TestTempDir("test-workflow-resume-failed"); const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index a94188569a..0fac258ed3 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -2,9 +2,9 @@ import { tool } from "ai"; import { getErrorMessage } from "@/common/utils/errors"; import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; -import type { WorkflowRunRecord } from "@/common/types/workflow"; +import { isTerminalWorkflowRunStatus, type WorkflowRunRecord } from "@/common/types/workflow"; import { getWorkflowCheckpointRetryEligibility } from "@/common/utils/workflowRetryEligibility"; -import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; +import { WorkflowRunRecordSchema, WorkflowRunStatusSchema } from "@/common/orpc/schemas"; import { WorkflowResumeToolResultSchema, TOOL_DEFINITIONS, @@ -154,9 +154,27 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const mode: WorkflowResumeMode = args.mode ?? "resume"; const invocationStartedAtMs = Date.now(); + // A kernel-nested resume (mux.workflow_resume inside code_execution) leaves no top-level + // workflow_resume part in history, so the history-walk consumption predicates cannot see + // that this turn already received the terminal result. Persist consumption durably so the + // terminal-attention drain never re-delivers it. + const markTerminalAttentionConsumed = async ( + terminalRun: Pick + ) => { + if (!isTerminalWorkflowRunStatus(terminalRun.status)) { + return; + } + await config.taskService?.markWorkflowRunTerminalAttentionConsumed?.({ + ownerWorkspaceId: workspaceId, + runId: terminalRun.id, + status: terminalRun.status, + }); + }; + // Idempotent success: the work is already done, so hand back the durable result instead // of failing the agent's recovery loop (e.g. resuming after a crash that actually finished). if (run.status === "completed" && mode === "resume") { + await markTerminalAttentionConsumed(run); return parseToolResult( WorkflowResumeToolResultSchema, { @@ -216,6 +234,12 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Background-style resumes outlive this turn; persist provenance so the run is // rediscoverable (task_await/task_list) and its terminal result re-engages the agent. + // Deliberately AFTER the dispatch, unlike workflow_run's onRunCreated record: the + // resumed run sits in an old terminal state until the dispatch durably restarts it, and + // a pre-dispatch reference would let a crash in that window make startup recovery + // deliver the stale failure/interruption as a current wake. Recording late fails safe + // instead (a crash loses this resume's wake); with the process alive, delivery always + // waits for the owner to go idle, by which point this record is durable. const isBackgroundDispatch = args.run_in_background === true || dispatched.status === "backgrounded"; if (isBackgroundDispatch) { @@ -231,6 +255,18 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const refreshedRunIsStale = isBackgroundDispatch && refreshedRun != null && refreshedRun.status === run.status; + // Foreground only: a background dispatch can still observe the stale pre-dispatch terminal + // status, and consuming it would tombstone the retried run's future terminal wake. + // Consumption derives from the dispatch result itself, not the refreshed record: the + // terminal result is returned to the model below even when the refresh read fails, and + // skipping the tombstone would let the pending terminal attention re-inject it later. + if (!isBackgroundDispatch) { + const dispatchedStatus = WorkflowRunStatusSchema.safeParse(dispatched.status); + if (dispatchedStatus.success) { + await markTerminalAttentionConsumed({ id: runId, status: dispatchedStatus.data }); + } + } + return parseToolResult( WorkflowResumeToolResultSchema, { diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index 0636a54d8b..3e394d5c35 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -13,9 +13,15 @@ import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { resolveWorkflowScript } from "@/node/services/workflows/workflowScriptResolver"; import { TestTempDir, createTestToolConfig, writeProjectSkill } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import type { TaskService } from "@/node/services/taskService"; +import type { ToolConfiguration } from "@/common/utils/tools/tools"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkflowRunRecord } from "@/common/types/workflow"; +type BackgroundStartInput = Parameters< + NonNullable["startWorkflowInBackground"]> +>[0]; + const mockToolCallOptions: ToolExecutionOptions = { toolCallId: "test-call-id", messages: [], @@ -644,15 +650,31 @@ describe("workflow_run tool", () => { const startWorkflow = mock(async () => { throw new Error("foreground start should not be used"); }); - const startWorkflowInBackground = mock(async () => ({ - runId: "wfr_background", - status: "running" as const, - result: null, - })); + // Faithful to WorkflowService: onRunCreated is awaited at run creation, before the runner + // starts executing. The probe checks the sidecar reference is already durable at that + // point, so a fast run (or a crash) after launch cannot lose its terminal wake. + let referenceDurableBeforeRunnerStart = false; + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_background", + status: "pending", + result: null, + run: null, + }); + const references = await readAgentWorkflowRunReferences(tempDir.path); + referenceDurableBeforeRunnerStart = references.some( + (reference) => reference.runId === "wfr_background" + ); + return { runId: "wfr_background", status: "running" as const, result: null }; + }); const getRun = mock(async () => null); + const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); const tool = createWorkflowRunTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, workflowService: { startWorkflow, startWorkflowInBackground, @@ -665,8 +687,23 @@ describe("workflow_run tool", () => { mockToolCallOptions ); + expect(referenceDurableBeforeRunnerStart).toBe(true); + // The reference must persist the invocation-boundary snapshot: currentness compares row + // identity, so a reference recorded without it fails safe and the wake is dropped. const references = await readAgentWorkflowRunReferences(tempDir.path); - expect(references.map((reference) => reference.runId)).toContain("wfr_background"); + expect(references).toHaveLength(1); + expect(references[0]).toMatchObject({ + runId: "wfr_background", + afterBoundaryMessageId: "boundary-row-1", + // The wake binds to the launching agent, so the reference must carry its identity + // and the launch turn's provenance pin. + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(getWorkflowInvocationBoundaryMessageId).toHaveBeenCalledWith( + "workspace-1", + "wfr_background" + ); expect(startWorkflowInBackground).toHaveBeenCalledWith( expect.objectContaining({ @@ -681,6 +718,83 @@ describe("workflow_run tool", () => { expect(result).toEqual({ status: "running", runId: "wfr_background", result: null }); }); + test("records a rediscovery-only reference when the boundary snapshot fails", async () => { + using tempDir = new TestTempDir("test-workflow-run-tool-boundary-error"); + const scriptPath = await writeWorkflowScript(tempDir.path); + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_boundary_error", + status: "pending", + result: null, + run: null, + }); + return { runId: "wfr_boundary_error", status: "running" as const, result: null }; + }); + const getWorkflowInvocationBoundaryMessageId = mock(async () => { + throw new Error("history read failed"); + }); + const tool = createWorkflowRunTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, + workflowService: { + startWorkflow: mock(async () => { + throw new Error("foreground start should not be used"); + }), + startWorkflowInBackground, + getRun: mock(async () => null), + }, + }); + + const result = await tool.execute!( + { script_path: scriptPath, args: { topic: "workflow tools" }, run_in_background: true }, + mockToolCallOptions + ); + expect(result).toEqual({ status: "running", runId: "wfr_boundary_error", result: null }); + + // A read failure must not persist a verified-empty boundary (null): the entry keeps the run + // rediscoverable while a later resume re-record can repair provenance. + const references = await readAgentWorkflowRunReferences(tempDir.path); + expect(references).toHaveLength(1); + expect(references[0]?.runId).toBe("wfr_boundary_error"); + expect(references[0] != null && "afterBoundaryMessageId" in references[0]).toBe(false); + }); + + test("a sidecar write failure aborts the background launch before the runner starts", async () => { + using tempDir = new TestTempDir("test-workflow-run-tool-sidecar-write-error"); + const scriptPath = await writeWorkflowScript(tempDir.path); + // The reference path exists as a directory, so every sidecar write fails. + await fs.mkdir(path.join(tempDir.path, "agent-workflow-runs.json")); + let runnerStarted = false; + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_sidecar_write_error", + status: "pending", + result: null, + run: null, + }); + runnerStarted = true; + return { runId: "wfr_sidecar_write_error", status: "running" as const, result: null }; + }); + const getRun = mock(async () => null); + const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); + const tool = createWorkflowRunTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + agentId: "plan", + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, + workflowService: { startWorkflowInBackground, getRun }, + }); + + // Starting the runner without durable provenance would let a fast terminal run have its + // wake permanently superseded; the launch must fail loudly instead, leaving the run + // pending and resumable (workflow_resume re-records provenance). + await expect( + tool.execute!({ script_path: scriptPath, run_in_background: true }, mockToolCallOptions) + ).rejects.toThrow(/created durable run wfr_sidecar_write_error/); + expect(runnerStarted).toBe(false); + }); + test("requires the workflow service", async () => { using tempDir = new TestTempDir("test-workflow-run-tool-missing"); const scriptPath = await writeWorkflowScript(tempDir.path); diff --git a/src/node/services/tools/workflow_run.ts b/src/node/services/tools/workflow_run.ts index 425b379eae..300da0f60f 100644 --- a/src/node/services/tools/workflow_run.ts +++ b/src/node/services/tools/workflow_run.ts @@ -262,6 +262,7 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => } } const createdRun: { id: string | null } = { id: null }; + const invocationStartedAtMs = Date.now(); const startInput = { script, workspaceId, @@ -271,6 +272,14 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => createdRun.id = event.runId; // The run record is durable now, so a concurrent duplicate launch will see it. releaseAdmission?.(); + // Provenance must be durable BEFORE the runner starts: a fast background run (or a + // process exit mid-dispatch) can reach terminal state before any post-dispatch + // write, and a terminal wake with no sidecar reference is permanently superseded. + if (args.run_in_background === true) { + await recordBackgroundWorkflowRunReference(config, event.runId, invocationStartedAtMs, { + propagateWriteFailure: true, + }); + } await emitWorkflowRunAttachedEvent({ config, workspaceId, @@ -280,7 +289,6 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => }); }, }; - const invocationStartedAtMs = Date.now(); let result: { runId: string; status: string; result: unknown }; try { result = @@ -335,7 +343,10 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => releaseAdmission?.(); } - if (isBackgroundWorkflowResult(args, result.status)) { + // Explicit background launches already recorded provenance in onRunCreated; this covers + // a foreground dispatch that backgrounded itself, where the run ID outcome is only + // knowable post-dispatch. + if (args.run_in_background !== true && isBackgroundWorkflowResult(args, result.status)) { await recordBackgroundWorkflowRunReference(config, result.runId, invocationStartedAtMs); } diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index 008acd2a5c..bfcb2ce081 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -1007,3 +1007,117 @@ describe("WorkflowRunStore.listActiveRunSummaries", () => { expect(summaries.map((summary) => summary.runId)).toEqual(["wfr_healthy"]); }); }); + +describe("WorkflowService crash recovery", () => { + test("crash resume fires provenance repair before the run reaches terminal", async () => { + using tmp = new DisposableTempDir("workflow-service-crash-repair"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_crash", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo workflow", scope: "built-in", executable: true }, + source: 'export default function workflow() { return { reportMarkdown: "done" }; }\n', + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + // Orphaned by a crash: durable status says running, but no runner holds the lease. + await runStore.appendStatus("wfr_crash", "running", "2026-05-29T00:00:01.000Z"); + + const events: string[] = []; + let resolveCompleted: (() => void) | undefined; + const completed = new Promise((resolve) => { + resolveCompleted = resolve; + }); + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + generateRunId: () => "wfr_unused", + runnerId: "runner-a", + onRunCrashResumed: (event) => { + events.push(`repair:${event.workspaceId}:${event.runId}`); + }, + onRunStatusChanged: (event) => { + events.push(`status:${event.status}`); + if (event.status === "completed") { + resolveCompleted?.(); + } + }, + }); + + const resumed = await service.resumeCrashedRuns({ + workspaceId: "workspace-1", + projectTrusted: true, + }); + expect(resumed).toEqual(["wfr_crash"]); + await completed; + // The repair hook is awaited before the runner restarts, so even an instantly completing + // run cannot reach terminal with unrepaired provenance. + expect(events[0]).toBe("repair:workspace-1:wfr_crash"); + expect(events).toContain("status:completed"); + await expect(runStore.getRun("wfr_crash")).resolves.toMatchObject({ status: "completed" }); + }); + + test("failed crash-resume provenance repair retries on a bounded timer", async () => { + using tmp = new DisposableTempDir("workflow-service-crash-repair-retry"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_crash_repair_retry", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo workflow", scope: "built-in", executable: true }, + source: 'export default function workflow() { return { reportMarkdown: "done" }; }\n', + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_crash_repair_retry", "running", "2026-05-29T00:00:01.000Z"); + + // The repair hook only fires at resume time; if its transient failure were terminal, the + // reference would stay boundaryless after the run settles and every drain would defer the + // wake as indeterminate with nothing left to repair it. + const repairCalls: string[] = []; + let failFirstRepair = true; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + generateRunId: () => "wfr_unused", + runnerId: "runner-a", + onRunCrashResumed: (event) => { + repairCalls.push(`${event.workspaceId}:${event.runId}`); + if (failFirstRepair) { + failFirstRepair = false; + throw new Error("EIO: sidecar write failed"); + } + }, + }); + ( + service as unknown as { crashResumeRepairRetryDelayMs: number } + ).crashResumeRepairRetryDelayMs = 10; + + const resumed = await service.resumeCrashedRuns({ + workspaceId: "workspace-1", + projectTrusted: true, + }); + expect(resumed).toEqual(["wfr_crash_repair_retry"]); + expect(repairCalls).toEqual(["workspace-1:wfr_crash_repair_retry"]); + + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && repairCalls.length < 2) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(repairCalls).toEqual([ + "workspace-1:wfr_crash_repair_retry", + "workspace-1:wfr_crash_repair_retry", + ]); + }); +}); diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 70b8fb35f2..83fa80f363 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -54,6 +54,12 @@ export interface WorkflowServiceOptions { resolveWorkflowScript?: (scriptPath: string) => Promise; onBackgroundRunTerminal?: (event: WorkflowBackgroundRunTerminalEvent) => Promise | void; onRunStatusChanged?: (event: WorkflowRunStatusChangedEvent) => Promise | void; + /** + * Fired when crash recovery is about to resume an orphaned run, before the runner restarts. + * Used to repair wake provenance a pre-boundary build stripped from the sidecar; awaited so + * a fast run cannot reach terminal before the repair lands. + */ + onRunCrashResumed?: (event: { workspaceId: string; runId: string }) => Promise | void; /** When true, background terminal notifications also fire for interrupted runs. */ notifyInterruptedBackgroundRunTerminal?: boolean; generateRunId?: () => string; @@ -112,6 +118,13 @@ const WORKFLOW_BACKGROUND_CONTINUATION_STATUSES = new Set([ // oRPC creates a WorkflowService per request, so workflow lifecycle state that spans requests // needs process-wide registries. const pendingCrashResumeTimers = new Map>(); +// Crash-resume provenance repair only fires at resume time: once the run settles, nothing +// else re-records the boundary and its terminal wake stays indeterminate on every drain. +// Retry a failed repair on a bounded timer. Module-level like pendingCrashResumeTimers +// because WorkflowService instances are per-request. +const CRASH_RESUME_REPAIR_RETRY_DELAY_MS = 30_000; +const CRASH_RESUME_REPAIR_MAX_ATTEMPTS = 5; +const pendingCrashResumeRepairTimers = new Map>(); const activeWorkflowInterruptStatusWrites = new Map>(); const activeWorkflowRunnerAbortControllers = new Map(); @@ -127,6 +140,12 @@ export class WorkflowService { private readonly onBackgroundRunTerminal?: ( event: WorkflowBackgroundRunTerminalEvent ) => Promise | void; + private readonly onRunCrashResumed?: (event: { + workspaceId: string; + runId: string; + }) => Promise | void; + // Field, not the constant, so tests can shrink the repair retry backoff. + private crashResumeRepairRetryDelayMs = CRASH_RESUME_REPAIR_RETRY_DELAY_MS; private readonly onRunStatusChanged?: ( event: WorkflowRunStatusChangedEvent ) => Promise | void; @@ -150,6 +169,7 @@ export class WorkflowService { this.taskAdapterFactory = options.taskAdapterFactory; this.resolveWorkflowScript = options.resolveWorkflowScript; this.onBackgroundRunTerminal = options.onBackgroundRunTerminal; + this.onRunCrashResumed = options.onRunCrashResumed; this.onRunStatusChanged = options.onRunStatusChanged; this.notifyInterruptedBackgroundRunTerminal = options.notifyInterruptedBackgroundRunTerminal === true; @@ -573,6 +593,19 @@ export class WorkflowService { return false; } + if (this.onRunCrashResumed != null) { + try { + await this.onRunCrashResumed({ workspaceId: run.workspaceId, runId: run.id }); + } catch (error) { + // Best-effort: an unrepaired reference defers its wake as indeterminate rather than + // losing it, so a failed repair must not block the resume itself. Retry off-path: + // repair is CAS-guarded and refuses once a boundary exists, so late success (even + // after the run settles) only unblocks the deferred wake. + console.error("Workflow crash-resume provenance repair failed:", error); + this.scheduleCrashResumeRepairRetry({ workspaceId: run.workspaceId, runId: run.id }, 1); + } + } + const retryDelayMs = await this.runStore.getLeaseRetryDelayMs( input.runId, this.clock?.nowMs() ?? Date.now() @@ -630,6 +663,32 @@ export class WorkflowService { pendingCrashResumeTimers.set(input.runId, timer); } + private scheduleCrashResumeRepairRetry( + input: { workspaceId: string; runId: string }, + attempt: number + ): void { + const repairHook = this.onRunCrashResumed; + if (repairHook == null || attempt > CRASH_RESUME_REPAIR_MAX_ATTEMPTS) { + return; + } + if (pendingCrashResumeRepairTimers.has(input.runId)) { + return; + } + const timer = setTimeout(() => { + pendingCrashResumeRepairTimers.delete(input.runId); + void (async () => { + try { + await repairHook({ workspaceId: input.workspaceId, runId: input.runId }); + } catch (error) { + console.error("Workflow crash-resume provenance repair retry failed:", error); + this.scheduleCrashResumeRepairRetry(input, attempt + 1); + } + })(); + }, this.crashResumeRepairRetryDelayMs); + unrefTimer(timer); + pendingCrashResumeRepairTimers.set(input.runId, timer); + } + private registerActiveRunnerAbortController( runId: string, workspaceId: string, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ec69f044d2..9efd4e1d6c 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -54,9 +54,15 @@ import type { SendMessageOptions, WorkspaceChatMessage } from "@/common/orpc/typ import { createMuxMessage } from "@/common/types/message"; import { buildStagedAttachmentNotice } from "@/browser/features/ChatInput/stagedAttachments"; import { + WORKFLOW_RESULT_METADATA_TYPE, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, + buildWorkflowResultContextMessage, } from "@/common/utils/workflowRunMessages"; +import { + readAgentWorkflowRunReferences, + recordAgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; import * as todoStorageModule from "@/node/services/todos/todoStorage"; import * as runtimeFactory from "@/node/runtime/runtimeFactory"; @@ -5924,6 +5930,794 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("counts a kernel-launched run recorded in the sidecar as the current invocation", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-kernel"; + const runId = "wfr_currentness_kernel"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-kernel", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // mux.workflow_run inside code_execution leaves no workflow_run tool part in history; the + // agent-workflow-runs sidecar reference is the only durable invocation evidence. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-kernel-launch", "assistant", "", { timestamp: 1_100 }, [ + { + type: "dynamic-tool", + toolCallId: "code-exec-1", + toolName: "code_execution", + state: "output-available", + input: { code: "return xum.workflow_run({ script_path: './workflows/demo.js' })" }, + output: { success: true, result: { status: "running", runId } }, + }, + ]) + ); + + // The nested runId in the code_execution output alone is not invocation evidence. + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A newer manual user message supersedes the sidecar reference. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A kernel workflow_resume re-records the reference after the supersession and + // re-establishes provenance (latest record wins). + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_250, + afterBoundaryMessageId: "manual-user-2", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // Once the terminal result was delivered, the sidecar must not resurrect the invocation. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("workflow-result", "user", "The workflow below has finished.", { + timestamp: 1_300, + synthetic: true, + muxMetadata: { + type: WORKFLOW_RESULT_METADATA_TYPE, + rawCommand: "workflow_run ./workflows/demo.js", + runId, + }, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A kernel background resume issued after the delivered result re-records the reference, + // so the retried run's next terminal wake must count as current again. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_350, + afterBoundaryMessageId: "workflow-result", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("does not treat sidecar references as current after a full history clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-cleared"; + const runId = "wfr_currentness_cleared"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-cleared", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A full clear (truncateHistory) removes every row without appending a reset boundary + // and leaves the sidecar intact; the surviving reference must not inject a workflow + // result into the freshly cleared conversation. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("delivers kernel launches recorded against a decision-free history", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-empty"; + const runId = "wfr_currentness_empty"; + const legacyRunId = "wfr_currentness_empty_legacy"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-empty", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A kernel launch from a synthetic turn in a new (or fully cleared) workspace records a + // verified-empty snapshot (null). History still having no decision row means the launch + // context is unchanged, so the wake must deliver. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A reference without a verified snapshot cannot claim the empty history as its launch + // context; it may merely have survived a full clear. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: legacyRunId, + createdAtMs: 1_150, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, legacyRunId)).toBe( + false + ); + + // A decision row appearing after the launch supersedes the verified-empty snapshot. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("retires kernel workflow run references on a full history clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire"; + const runId = "wfr_currentness_retire"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // Launched from a decision-free history: the verified-empty snapshot delivers. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A full clear returns history to decision-free, making the pre-clear null snapshot + // indistinguishable from a fresh empty-history launch; the clear must retire the + // reference so the stale result cannot inject into the fresh conversation. + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(true); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("retires kernel workflow run references even when a later post-clear step fails", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire-early"; + const runId = "wfr_currentness_retire_early"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire-early", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + + // The truncation commits, then a later post-clear step fails. Retirement must already + // have happened, or the stale null-snapshot reference survives the committed clear and + // reads current against the emptied history. + const sessionAccessor = workspaceService as unknown as { + getOrCreateSession(id: string): { clearPostCompactionState(): Promise }; + }; + const session = sessionAccessor.getOrCreateSession(workspaceId); + const carryoverSpy = spyOn(session, "clearPostCompactionState").mockImplementationOnce(() => + Promise.reject(new Error("carryover discard failed")) + ); + try { + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(false); + } finally { + carryoverSpy.mockRestore(); + } + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-coalesced"; + const runId = "wfr_currentness_coalesced"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-coalesced", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // Another run's payload quoting nothing about this run must not count as consumption. + await historyService.appendToHistory( + workspaceId, + createMuxMessage( + "coalesced-other", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run other.js", + name: "other.js", + runId: "wfr_currentness_other", + status: "completed", + result: { reportMarkdown: "other done" }, + run: null, + }), + { timestamp: 1_250, synthetic: true } + ) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a + // crash between durable acceptance and the outbox delivery mark, this row is the only + // evidence the result already reached history; it must read as consumption or restart + // recovery injects the same terminal result again. + await historyService.appendToHistory( + workspaceId, + createMuxMessage( + "coalesced-result", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run research.js", + name: "research.js", + runId, + status: "completed", + result: { reportMarkdown: "done" }, + run: null, + }), + { timestamp: 1_300, synthetic: true } + ) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-clock"; + const runId = "wfr_currentness_clock"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-clock", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // A backward clock correction after recording makes the reference timestamp future-dated + // relative to every later history row; identity comparison must still deliver the wake. + const skewedCreatedAtMs = Date.now() + 30 * 60_000; + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: skewedCreatedAtMs, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A user message written after the correction has a smaller timestamp than the reference; + // wall-clock ordering would keep the stale reference current, identity must not. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("defers boundaryless sidecar references instead of trusting wall-clock order", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-legacy"; + const runId = "wfr_currentness_legacy"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-legacy", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // A reference without a boundary snapshot (pre-upgrade entry or record-time history read + // failure) cannot be ordered against the decision row by identity: the wake defers + // instead of delivering, and the boolean caller stays fail-safe. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + }); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A backward clock correction gives the newer superseding turn an OLDER timestamp than + // the reference. Wall-clock ordering would resurrect the superseded reference as current + // and deliver its output under the newer turn's tool policy; it must stay deferred. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_100, + }) + ); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("crash-resume repair restores provenance only on supersession-free evidence", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-crash-repair"; + const strippedRunId = "wfr_crash_repair_stripped"; + const supersededRunId = "wfr_crash_repair_superseded"; + const anchoredRunId = "wfr_crash_repair_anchored"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-crash-repair", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A downgrade rewrote the sidecar without boundary fields. With a decision-free history + // the repair has supersession-free evidence: it records a verified-empty boundary, + // keeps the recorded launch identity, and the deferred wake becomes deliverable. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: strippedRunId, + createdAtMs: 1_150, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, strippedRunId); + const repaired = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect(repaired.find((reference) => reference.runId === strippedRunId)).toMatchObject({ + createdAtMs: 1_150, + afterBoundaryMessageId: null, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, strippedRunId) + ).toBe("current"); + + // Once a manual row is the newest decision row, a stripped launch cannot be ordered + // against it: the run may predate the supersession, so repair must refuse and the wake + // must stay deferred rather than resurrect a possibly superseded result. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, do something else", { + timestamp: 1_200, + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: supersededRunId, + createdAtMs: 1_100, + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, supersededRunId); + const refused = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + const refusedReference = refused.find((reference) => reference.runId === supersededRunId); + expect(refusedReference).toBeDefined(); + expect(refusedReference != null && "afterBoundaryMessageId" in refusedReference).toBe(false); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, supersededRunId) + ).toBe("indeterminate"); + + // A reference that still carries its boundary may record a pre-supersession launch: + // repair must not refresh it into the current context. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: anchoredRunId, + createdAtMs: 1_050, + afterBoundaryMessageId: "older-row", + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, anchoredRunId); + const untouched = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect( + untouched.find((reference) => reference.runId === anchoredRunId)?.afterBoundaryMessageId + ).toBe("older-row"); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, anchoredRunId) + ).toBe("not_current"); + + // Unknown run: nothing to repair, nothing recorded. + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, "wfr_unknown"); + const after = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect(after.map((reference) => reference.runId).sort()).toEqual([ + anchoredRunId, + strippedRunId, + supersededRunId, + ]); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("treats an unreadable history as indeterminate, not superseded", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-io-error"; + const runId = "wfr_currentness_io_error"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-io-error", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + const readSpy = spyOn(historyService, "iterateFullHistory").mockResolvedValue( + Err("disk read failed") + ); + try { + // The drain distinguishes a read failure (retain and retry) from supersession + // (tombstone); the boolean view stays fail-safe false for non-destructive callers. + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + // The record path must fail loudly instead of persisting a verified-empty boundary that + // would permanently strand the run's wake after storage recovers. + let boundaryError: unknown; + try { + await workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); + } catch (error: unknown) { + boundaryError = error; + } + expect(String(boundaryError)).toContain("boundary unavailable"); + } finally { + readSpy.mockRestore(); + } + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "current" + ); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("treats an unreadable sidecar as indeterminate, not superseded", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-sidecar-error"; + const runId = "wfr_currentness_sidecar_error"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-sidecar-error", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // The sidecar is the only invocation evidence for kernel-launched runs: an unreadable + // file must read as "cannot know right now", not "no reference", or the drain would + // tombstone the wake on a transient storage fault. + const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); + await fsPromises.rm(sidecarPath); + await fsPromises.mkdir(sidecarPath); + try { + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + } finally { + await fsPromises.rmdir(sidecarPath); + } + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "current" + ); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6393454098..40dbef1620 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3,6 +3,12 @@ import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { EventEmitter } from "events"; import * as path from "path"; import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; +import { + clearAgentWorkflowRunReferences, + readAgentWorkflowRunReferences, + repairAgentWorkflowRunReferenceBoundary, + type AgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; @@ -199,6 +205,7 @@ import { } from "@/node/services/workflows/workflowArchiveAdmission"; import { WORKFLOW_RESULT_METADATA_TYPE, + textContainsWorkflowResultPayload, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowRunCardMessage, @@ -470,6 +477,23 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) ); } +/** + * The terminal-attention drain delivers workflow results as one synthetic user prompt that may + * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands + * between the send's durable acceptance and the outbox delivery mark, restart recovery drains + * the notification again; recognizing the accepted row as consumption is what suppresses the + * replay. Only synthetic rows qualify: a manual user message is a supersession boundary and is + * classified before this check runs. + */ +function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { + if (message.role !== "user" || message.metadata?.synthetic !== true) { + return false; + } + return message.parts.some( + (part) => part.type === "text" && textContainsWorkflowResultPayload(part.text, runId) + ); +} + function isResetBoundaryMessage(message: MuxMessage): boolean { return message.metadata?.contextBoundaryKind === CONTEXT_BOUNDARY_KINDS.RESET; } @@ -10932,38 +10956,132 @@ export class WorkspaceService extends EventEmitter { } async isWorkflowInvocationCurrent(workspaceId: string, runId: string): Promise { - assert(workspaceId.length > 0, "isWorkflowInvocationCurrent requires workspaceId"); - assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); + return (await this.getWorkflowInvocationCurrentness(workspaceId, runId)) === "current"; + } - let current = false; - let foundDecision = false; + /** + * Three-state currentness: "indeterminate" means history/provenance could not be read or + * ordered, so the answer is unknown rather than no. Callers that would permanently drop a + * terminal wake on a negative answer (the terminal-attention drain tombstones notifications) + * must retain and retry on "indeterminate" instead; boolean callers treat it as not-current, + * the pre-existing fail-safe for non-destructive decisions. + */ + async getWorkflowInvocationCurrentness( + workspaceId: string, + runId: string + ): Promise<"current" | "not_current" | "indeterminate"> { + assert(workspaceId.length > 0, "getWorkflowInvocationCurrentness requires workspaceId"); + assert(runId.length > 0, "getWorkflowInvocationCurrentness requires runId"); + + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + if (decision.status === "error") { + return "indeterminate"; + } + if (decision.status === "found" && decision.outcome === "invocation") { + return "current"; + } + + // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave + // no recognizable invocation part in history, so the backward walk above stops at the prior + // real user message (or, after a delivered result, at that consumed terminal message) and + // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the + // agent-workflow-runs sidecar, which snapshots the ID of the decision row that was newest + // at record time: the run is current exactly when that row is still the newest decision row. + // Row identity, not wall-clock ordering, so a backward clock correction can neither strand + // a legitimate wake nor let a pre-supersession reference outrank a newer boundary. For a + // consumed boundary, equality means a background resume/retry was recorded after the prior + // result was delivered. References without a boundary snapshot (pre-upgrade entries, + // record-time read failures) cannot be ordered against the decision row at all and defer + // as indeterminate below. + let references: AgentWorkflowRunReference[]; + try { + references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error: unknown) { + // The sidecar is the only invocation evidence a kernel-launched run has, so an + // unreadable file is "cannot know right now", not "no reference": defer wake decisions + // exactly like an unreadable history. + log.warn("Could not read workflow run references for currentness", { + workspaceId, + runId, + error, + }); + return "indeterminate"; + } + const reference = references.find((candidate) => candidate.runId === runId); + if (decision.status === "none") { + // A decision-free history is current only for a reference whose snapshot verified an + // empty history at record time (null): kernel launches from a new or fully cleared + // workspace (e.g. a heartbeat turn) have no decision row before or after, and their wake + // must still deliver. Every other surviving reference fails safe, because a full clear + // (truncateHistory) removes every row WITHOUT appending a reset boundary while leaving + // the sidecar intact, and a reference pointing at a cleared row, or one without a + // verified snapshot, must not inject a workflow result into the freshly cleared + // conversation. + return reference?.afterBoundaryMessageId === null ? "current" : "not_current"; + } + if (reference == null) { + return "not_current"; + } + if (reference.afterBoundaryMessageId === undefined) { + // No boundary snapshot (pre-upgrade entry or record-time history read failure): row + // identity cannot be verified, and wall-clock ordering is the exact hole the identity + // path exists to close (a backward clock correction would let a pre-supersession + // reference outrank a newer manual turn and deliver its output under that turn's tool + // policy). Defer like an unreadable history: the wake stays pending, a workflow_resume + // re-record repairs provenance, and an explicit resume/await still consumes the run. + return "indeterminate"; + } + if (reference.afterBoundaryMessageId === null) { + // Verified-empty snapshot: a decision row now exists, so it appeared after the record. + return "not_current"; + } + return reference.afterBoundaryMessageId === decision.messageId ? "current" : "not_current"; + } + + /** + * The newest invocation-decision row for this run: a manual user/reset supersession, a + * consumed terminal result for the run, or a direct invocation part. Shared by + * isWorkflowInvocationCurrent and the sidecar record path so both sides of the identity + * comparison classify rows identically. + */ + private async findWorkflowInvocationDecisionRow( + workspaceId: string, + runId: string + ): Promise< + | { + status: "found"; + outcome: "invocation" | "consumed" | "superseded"; + messageId: string; + } + | { status: "none" } + | { status: "error" } + > { + const state: { + found: { + outcome: "invocation" | "consumed" | "superseded"; + messageId: string; + } | null; + } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", (messages) => { for (const message of messages) { - if (isManualUserSupersessionMessage(message)) { - current = false; - foundDecision = true; - return false; - } - if (isResetBoundaryMessage(message)) { - current = false; - foundDecision = true; + if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { + state.found = { outcome: "superseded", messageId: message.id }; return false; } if ( isWorkflowResultContinuationMessage(message, runId) || + isCoalescedWorkflowResultMessage(message, runId) || isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - current = false; - foundDecision = true; + state.found = { outcome: "consumed", messageId: message.id }; return false; } if (isWorkflowInvocationMessage(message, runId)) { - current = true; - foundDecision = true; + state.found = { outcome: "invocation", messageId: message.id }; return false; } } @@ -10976,10 +11094,77 @@ export class WorkspaceService extends EventEmitter { runId, error: historyResult.error, }); - return false; + return { status: "error" }; } + return state.found != null + ? { status: "found", outcome: state.found.outcome, messageId: state.found.messageId } + : { status: "none" }; + } - return foundDecision && current; + /** + * Boundary snapshot for the agent-workflow-runs sidecar: the message ID of the newest + * invocation-decision row for this run, or null when history has none. Recorded at + * background launch/resume so isWorkflowInvocationCurrent can compare row identity instead + * of wall-clock timestamps, which clock corrections can reorder. + */ + async getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise { + assert(workspaceId.length > 0, "getWorkflowInvocationBoundaryMessageId requires workspaceId"); + assert(runId.length > 0, "getWorkflowInvocationBoundaryMessageId requires runId"); + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + // A read failure must not masquerade as a verified-empty history: persisting null would + // permanently fail the run's currentness check even after storage recovers. Throw so the + // record path can distinguish and record a rediscovery-only reference instead. + if (decision.status === "error") { + throw new Error("workflow invocation boundary unavailable: history read failed"); + } + return decision.status === "found" ? decision.messageId : null; + } + + /** + * Re-snapshot the boundary for a run reference that lost it: a pre-boundary build rewrites + * the sidecar with only runId/createdAtMs on any record (upgrade -> downgrade -> upgrade + * strips the field), and a boundaryless reference defers its terminal wake as indeterminate + * until provenance is re-established. Crash recovery calls this before restarting an + * orphaned run, but the repair proceeds only on supersession-free evidence: a decision-free + * history (recorded as a verified-empty boundary) or a newest decision row that belongs to + * this run. A newest manual/reset row is refused: the stripped launch cannot be ordered + * against it by identity, and snapshotting it would resurrect a possibly pre-supersession + * result into the newer conversation (the same reference with a surviving boundary would + * stay not_current). Those wakes stay deferred until an explicit workflow_resume, which + * carries current-context intent. References that still carry a boundary (including + * verified-empty null) are left untouched: refreshing them would forgive manual + * supersessions on every restart. + */ + async repairWorkflowRunReferenceBoundary(workspaceId: string, runId: string): Promise { + assert(workspaceId.length > 0, "repairWorkflowRunReferenceBoundary requires workspaceId"); + assert(runId.length > 0, "repairWorkflowRunReferenceBoundary requires runId"); + const sessionDir = this.config.getSessionDir(workspaceId); + const references = await readAgentWorkflowRunReferences(sessionDir); + const reference = references.find((candidate) => candidate.runId === runId); + if (reference == null || reference.afterBoundaryMessageId !== undefined) { + return; + } + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + if (decision.status === "error") { + throw new Error("workflow invocation boundary unavailable: history read failed"); + } + if (decision.status === "found" && decision.outcome === "superseded") { + return; + } + // Supersession-free evidence only: no decision row at all (verified-empty null), or the + // newest decision row is this run's own invocation/consumed row, which no manual row can + // postdate (the backward walk would have found that manual row first). The write is a + // compare-and-set under the sidecar lock: a full clear landing after the reads above + // deletes the sidecar, and an unconditional record would recreate it with a + // verified-empty boundary, resurrecting the retired pre-clear result as "current". + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir: sessionDir, + runId, + afterBoundaryMessageId: decision.status === "found" ? decision.messageId : null, + }); } /** @@ -12770,6 +12955,22 @@ export class WorkspaceService extends EventEmitter { // admitted afterwards (their content references the discarded context). if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); + // Kernel workflow run references belong to the cleared conversation: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one + // recorded after it, so a surviving reference could inject a pre-clear workflow result + // into the fresh conversation. Retire them immediately after the truncation commits, + // before any later post-clear step that can fail and return early (goal acknowledgment, + // carryover discard), or the stale reference would survive the committed clear. A + // post-clear resume re-records provenance. + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `History was cleared, but stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + + `result into the cleared conversation; retry once the session storage is writable.` + ); + } } // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the