Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
9665489
🤖 fix: deliver terminal wakes for kernel-launched background workflow…
ibetitsmike Aug 27, 2026
f41cbfb
🤖 fix: persist workflow_resume terminal consumption for kernel-nested…
ibetitsmike Aug 27, 2026
a19c5a3
🤖 fix: let newer sidecar records outrank consumed results; clamp futu…
ibetitsmike Aug 27, 2026
30a3437
🤖 fix: reject persisted future-dated sidecar references instead of cl…
ibetitsmike Aug 27, 2026
89263dd
🤖 fix: fail safe after full history clear; dedupe sidecar references …
ibetitsmike Aug 27, 2026
59ce349
🤖 fix: tolerate bounded backward-clock skew in sidecar reference parsing
ibetitsmike Aug 27, 2026
76e6e80
🤖 fix: decide kernel workflow currentness by boundary-row identity, n…
ibetitsmike Aug 27, 2026
046286a
🤖 fix: never persist a boundary snapshot from an unreadable history; …
ibetitsmike Aug 27, 2026
e5de1dd
🤖 fix: migrate pre-snapshot sidecar references through a wall-clock f…
ibetitsmike Aug 27, 2026
60bb82d
🤖 fix: deliver kernel workflow wakes launched from a decision-free hi…
ibetitsmike Aug 27, 2026
a73a354
🤖 fix: harden kernel workflow wake delivery against sidecar faults
ibetitsmike Aug 27, 2026
983d3c5
🤖 fix: retire kernel workflow run references on a full history clear
ibetitsmike Aug 27, 2026
56c789b
🤖 fix: close round-10 wake-delivery gaps: retirement ordering, record…
ibetitsmike Aug 27, 2026
521fba2
🤖 fix: close round-11 gaps: record retry, indeterminate recovery, cle…
ibetitsmike Aug 27, 2026
3eb19a5
🤖 fix: close round-12 lifecycle gaps for kernel workflow wake provenance
ibetitsmike Aug 27, 2026
62dd290
🤖 fix: round-13 sidecar lifecycle hardening: boundary repair, removal…
ibetitsmike Aug 27, 2026
6866c5b
🤖 fix: round-14 provenance integrity: supersede-older retries, genera…
ibetitsmike Aug 27, 2026
e14574d
🤖 fix: restore the caller tool policy on kernel workflow wakes
ibetitsmike Aug 27, 2026
95ca943
🤖 refactor: strip rounds 11-14 retry/repair machinery; keep identity-…
ibetitsmike Aug 28, 2026
d35491c
🤖 fix: resolve the wake's agent identity with an unbounded history walk
ibetitsmike Aug 28, 2026
0792396
🤖 fix: sanitize persisted wake restrictions before restoring them
ibetitsmike Aug 28, 2026
9280614
🤖 fix: persist kernel workflow provenance before the runner can reach…
ibetitsmike Aug 28, 2026
42a596b
🤖 fix: preserve the caller tool policy through on-send compaction fol…
ibetitsmike Aug 28, 2026
c59964f
🤖 fix: record resume provenance only after the dispatch restarts the run
ibetitsmike Aug 28, 2026
2befd06
🤖 fix: restore the strict-agent pin on terminal wakes
ibetitsmike Aug 28, 2026
36bb23c
🤖 fix: forward the object-form strict-agent pin on terminal wakes
ibetitsmike Aug 28, 2026
cf0323c
🤖 fix: stop compaction recovery from clobbering preserved follow-up f…
ibetitsmike Aug 28, 2026
9f57ada
🤖 fix: bind workflow terminal wakes to the initiating agent
ibetitsmike Aug 28, 2026
125aae5
🤖 fix: schema-validate persisted initiating agent IDs
ibetitsmike Aug 28, 2026
e1b1954
🤖 fix: split coalesced workflow wakes by initiating agent
ibetitsmike Aug 28, 2026
cb31308
🤖 fix: isolate wake identity groups and honor synthetic launch pins
ibetitsmike Aug 28, 2026
8e1bcd6
🤖 fix: harden wake provenance writes, reads, and pin pairing
ibetitsmike Aug 28, 2026
d53ac1a
🤖 fix: defer boundaryless workflow references instead of wall-clock o…
ibetitsmike Aug 28, 2026
cd1b200
🤖 fix: split wakes by launch pin and repair downgrade-stripped proven…
ibetitsmike Aug 28, 2026
578d72b
🤖 fix: gate crash-resume provenance repair on supersession-free evidence
ibetitsmike Aug 28, 2026
56171b7
🤖 fix: make crash-resume boundary repair a compare-and-set under the …
ibetitsmike Aug 28, 2026
6776f48
🤖 fix: defer identity-less wakes and wire terminal attention into cra…
ibetitsmike Aug 28, 2026
ce9085a
🤖 fix: retain and retry failed workflow terminal attention enqueues
ibetitsmike Aug 28, 2026
7edfe81
🤖 fix: harden workflow wake recovery (resume reset, repair retry, res…
ibetitsmike Aug 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions src/browser/utils/chatCommands.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1582,6 +1582,32 @@ describe("prepareCompactionMessage", () => {
expect(metadata.parsed.followUpContent?.agentId).toBe("exec");
});

test("compaction recovery keeps the persisted follow-up's restrictions when retry options lack them", () => {
// Retrying a failed compaction passes the already-persisted follow-up together with
// storage-derived send options, which never carry a caller toolPolicy. The preserved
// restrictions must survive that recomposition or the recovered follow-up resumes with
// unrestricted caller tools.
const recoveredFollowUp = {
text: "Keep building",
model: "openai:gpt-4o",
agentId: "code",
toolPolicy: [{ regex_match: "^bash$", action: "disable" as const }],
disableWorkspaceAgents: true,
};
const { metadata } = prepareCompactionMessage({
workspaceId: "ws-1",
followUpContent: recoveredFollowUp,
sendMessageOptions: { model: "anthropic:claude-sonnet-4-6", agentId: "exec" },
});

expectCompactionMetadata(metadata);
expect(metadata.parsed.followUpContent?.toolPolicy).toEqual(recoveredFollowUp.toolPolicy);
expect(metadata.parsed.followUpContent?.disableWorkspaceAgents).toBe(true);
// Existing model/agentId still win over the retry-time options.
expect(metadata.parsed.followUpContent?.model).toBe("openai:gpt-4o");
expect(metadata.parsed.followUpContent?.agentId).toBe("code");
});

test("does not create followUpContent when no text or images provided", () => {
const sendMessageOptions = createBaseOptions();
const { metadata } = prepareCompactionMessage({
Expand Down
37 changes: 28 additions & 9 deletions src/common/types/message.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ type PreservedSendOptions = Pick<
| "providerOptions"
| "experiments"
| "disableWorkspaceAgents"
| "toolPolicy"
| "strictAgentResolution"
| "allowAgentSetGoal"
| "skipAiSettingsPersistence"
Expand All @@ -73,21 +74,39 @@ type PreservedSendOptions = Pick<
* Use this helper to avoid duplicating the field list when building CompactionFollowUpRequest.
*/
export function pickPreservedSendOptions(options: SendMessageOptions): PreservedSendOptions {
// Unset fields are OMITTED, not emitted as explicit undefined: compaction recovery spreads
// this pick over an already-persisted follow-up, and an undefined key would clobber the
// original preserved value (e.g. a restricted turn's toolPolicy) instead of leaving it.
return {
thinkingLevel: options.thinkingLevel,
reasoningMode: options.reasoningMode,
additionalSystemInstructions: options.additionalSystemInstructions,
providerOptions: options.providerOptions,
...(options.thinkingLevel !== undefined ? { thinkingLevel: options.thinkingLevel } : {}),
...(options.reasoningMode !== undefined ? { reasoningMode: options.reasoningMode } : {}),
...(options.additionalSystemInstructions !== undefined
? { additionalSystemInstructions: options.additionalSystemInstructions }
: {}),
...(options.providerOptions !== undefined ? { providerOptions: options.providerOptions } : {}),
// Downgrade-compat (see withLegacyPtcExclusiveMirror): preserved options
// can persist across restarts and build versions.
experiments: withLegacyPtcExclusiveMirror(options.experiments),
disableWorkspaceAgents: options.disableWorkspaceAgents,
...(options.experiments !== undefined
? { experiments: withLegacyPtcExclusiveMirror(options.experiments) }
: {}),
...(options.disableWorkspaceAgents !== undefined
? { disableWorkspaceAgents: options.disableWorkspaceAgents }
: {}),
// Security: a restricted turn (including a terminal-wake send restoring the caller's
// policy) that triggers on-send compaction must not redispatch its follow-up allow-all.
...(options.toolPolicy !== undefined ? { toolPolicy: options.toolPolicy } : {}),
// Delegated turns with explicit agent overrides must stay loud across the
// compaction replay too — dropping this would let the follow-up silently
// fall back to exec if the agent vanished in the meantime.
strictAgentResolution: options.strictAgentResolution,
allowAgentSetGoal: options.allowAgentSetGoal,
skipAiSettingsPersistence: options.skipAiSettingsPersistence,
...(options.strictAgentResolution !== undefined
? { strictAgentResolution: options.strictAgentResolution }
: {}),
...(options.allowAgentSetGoal !== undefined
? { allowAgentSetGoal: options.allowAgentSetGoal }
: {}),
...(options.skipAiSettingsPersistence !== undefined
? { skipAiSettingsPersistence: options.skipAiSettingsPersistence }
: {}),
};
}

Expand Down
6 changes: 5 additions & 1 deletion src/common/utils/tools/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { xai } from "@ai-sdk/xai";
import { type LanguageModel, type Tool } from "ai";
import type { LanguageModelV2Usage } from "@ai-sdk/provider";
import type { MuxProviderOptions } from "@/common/types/providerOptions";
import type { ProvidersConfigMap } from "@/common/orpc/types";
import type { ProvidersConfigMap, SendMessageOptions } from "@/common/orpc/types";
import { isGrokFrontierModel } from "@/common/types/thinking";
import type { BackgroundWorkAttentionPolicy } from "@/common/types/backgroundWorkAttention";
import { cloneToolPreservingDescriptors } from "@/common/utils/tools/cloneToolPreservingDescriptors";
Expand Down Expand Up @@ -184,6 +184,10 @@ export interface ToolConfiguration {
workspaceSessionDir?: string;
/** Workspace ID for tracking background processes and plan storage */
workspaceId?: string;
/** Resolved agent identity of the turn executing the tools (workflow wake provenance). */
agentId?: string;
/** The turn's strict-agent pin, persisted with workflow run provenance so wakes re-pin the launch agent. */
strictAgentResolution?: SendMessageOptions["strictAgentResolution"];
/** Pre-resolved mux-managed resource scope (global ~/.xum vs project root). */
xumScope?: XumToolScope;
/** Memory service for the memory tool (present only when the memory experiment is enabled). */
Expand Down
35 changes: 35 additions & 0 deletions src/common/utils/workflowRunMessages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,41 @@ export function buildWorkflowResultContextMessage(input: {
].join("\n\n");
}

/**
* Recognize this run's result payload inside a coalesced terminal-attention prompt from the
* builder's own output format. The drain's synthetic user row can coalesce several runs into
* one message and carries no workflow-result metadata, so currentness checks must read the
* consumption evidence out of the text: each payload block is parsed back and matched on the
* exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted
* inside another run's report cannot count as consumption.
*/
export function textContainsWorkflowResultPayload(text: string, runId: string): boolean {
assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required");
if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) {
return false;
}
const blockPattern = new RegExp(
`<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n</${WORKFLOW_RESULT_XML_TAG}>`,
"g"
);
for (const match of text.matchAll(blockPattern)) {
let payload: unknown;
try {
payload = JSON.parse(match[1] ?? "");
} catch {
continue;
}
if (!isRecordValue(payload)) {
continue;
}
const workflow = payload.workflow;
if (isRecordValue(workflow) && workflow.runId === runId) {
return true;
}
}
return false;
}

export interface WorkflowRunCardInput {
scriptPath?: string;
scriptSource?: string;
Expand Down
80 changes: 79 additions & 1 deletion src/node/orpc/router.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,11 @@ describe("router workflow routes", () => {
getWorkflowContinuationSendOptions: mock(() => null),
sendMessage: mock(async () => ({ success: true, data: undefined })),
},
taskService: {},
// Nonterminal run status changes reset any stale terminal notification; the stub keeps
// that call observable without wiring a full TaskService.
taskService: {
resetWorkflowRunTerminalAttention: mock(async () => undefined),
},
experimentsService: {
isExperimentEnabled: mock(() => options.enabled),
},
Expand Down Expand Up @@ -951,6 +955,80 @@ export default function workflow() { return { reportMarkdown: "should not run" }
expect(result.result).toBeNull();
await waitForRouterWorkflowStatus(client, "workspace-1", result.runId, "completed");
});

test("crash-resumed background runs enqueue terminal attention on settle", async () => {
const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") });
await runStore.createRun({
id: "wfr_crash_wake",
workspaceId: "workspace-1",
workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true },
source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n",
args: {},
attentionPolicy: "notify_on_terminal",
now: "2026-05-29T00:00:00.000Z",
});
// Orphaned by a crash: durable status says running, but no live runner.
await runStore.appendStatus("wfr_crash_wake", "running", "2026-05-29T00:00:01.000Z");

const enqueueWorkflowRunTerminalAttention = mock(async () => undefined);
const context = createContext({ enabled: true });
(context as unknown as Record<string, unknown>).taskService = {
enqueueWorkflowRunTerminalAttention,
resetWorkflowRunTerminalAttention: mock(async () => undefined),
};
(
context.workspaceService as unknown as Record<string, unknown>
).repairWorkflowRunReferenceBoundary = mock(async () => undefined);
const client = createRouterClient(router(), { context });

// A read path triggers crash recovery; the resumed run's settle must land in the
// terminal-attention outbox instead of waiting for the next restart's sweep.
await client.workflows.listRuns({ workspaceId: "workspace-1" });
await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_crash_wake", "completed");
const deadline = Date.now() + 5_000;
while (enqueueWorkflowRunTerminalAttention.mock.calls.length === 0 && Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, 25));
}
expect(enqueueWorkflowRunTerminalAttention).toHaveBeenCalledWith({
ownerWorkspaceId: "workspace-1",
runId: "wfr_crash_wake",
status: "completed",
});
});

test("router-managed resume resets a stale terminal notification before restart", async () => {
const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") });
await runStore.createRun({
id: "wfr_resume_reset",
workspaceId: "workspace-1",
workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true },
source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n",
args: {},
attentionPolicy: "notify_on_terminal",
now: "2026-05-29T00:00:00.000Z",
});

const context = createContext({ enabled: true });
const resetWorkflowRunTerminalAttention = (
context.taskService as unknown as {
resetWorkflowRunTerminalAttention: ReturnType<typeof mock>;
}
).resetWorkflowRunTerminalAttention;
const client = createRouterClient(router(), { context });

await client.workflows.interrupt({ workspaceId: "workspace-1", runId: "wfr_resume_reset" });
resetWorkflowRunTerminalAttention.mockClear();

// The prior run's notification survives under the stable workflow_run:<runId> id as
// delivered/superseded; without a reset on restart, enqueueIfAbsent preserves that
// record and the resumed run's terminal wake is silently dropped.
await client.workflows.resume({ workspaceId: "workspace-1", runId: "wfr_resume_reset" });
expect(resetWorkflowRunTerminalAttention).toHaveBeenCalledWith({
ownerWorkspaceId: "workspace-1",
runId: "wfr_resume_reset",
});
await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_resume_reset", "completed");
});
});

describe("router config.saveConfig", () => {
Expand Down
38 changes: 34 additions & 4 deletions src/node/orpc/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ import * as path from "node:path";

import type { DevToolsEvent } from "@/common/types/devtools";
import type { WorkflowRunStreamEvent } from "@/common/types/workflow";
import { isTerminalWorkflowRunStatus } from "@/common/types/workflow";
import type { WorkflowRunLivenessEntry } from "@/common/orpc/schemas/api";
import type { MuxMessage } from "@/common/types/message";
import { coerceThinkingLevel } from "@/common/types/thinking";
Expand Down Expand Up @@ -583,10 +584,39 @@ export async function resolveWorkflowContext(
includeAgentPlugins,
skillStorageContext,
}),
onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event),
...(options.onBackgroundRunTerminal != null
? { onBackgroundRunTerminal: options.onBackgroundRunTerminal }
: {}),
onRunStatusChanged: async (event) => {
// Router-managed restarts (resume / retry / crash recovery) must clear a prior
// delivered or superseded notification when the run leaves terminal state, or
// enqueueIfAbsent would preserve the stale record and silently drop the resumed
// run's next terminal wake (mirrors the AIService-owned service).
if (!isTerminalWorkflowRunStatus(event.status)) {
await context.taskService.resetWorkflowRunTerminalAttention({
ownerWorkspaceId: event.workspaceId,
runId: event.runId,
});
Comment on lines +592 to +596

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Propagate terminal-attention reset failures

For router-managed resumes or retries whose previous workflow notification is already delivered or superseded, a transient failure deleting that notification rejects this callback, but WorkflowService.notifyRunStatusChanged catches the error and starts the runner anyway. When the run settles, enqueueIfAbsent preserves the stale terminal status, so neither the live callback nor startup recovery creates the new wake; make this reset a required durable precondition for restarting, or retain and retry it before terminal settlement.

AGENTS.md reference: AGENTS.md:L112-L112

Useful? React with 👍 / 👎.

}
await context.workspaceService.emitWorkflowRunActivity(event);
},
onRunCrashResumed: (event) =>
context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId),
Comment thread
ibetitsmike marked this conversation as resolved.
// Read paths (listRuns / stream subscribe) create services purely to observe runs, but
// crash recovery can resume an orphaned background run on them: without a terminal
// callback the settled run would enqueue no terminal attention until the next restart's
// sweep. Default to the standard outbox enqueue (idempotent via enqueueIfAbsent);
// explicit callbacks (slash-command continuations, retry) keep their custom behavior.
onBackgroundRunTerminal:
options.onBackgroundRunTerminal ??
(async (event) => {
Comment thread
ibetitsmike marked this conversation as resolved.
// Nested runs surface through their parent workflow, not their own wake.
if (event.run.parentWorkflow != null) {
return;
}
await context.taskService.enqueueWorkflowRunTerminalAttention({
ownerWorkspaceId: workspaceId,
runId: event.runId,
status: event.status,
});
Comment thread
ibetitsmike marked this conversation as resolved.
}),
getCurrentProjectTrusted: resolveWorkflowProjectTrusted,
runnerId: `workflow-runner:${workspaceId}`,
}),
Expand Down
11 changes: 11 additions & 0 deletions src/node/services/agentSession.autoCompaction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,12 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => {
getThreshold: mock(() => 0.85),
} as unknown as CompactionMonitor;

const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }];
const result = await session.sendMessage("please inspect @foo.ts", {
model: "openai:gpt-4o",
agentId: "exec",
disableWorkspaceAgents: true,
toolPolicy: restrictedPolicy,
});

expect(result.success).toBe(true);
Expand All @@ -120,6 +122,15 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => {
);
expect(persistedCompactionMessage).toBeDefined();
expect(persistedCompactionMessage?.metadata?.disableWorkspaceAgents).toBe(true);
// The durable follow-up must keep the caller's restrictions: the redispatched turn
// reconstructs its options from this persisted request, and dropping the policy there
// would resume the conversation allow-all after compaction.
const followUpContent =
persistedCompactionMessage?.metadata?.muxMetadata?.type === "compaction-request"
? persistedCompactionMessage.metadata.muxMetadata.parsed.followUpContent
: undefined;
expect(followUpContent?.toolPolicy).toEqual(restrictedPolicy);
expect(followUpContent?.disableWorkspaceAgents).toBe(true);

const emittedSnapshot = events.some(
(message) =>
Expand Down
18 changes: 18 additions & 0 deletions src/node/services/agentSession.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import {
SendMessageOptionsSchema,
SkillNameSchema,
} from "@/common/orpc/schemas";
import { ToolPolicySchema } from "@/common/orpc/schemas/stream";
import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds";
import {
buildStreamErrorEventData,
Expand Down Expand Up @@ -7200,6 +7201,23 @@ export class AgentSession {
experiments: aliasLegacyPtcExclusive(followUp.experiments),
allowAgentSetGoal: followUp.allowAgentSetGoal,
disableWorkspaceAgents: followUp.disableWorkspaceAgents,
// Same raw JSON boundary: a persisted follow-up may carry a malformed toolPolicy, and
// restoring it unvalidated would throw during resolution. Invalid values are dropped
// like any corrupt persisted policy (self-healing doctrine); a restricted turn's
// follow-up must otherwise keep its policy instead of redispatching allow-all.
...(() => {
if (followUp.toolPolicy == null) {
return {};
}
const parsed = ToolPolicySchema.safeParse(followUp.toolPolicy);
if (!parsed.success) {
log.warn("Ignoring malformed persisted toolPolicy on compaction follow-up", {
workspaceId: this.workspaceId,
});
return {};
}
return { toolPolicy: parsed.data };
})(),
// Explicit-agent turns stay loud on the resumed turn too: the requested agent
// may have been removed/hidden/disabled while compaction ran.
strictAgentResolution: followUp.strictAgentResolution,
Expand Down
Loading