From 2963df3a3ccc1187d3e9d1f2473da3f0f842c0ae Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 24 Aug 2026 08:31:11 -0400 Subject: [PATCH 1/8] fix(agent): support Codex evolution schemas - Keep reflector and curator schemas within the Codex-supported JSON Schema subset - Preserve duplicate rejection through application-side response validation --- packages/evolution/src/curator.ts | 27 +++++++++-------- packages/evolution/src/reflector.ts | 7 +---- packages/evolution/test/curator.test.ts | 12 ++++++++ packages/evolution/test/reflector.test.ts | 21 ++++++++++++++ .../test/structured-output-schema.ts | 29 +++++++++++++++++++ 5 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 packages/evolution/test/structured-output-schema.ts diff --git a/packages/evolution/src/curator.ts b/packages/evolution/src/curator.ts index 490223d18..3c3017874 100644 --- a/packages/evolution/src/curator.ts +++ b/packages/evolution/src/curator.ts @@ -53,11 +53,10 @@ interface CuratorSchemaReferences { } export function createCuratorOutputSchema(references: CuratorSchemaReferences): JsonObject { - const stringArray = { type: "array", uniqueItems: true, items: { type: "string", minLength: 1 } }; + const stringArray = { type: "array", items: { type: "string", minLength: 1 } }; const referenceArraySchema = (values: readonly string[], minItems = 0): JsonObject => ({ type: "array", minItems, - uniqueItems: true, items: { type: "string", enum: values }, }); return { @@ -78,12 +77,12 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): causalAgentBehavior: { type: "string", minLength: 1 }, futureBehaviorChange: { type: "string", minLength: 1 }, verifiableExpectedResult: { type: "string", minLength: 1 }, - observerDocumentationOnly: { const: false }, - telemetryFormatOnly: { const: false }, + observerDocumentationOnly: { enum: [false] }, + telemetryFormatOnly: { enum: [false] }, }, }, strengthening: { - oneOf: [ + anyOf: [ { type: "null" }, { type: "object", @@ -108,19 +107,19 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): }, }, target: { - oneOf: [ + anyOf: [ { type: "object", additionalProperties: false, required: ["kind"], - properties: { kind: { const: "evolution-memory" } }, + properties: { kind: { enum: ["evolution-memory"] } }, }, { type: "object", additionalProperties: false, required: ["kind", "skillName"], properties: { - kind: { const: "skill" }, + kind: { enum: ["skill"] }, skillName: { type: "string", minLength: 1 }, }, }, @@ -129,7 +128,7 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): additionalProperties: false, required: ["kind", "sourcePath"], properties: { - kind: { const: "orchestration-source" }, + kind: { enum: ["orchestration-source"] }, sourcePath: { type: "string", minLength: 1 }, }, }, @@ -168,13 +167,13 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): }, }, exactDelta: { - oneOf: [ + anyOf: [ { type: "object", additionalProperties: false, required: ["operation", "destination", "content", "intendedScope"], properties: { - operation: { const: "ADD" }, + operation: { enum: ["ADD"] }, destination: { $ref: "#/$defs/destination" }, content: { type: "string", @@ -189,7 +188,7 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): additionalProperties: false, required: ["operation", "memoryRef", "reason"], properties: { - operation: { const: "REMOVE" }, + operation: { enum: ["REMOVE"] }, memoryRef: { type: "string", enum: references.memoryRefs }, reason: { type: "string", minLength: 1 }, }, @@ -199,7 +198,7 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): additionalProperties: false, required: ["operation", "removal", "addition"], properties: { - operation: { const: "REPLACE" }, + operation: { enum: ["REPLACE"] }, removal: { $ref: "#/$defs/removal" }, addition: { $ref: "#/$defs/addition" }, }, @@ -255,7 +254,7 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): lessonRefs: referenceArraySchema(references.lessonRefs, 1), relevance: { $ref: "#/$defs/relevance" }, relationship: { $ref: "#/$defs/relationship" }, - proposal: { oneOf: [{ $ref: "#/$defs/proposal" }, { type: "null" }] }, + proposal: { anyOf: [{ $ref: "#/$defs/proposal" }, { type: "null" }] }, }, }, }, diff --git a/packages/evolution/src/reflector.ts b/packages/evolution/src/reflector.ts index ed0a42d79..af4d1d994 100644 --- a/packages/evolution/src/reflector.ts +++ b/packages/evolution/src/reflector.ts @@ -299,12 +299,10 @@ function reflectorOutputSchema( supportingEvidenceRefs: { type: "array", minItems: 1, - uniqueItems: true, items: { type: "string", enum: references }, }, counterevidenceRefs: { type: "array", - uniqueItems: true, items: { type: "string", enum: references }, }, } @@ -312,12 +310,10 @@ function reflectorOutputSchema( sourceLessonRefs: { type: "array", minItems: 1, - uniqueItems: true, items: { type: "string", enum: references }, }, counterLessonRefs: { type: "array", - uniqueItems: true, items: { type: "string", enum: references }, }, }; @@ -348,12 +344,11 @@ function reflectorOutputSchema( properties: { kind: { enum: [...LESSON_KINDS] }, summary: { type: "string", minLength: 1 }, - preconditions: { type: "array", uniqueItems: true, items: { type: "string", minLength: 1 } }, + preconditions: { type: "array", items: { type: "string", minLength: 1 } }, observedOutcome: { type: "string", minLength: 1 }, ...referenceProperties, applicabilityBoundaries: { type: "array", - uniqueItems: true, items: { type: "string", minLength: 1 }, }, uncertainty: { type: ["string", "null"] }, diff --git a/packages/evolution/test/curator.test.ts b/packages/evolution/test/curator.test.ts index c92a0da16..4fac1d906 100644 --- a/packages/evolution/test/curator.test.ts +++ b/packages/evolution/test/curator.test.ts @@ -16,6 +16,7 @@ import { type ReflectionResult, StructuredDeltaCurator, } from "../src/index.ts"; +import { findUnsupportedCodexSchemaKeywords } from "./structured-output-schema.ts"; const FIRST_HARNESS_VERSION = `sha256:${"1".repeat(64)}`; const SECOND_HARNESS_VERSION = `sha256:${"2".repeat(64)}`; @@ -306,6 +307,17 @@ function decision( } describe("structured delta curator", () => { + it("publishes a Codex-compatible response schema", async () => { + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + + await curator.curate( + curatorInput((request) => { + expect(findUnsupportedCodexSchemaKeywords(request.outputSchema)).toEqual([]); + return Promise.resolve(response([])); + }), + ); + }); + it("emits a provenance-rich ADD from final lessons and ordered snapshots", async () => { const capturedInputs: JsonObject[] = []; const toolOutputs: JsonObject[] = []; diff --git a/packages/evolution/test/reflector.test.ts b/packages/evolution/test/reflector.test.ts index 34e012dba..cbce1fa98 100644 --- a/packages/evolution/test/reflector.test.ts +++ b/packages/evolution/test/reflector.test.ts @@ -16,6 +16,7 @@ import { RecursiveReflector, type ReflectorInput, } from "../src/index.ts"; +import { findUnsupportedCodexSchemaKeywords } from "./structured-output-schema.ts"; const temporaryDirectories: string[] = []; const HARNESS_VERSION = `sha256:${"1".repeat(64)}`; @@ -258,6 +259,26 @@ function jsonObject(value: unknown, description: string): JsonObject { } describe("recursive reflector", () => { + it("publishes Codex-compatible response schemas", async () => { + const { reflector } = await reflectorFixture(); + const schemas: JsonObject[] = []; + const input = reflectInput({ + async execute(request) { + if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + schemas.push(request.outputSchema); + return { + output: { complete: true, lessons: [lessonForRequest(request.input)] }, + model: { provider: "faux", model: "fixture" }, + }; + }, + }); + + await reflector.reflect(input); + + expect(schemas.length).toBeGreaterThan(1); + for (const schema of schemas) expect(findUnsupportedCodexSchemaKeywords(schema)).toEqual([]); + }); + it("limits each leaf to three high-impact lessons", async () => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( diff --git a/packages/evolution/test/structured-output-schema.ts b/packages/evolution/test/structured-output-schema.ts new file mode 100644 index 000000000..268a95a80 --- /dev/null +++ b/packages/evolution/test/structured-output-schema.ts @@ -0,0 +1,29 @@ +const UNSUPPORTED_CODEX_SCHEMA_KEYWORDS = new Set([ + "allOf", + "const", + "dependentRequired", + "dependentSchemas", + "else", + "if", + "not", + "oneOf", + "then", + "uniqueItems", +]); + +export function findUnsupportedCodexSchemaKeywords(schema: unknown): readonly string[] { + const found = new Set(); + const visit = (value: unknown): void => { + if (Array.isArray(value)) { + for (const item of value) visit(item); + return; + } + if (typeof value !== "object" || value === null) return; + for (const [key, child] of Object.entries(value)) { + if (UNSUPPORTED_CODEX_SCHEMA_KEYWORDS.has(key)) found.add(key); + visit(child); + } + }; + visit(schema); + return [...found].sort(); +} From d62362545d646f3900048c2edeffa9fe06ebe4dc Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 24 Aug 2026 18:10:33 -0400 Subject: [PATCH 2/8] feat(agent): preserve evolution response transcripts - Validate response-tool and JSON-text submissions before completing an executor invocation. - Retain tool results across corrections and transport retries while isolating node conversations. --- .../src/core/evolution-runtime.ts | 65 ++++- .../test/evolution-runtime.test.ts | 229 +++++++++++++++++- packages/evolution/src/ports.ts | 1 + 3 files changed, 276 insertions(+), 19 deletions(-) diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index ae93d7134..6d41aca01 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -7,6 +7,7 @@ import { type AssistantMessage, contentText, type Message, type Model, type Tool import type { HarnessStateStore, HarnessVersion } from "@earendil-works/diffex-harness-state"; import lockfile from "proper-lockfile"; import type { TSchema } from "typebox"; +import { Check, Errors } from "typebox/value"; import { classifyEvolutionEvidence, computeEvolutionContentDigest, @@ -15,6 +16,7 @@ import { EvolutionControlPlane, EvolutionControlService, type EvolutionEvidenceCatalog, + EvolutionExecutionError, type EvolutionHarnessVersionInfo, type EvolutionJobDetail, type EvolutionJobRunner, @@ -127,6 +129,25 @@ function evolutionToolError(error: unknown): JsonObject { }; } +function validateEvolutionOutput(request: EvolutionModelRequest, output: JsonObject): string | null { + const schema = request.outputSchema as TSchema; + if (!Check(schema, output)) { + const issue = [...Errors(schema, output)][0]; + return issue?.message ?? "Response does not match the required schema"; + } + if (request.stage !== "reflector" || request.validateOutput === undefined) return null; + try { + request.validateOutput(output); + return null; + } catch (error) { + return error instanceof Error ? error.message : "Response validation failed"; + } +} + +function correctionMessage(reason: string): string { + return `The submitted response is invalid: ${reason}. Correct it and submit it again through ${RESPONSE_TOOL_NAME}.`; +} + function abortableDelay(delayMs: number, signal: AbortSignal): Promise { if (signal.aborted) return Promise.reject(signal.reason); return new Promise((resolveDelay, reject) => { @@ -200,6 +221,7 @@ export class CodingAgentEvolutionModelExecutor implements EvolutionModelExecutor timestamp: Date.now(), }, ]; + let lastCorrection: string | null = null; for (let round = 0; round < MAX_MODEL_ROUNDS; round += 1) { request.signal?.throwIfAborted(); const waitStartedAt = performance.now(); @@ -247,11 +269,30 @@ export class CodingAgentEvolutionModelExecutor implements EvolutionModelExecutor } for (const call of calls) { if (call.name === RESPONSE_TOOL_NAME) { - return { - output: call.arguments as JsonObject, - submissionMethod: "response-tool", - model: { provider: model.provider, model: model.id }, - }; + const output = call.arguments as JsonObject; + const validationError = validateEvolutionOutput(request, output); + if (validationError === null) { + return { + output, + submissionMethod: "response-tool", + model: { provider: model.provider, model: model.id }, + }; + } + lastCorrection = validationError; + messages.push({ + role: "toolResult", + toolCallId: call.id, + toolName: call.name, + content: [ + { + type: "text", + text: JSON.stringify(evolutionToolError(new Error(correctionMessage(validationError)))), + }, + ], + isError: true, + timestamp: Date.now(), + }); + continue; } const tool = evolutionTools.find((candidate) => candidate.name === call.name); let output: JsonObject; @@ -280,20 +321,30 @@ export class CodingAgentEvolutionModelExecutor implements EvolutionModelExecutor } if (calls.length === 0) { const parsed = parseJsonObject(contentText(assistant.content)); - if (parsed) { + const validationError = + parsed === null ? "Response is not a JSON object" : validateEvolutionOutput(request, parsed); + if (parsed !== null && validationError === null) { return { output: parsed, submissionMethod: "json-text", model: { provider: model.provider, model: model.id }, }; } + lastCorrection = validationError; messages.push({ role: "user", - content: `Return the result through ${RESPONSE_TOOL_NAME}.`, + content: correctionMessage(validationError ?? "Response validation failed"), timestamp: Date.now(), }); } } + if (lastCorrection !== null) { + throw new EvolutionExecutionError(`Evolution model did not submit a valid response: ${lastCorrection}`, { + reasonCode: "reflection-invalid-response", + retryable: false, + diagnostics: { lastCorrection }, + }); + } throw new Error(`Evolution model exceeded ${MAX_MODEL_ROUNDS} tool rounds`); } diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index e013f729e..5f1ade0d4 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -1,7 +1,7 @@ import { existsSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; -import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/diffex-ai"; +import { fauxAssistantMessage, fauxToolCall, type Message } from "@earendil-works/diffex-ai"; import { HarnessStateStore } from "@earendil-works/diffex-harness-state"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -20,6 +20,7 @@ import { TraceEvidenceCatalog, } from "../src/core/evolution-runtime.ts"; import type { InvocationTrace } from "../src/core/invocation-trace.ts"; +import type { ModelRuntime } from "../src/core/model-runtime.ts"; import { createHarness, type Harness } from "./suite/harness.ts"; function materializedInvocation(traceId: string, sessionPath: string): MaterializedInvocation { @@ -427,6 +428,107 @@ describe("coding-agent evolution runtime", () => { }); }); + it("corrects an invalid response-tool submission within the active transcript", async () => { + const harness = await createHarness(); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage(fauxToolCall("submit_evolution_response", { complete: false, lessons: [] }), { + stopReason: "toolUse", + }), + (context) => { + expect(context.messages).toHaveLength(3); + expect(context.messages[1]).toMatchObject({ + role: "assistant", + content: [expect.objectContaining({ type: "toolCall", name: "submit_evolution_response" })], + }); + expect(context.messages[2]).toMatchObject({ + role: "toolResult", + toolName: "submit_evolution_response", + isError: true, + }); + return fauxAssistantMessage(fauxToolCall("submit_evolution_response", { complete: true, lessons: [] }), { + stopReason: "toolUse", + }); + }, + ]); + const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, { + async waitUntilBackgroundAllowed() {}, + }); + const model = harness.getModel(); + let validations = 0; + + await expect( + executor.execute({ + stage: "reflector", + model: { + provider: model.provider, + model: model.id, + configurationDigest: `sha256:${"1".repeat(64)}`, + }, + promptVersion: "reflector-v1", + idempotencyKey: "response-tool-correction", + input: {}, + outputSchema: { + type: "object", + required: ["complete", "lessons"], + properties: { complete: { enum: [true] }, lessons: { type: "array" } }, + }, + tools: [], + validateOutput() { + validations += 1; + }, + }), + ).resolves.toMatchObject({ + output: { complete: true, lessons: [] }, + submissionMethod: "response-tool", + }); + expect(validations).toBe(1); + }); + + it("isolates transcripts between node executor invocations", async () => { + const harness = await createHarness(); + harnesses.push(harness); + const initialPrompts: string[] = []; + harness.setResponses([ + (context) => { + expect(context.messages).toHaveLength(1); + initialPrompts.push(String(context.messages[0]?.content)); + return fauxAssistantMessage(fauxToolCall("submit_evolution_response", { complete: true, lessons: [] }), { + stopReason: "toolUse", + }); + }, + (context) => { + expect(context.messages).toHaveLength(1); + initialPrompts.push(String(context.messages[0]?.content)); + return fauxAssistantMessage(fauxToolCall("submit_evolution_response", { complete: true, lessons: [] }), { + stopReason: "toolUse", + }); + }, + ]); + const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, { + async waitUntilBackgroundAllowed() {}, + }); + const model = harness.getModel(); + const request = (node: string): EvolutionModelRequest => ({ + stage: "reflector", + model: { + provider: model.provider, + model: model.id, + configurationDigest: `sha256:${"1".repeat(64)}`, + }, + promptVersion: "reflector-v1", + idempotencyKey: `isolation-${node}`, + input: { node }, + outputSchema: { type: "object" }, + tools: [], + }); + + await executor.execute(request("first")); + await executor.execute(request("second")); + + expect(initialPrompts).toEqual(['{"node":"first"}', '{"node":"second"}']); + }); + it("stops the internal evolution tool loop after ten provider rounds", async () => { const harness = await createHarness(); harnesses.push(harness); @@ -460,30 +562,117 @@ describe("coding-agent evolution runtime", () => { expect(providerCalls).toBe(10); }); - it("returns a denied evidence-tool request to the model for correction", async () => { + it("retries a timed-out transport with the unchanged active transcript", async () => { + vi.useFakeTimers(); + const harness = await createHarness(); + harnesses.push(harness); + const model = harness.getModel(); + const response = fauxAssistantMessage( + fauxToolCall("submit_evolution_response", { complete: true, lessons: [] }), + { stopReason: "toolUse" }, + ); + const transcripts: Message[][] = []; + let calls = 0; + const modelRuntime = { + getModel() { + return model; + }, + streamSimple(_model: unknown, context: { messages: Message[] }, options?: { signal?: AbortSignal }) { + calls += 1; + transcripts.push(structuredClone(context.messages)); + if (calls === 1) { + return { + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((_resolve, reject) => { + options?.signal?.addEventListener("abort", () => reject(options.signal?.reason), { + once: true, + }); + }), + }; + }, + }; + } + return { + [Symbol.asyncIterator]() { + return { + next: async () => ({ done: false as const, value: { type: "done" as const, message: response } }), + }; + }, + }; + }, + } as unknown as ModelRuntime; + const executor = new CodingAgentEvolutionModelExecutor(harness.session, modelRuntime, { + async waitUntilBackgroundAllowed() {}, + }); + + try { + const execution = executor.execute({ + stage: "reflector", + model: { + provider: model.provider, + model: model.id, + configurationDigest: `sha256:${"1".repeat(64)}`, + }, + promptVersion: "reflector-v1", + idempotencyKey: "transport-retry", + input: { node: "isolated" }, + outputSchema: { type: "object" }, + tools: [], + }); + await vi.advanceTimersByTimeAsync(60_001); + + await expect(execution).resolves.toMatchObject({ output: { complete: true, lessons: [] } }); + expect(calls).toBe(2); + expect(transcripts[1]).toEqual(transcripts[0]); + expect(transcripts[0]).toHaveLength(1); + } finally { + vi.useRealTimers(); + } + }); + + it("preserves successful evidence, a denial, and adjacent inspection in one transcript", async () => { const harness = await createHarness(); harnesses.push(harness); harness.setResponses([ - fauxAssistantMessage(fauxToolCall("read_slice", { documentRef: "trace-id", startOrdinal: 0, endOrdinal: 1 }), { - stopReason: "toolUse", - }), + fauxAssistantMessage(fauxToolCall("search_evidence", { query: "adjacent" }), { stopReason: "toolUse" }), (context) => { - const toolResult = context.messages.find((message) => message.role === "toolResult"); - expect(toolResult).toMatchObject({ + expect(context.messages).toHaveLength(3); + expect(context.messages[2]).toMatchObject({ role: "toolResult", - toolName: "read_slice", - isError: true, + toolName: "search_evidence", + isError: false, }); + return fauxAssistantMessage( + fauxToolCall("read_slice", { documentRef: "trace-id", startOrdinal: 0, endOrdinal: 1 }), + { stopReason: "toolUse" }, + ); + }, + (context) => { + expect(context.messages).toHaveLength(5); + const denial = context.messages[4]; + expect(denial).toMatchObject({ role: "toolResult", toolName: "read_slice", isError: true }); const errorText = - toolResult?.role === "toolResult" - ? toolResult.content + denial?.role === "toolResult" + ? denial.content .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join("\n") : ""; expect(errorText).not.toContain("trace-id"); expect(errorText).toContain("D1"); - expect(errorText).toContain("is not permitted"); + return fauxAssistantMessage(fauxToolCall("inspect_adjacent", { sliceRef: "S1", after: 1 }), { + stopReason: "toolUse", + }); + }, + (context) => { + expect(context.messages).toHaveLength(7); + expect(context.messages.filter((message) => message.role === "toolResult")).toMatchObject([ + { toolName: "search_evidence", isError: false }, + { toolName: "read_slice", isError: true }, + { toolName: "inspect_adjacent", isError: false }, + ]); return fauxAssistantMessage(fauxToolCall("submit_evolution_response", { complete: true, lessons: [] }), { stopReason: "toolUse", }); @@ -507,6 +696,14 @@ describe("coding-agent evolution runtime", () => { input: {}, outputSchema: { type: "object" }, tools: [ + { + name: "search_evidence", + description: "Search permitted evidence.", + inputSchema: { type: "object" }, + execute() { + return Promise.resolve({ matches: [{ sliceRef: "S1" }] }); + }, + }, { name: "read_slice", description: "Read a contiguous evidence slice.", @@ -520,6 +717,14 @@ describe("coding-agent evolution runtime", () => { ); }, }, + { + name: "inspect_adjacent", + description: "Inspect permitted adjacent evidence.", + inputSchema: { type: "object" }, + execute() { + return Promise.resolve({ sliceRef: "S1", evidenceRefs: ["E2"] }); + }, + }, ], }), ).resolves.toMatchObject({ diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 5ef3fd207..846632b36 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -78,6 +78,7 @@ export interface ReflectorModelRequest { readonly input: JsonObject; readonly outputSchema: JsonObject; readonly tools: readonly EvolutionModelTool[]; + readonly validateOutput?: (output: JsonObject) => void; readonly beforeProviderCall?: () => void | Promise; readonly onProviderCallComplete?: (metrics: ReflectorProviderCallMetrics) => void; readonly onInternalToolRound?: () => void; From a895383d47a2446a73bccb22337c9e866d7d498c Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 24 Aug 2026 18:10:40 -0400 Subject: [PATCH 3/8] feat(agent): continue reflection after leaf failures - Persist one fresh leaf retry with independent deadlines, transcripts, tools, and terminal diagnostics. - Synthesize successful child lessons past unavailable or empty branches and report skipped coverage. --- packages/coding-agent/CHANGELOG.md | 1 + packages/evolution/src/contracts.ts | 15 + packages/evolution/src/index.ts | 2 + packages/evolution/src/reflector-tools.ts | 33 +- packages/evolution/src/reflector.ts | 621 +++++++++++++--------- packages/evolution/test/reflector.test.ts | 215 ++++++-- 6 files changed, 575 insertions(+), 312 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 8a05e8c9d..b1cead607 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,6 +13,7 @@ ### Fixed +- Fixed evolution reflection nodes losing tool and denial history between response corrections, and continued synthesis past terminal leaf failures. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). - Fixed right-click not pasting clipboard text in fullscreen mode on Windows. diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 45dac9a53..04f2989f3 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -321,6 +321,11 @@ export interface EvolutionReferenceNormalization { readonly normalizedReference: string; } +export interface ReflectionNodeFailure { + readonly diagnosticCode: string; + readonly message: string; +} + export interface ReflectionNode { readonly metadata: EvolutionArtifactMetadata; readonly nodeId: string; @@ -333,6 +338,7 @@ export interface ReflectionNode { readonly lessonIds: readonly string[]; readonly resultDigest: string; readonly status: ReflectionNodeStatus; + readonly failure?: ReflectionNodeFailure; readonly runtime?: ReflectionNodeRuntime; } @@ -837,6 +843,14 @@ export interface ReflectorModelCallLedgerEvent extends EvolutionLedgerEventBase readonly budgetCharge: ReflectorBudgetCharge; } +export interface ReflectionLeafRetryPendingLedgerEvent extends EvolutionLedgerEventBase { + readonly type: "reflection-leaf-retry-pending"; + readonly nodeId: string; + readonly manifestDigest: string; + readonly failure: ReflectionNodeFailure; + readonly runtime: ReflectionNodeRuntime; +} + export interface ReflectionNodeCheckpointedLedgerEvent extends EvolutionLedgerEventBase { readonly type: "reflection-node-checkpointed"; readonly nodeId: string; @@ -855,6 +869,7 @@ export type EvolutionLedgerEvent = | JobCompletedLedgerEvent | JobFailedLedgerEvent | ProposalRolledBackLedgerEvent + | ReflectionLeafRetryPendingLedgerEvent | ReflectionNodeCheckpointedLedgerEvent | CuratorToolCallLedgerEvent | ReflectorModelCallLedgerEvent diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 6dde564dc..8df13b503 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -100,8 +100,10 @@ export { type ProposalDispositionLedgerEvent, type ProposalRolledBackLedgerEvent, type ProposalValidationResult, + type ReflectionLeafRetryPendingLedgerEvent, type ReflectionNode, type ReflectionNodeCheckpointedLedgerEvent, + type ReflectionNodeFailure, type ReflectionNodeStatus, type ReflectorBudgetCharge, type ReflectorModelCallLedgerEvent, diff --git a/packages/evolution/src/reflector-tools.ts b/packages/evolution/src/reflector-tools.ts index 472d24f96..c5cd010ca 100644 --- a/packages/evolution/src/reflector-tools.ts +++ b/packages/evolution/src/reflector-tools.ts @@ -72,6 +72,7 @@ export interface ReflectorRecoverableToolDenial { interface NodeToolBudget { readonly maxToolCalls: number; + readonly signal?: AbortSignal; toolCalls: number; } @@ -207,9 +208,9 @@ export class ReflectorToolRegistry { readonly #signal?: AbortSignal; readonly #audit: (event: ReflectorToolAudit) => Promise; readonly #recoverableDenials: ReflectorRecoverableToolDenial[] = []; + readonly #toolBudgets = new WeakMap(); #revoked = false; #toolCalls = 0; - #nodeBudgetExhausted = false; constructor(options: ReflectorToolRegistryOptions) { this.#jobId = options.jobId; @@ -289,13 +290,18 @@ export class ReflectorToolRegistry { } get budgetExhausted(): boolean { - return this.#toolCalls > this.#maxToolCalls || this.#nodeBudgetExhausted; + return this.#toolCalls > this.#maxToolCalls; } get recoverableDenials(): readonly ReflectorRecoverableToolDenial[] { return this.#recoverableDenials; } + toolBudgetExhausted(tools: readonly EvolutionModelTool[]): boolean { + const budget = this.#toolBudgets.get(tools); + return budget !== undefined && budget.toolCalls > budget.maxToolCalls; + } + revoke(): void { this.#revoked = true; } @@ -342,8 +348,12 @@ export class ReflectorToolRegistry { }; } - toolsForEvidence(references: ReflectorNodeReferences, maxToolCalls: number): readonly EvolutionModelTool[] { - const budget: NodeToolBudget = { maxToolCalls, toolCalls: 0 }; + toolsForEvidence( + references: ReflectorNodeReferences, + maxToolCalls: number, + signal?: AbortSignal, + ): readonly EvolutionModelTool[] { + const budget: NodeToolBudget = { maxToolCalls, signal, toolCalls: 0 }; const permittedEvidenceIds = new Set(references.evidence.entries.map((entry) => entry.value.evidence.evidenceId)); const definitions: readonly [Exclude, string, JsonObject][] = [ [ @@ -376,20 +386,23 @@ export class ReflectorToolRegistry { ), ], ]; - return definitions.map(([name, description, inputSchema]) => ({ + const tools = definitions.map(([name, description, inputSchema]) => ({ name, description, inputSchema, - execute: (input) => this.#executeEvidence(name, input, references, permittedEvidenceIds, budget), + execute: (input: JsonObject) => this.#executeEvidence(name, input, references, permittedEvidenceIds, budget), })); + this.#toolBudgets.set(tools, budget); + return tools; } toolsForLessons( lessons: EvolutionModelReferenceCatalog, maxToolCalls: number, + signal?: AbortSignal, ): readonly EvolutionModelTool[] { - const budget: NodeToolBudget = { maxToolCalls, toolCalls: 0 }; - return [ + const budget: NodeToolBudget = { maxToolCalls, signal, toolCalls: 0 }; + const tools: readonly EvolutionModelTool[] = [ { name: "inspect_child_lessons", description: "Inspect complete child lesson content without opening raw corpus evidence.", @@ -406,6 +419,8 @@ export class ReflectorToolRegistry { execute: (input) => this.#executeLessonInspection(input, lessons, budget), }, ]; + this.#toolBudgets.set(tools, budget); + return tools; } planInitialSlices(maxUnitsPerSlice: number): readonly EvidenceSlice[] { @@ -503,6 +518,7 @@ export class ReflectorToolRegistry { readonly resolvedSliceIds: readonly string[]; }, ): Promise { + if (!this.#revoked) budget.signal?.throwIfAborted(); this.#toolCalls += 1; budget.toolCalls += 1; const toolCallId = `reflector-tool-${this.#toolCalls}`; @@ -523,7 +539,6 @@ export class ReflectorToolRegistry { throw new EvolutionToolDeniedError("tool-budget-exhausted", "Reflector tool-call budget is exhausted"); } if (budget.toolCalls > budget.maxToolCalls) { - this.#nodeBudgetExhausted = true; throw new EvolutionToolDeniedError( "tool-budget-exhausted", "Reflection node tool-call budget is exhausted", diff --git a/packages/evolution/src/reflector.ts b/packages/evolution/src/reflector.ts index af4d1d994..554124996 100644 --- a/packages/evolution/src/reflector.ts +++ b/packages/evolution/src/reflector.ts @@ -9,8 +9,10 @@ import type { JsonObject, Lesson, LessonKind, + ReflectionLeafRetryPendingLedgerEvent, ReflectionNode, ReflectionNodeCheckpointedLedgerEvent, + ReflectionNodeFailure, ReflectionNodeRuntime, ReflectorModelCallLedgerEvent, ReflectorToolCallLedgerEvent, @@ -32,8 +34,6 @@ import { type ReflectorNodeReferences, type ReflectorToolAudit, ReflectorToolReg const DEFAULT_MAX_TOOL_CALLS = 200; const DEFAULT_MAX_MODEL_CALLS = 100; const DEFAULT_MAX_RECURSION_DEPTH = 8; -const DEFAULT_MAX_REFLECTION_ROUNDS = 2; -const DEFAULT_MAX_RESPONSE_RETRIES = 2; const DEFAULT_MAX_EVIDENCE_TOOL_CALLS_PER_LEAF = 6; const DEFAULT_MAX_LESSON_TOOL_CALLS_PER_SYNTHESIS_NODE = 2; const DEFAULT_MAX_LESSONS_PER_LEAF = 3; @@ -63,8 +63,6 @@ export interface RecursiveReflectorOptions { readonly maxToolCalls?: number; readonly maxModelCalls?: number; readonly maxRecursionDepth?: number; - readonly maxReflectionRounds?: number; - readonly maxResponseRetries?: number; readonly maxEvidenceToolCallsPerLeaf?: number; readonly maxLessonToolCallsPerSynthesisNode?: number; readonly maxLessonsPerLeaf?: number; @@ -104,6 +102,7 @@ interface MutableNodeRuntime { outputTokens: number; retries: number; foregroundWaitMs: number; + priorNodeDurationMs: number; readonly startedAt: number; } @@ -115,18 +114,19 @@ function nodeRuntime(runtime: MutableNodeRuntime): ReflectionNodeRuntime { outputTokens: runtime.outputTokens, retries: runtime.retries, foregroundWaitMs: runtime.foregroundWaitMs, - nodeDurationMs: performance.now() - runtime.startedAt, + nodeDurationMs: runtime.priorNodeDurationMs + performance.now() - runtime.startedAt, }; } -function newNodeRuntime(): MutableNodeRuntime { +function newNodeRuntime(prior?: ReflectionNodeRuntime): MutableNodeRuntime { return { - providerCalls: 0, - providerLatencyMs: 0, - internalToolRounds: 0, - outputTokens: 0, - retries: 0, - foregroundWaitMs: 0, + providerCalls: prior?.providerCalls ?? 0, + providerLatencyMs: prior?.providerLatencyMs ?? 0, + internalToolRounds: prior?.internalToolRounds ?? 0, + outputTokens: prior?.outputTokens ?? 0, + retries: prior?.retries ?? 0, + foregroundWaitMs: prior?.foregroundWaitMs ?? 0, + priorNodeDurationMs: prior?.nodeDurationMs ?? 0, startedAt: performance.now(), }; } @@ -265,8 +265,8 @@ function parseResponse( readonly lessons: readonly LessonDraft[]; readonly referenceNormalizations: readonly EvolutionReferenceNormalization[]; } { - if (response.output.complete !== true && response.output.complete !== false) { - throw new Error("Reflector response complete must be boolean"); + if (response.output.complete !== true) { + throw new Error("Reflector response complete must be true"); } if (!Array.isArray(response.output.lessons)) throw new Error("Reflector response lessons must be an array"); const referenceNormalizations: EvolutionReferenceNormalization[] = []; @@ -293,17 +293,18 @@ function reflectorOutputSchema( references: readonly string[], maxLessons: number | undefined, ): JsonObject { + const evidenceReferences = references.flatMap((reference) => [reference, reference.slice(1)]); const referenceProperties: JsonObject = kind === "leaf" ? { supportingEvidenceRefs: { type: "array", minItems: 1, - items: { type: "string", enum: references }, + items: { type: "string", enum: evidenceReferences }, }, counterevidenceRefs: { type: "array", - items: { type: "string", enum: references }, + items: { type: "string", enum: evidenceReferences }, }, } : { @@ -324,7 +325,7 @@ function reflectorOutputSchema( additionalProperties: false, required: ["complete", "lessons"], properties: { - complete: { type: "boolean" }, + complete: { enum: [true] }, lessons: { type: "array", ...(maxLessons === undefined ? {} : { maxItems: maxLessons }), @@ -462,8 +463,6 @@ export class RecursiveReflector implements Reflector { readonly #maxToolCalls: number; readonly #maxModelCalls: number; readonly #maxRecursionDepth: number; - readonly #maxReflectionRounds: number; - readonly #maxResponseRetries: number; readonly #maxEvidenceToolCallsPerLeaf: number; readonly #maxLessonToolCallsPerSynthesisNode: number; readonly #maxLessonsPerLeaf: number; @@ -483,16 +482,6 @@ export class RecursiveReflector implements Reflector { DEFAULT_MAX_RECURSION_DEPTH, "maxRecursionDepth", ); - this.#maxReflectionRounds = positiveInteger( - options.maxReflectionRounds, - DEFAULT_MAX_REFLECTION_ROUNDS, - "maxReflectionRounds", - ); - this.#maxResponseRetries = positiveInteger( - options.maxResponseRetries, - DEFAULT_MAX_RESPONSE_RETRIES, - "maxResponseRetries", - ); this.#maxEvidenceToolCallsPerLeaf = positiveInteger( options.maxEvidenceToolCallsPerLeaf, DEFAULT_MAX_EVIDENCE_TOOL_CALLS_PER_LEAF, @@ -562,6 +551,7 @@ export class RecursiveReflector implements Reflector { const slices = registry.planInitialSlices(this.#maxUnitsPerSlice); const plans = this.#buildPlan(input, slices); const completed = await this.#loadCompleted(input, plans, priorEvents); + const retryPending = this.#loadRetryPending(input, plans, priorEvents, completed); let modelCalls = priorModelCalls; const executePlan = async (plan: NodePlan) => { if (stageDeadline.expired) { @@ -572,48 +562,98 @@ export class RecursiveReflector implements Reflector { }); } if (completed.has(plan.nodeId)) return; - const timedOutChild = plan.childNodeIds.some((nodeId) => completed.get(nodeId)?.node.status !== "complete"); - if (timedOutChild) { - completed.set(plan.nodeId, await this.#checkpointTimedOut(input, producer, plan, newNodeRuntime())); + const unavailableChildNodeIds = plan.childNodeIds.filter( + (nodeId) => completed.get(nodeId)?.node.status !== "complete", + ); + const childLessons = plan.childNodeIds.flatMap((nodeId) => { + const child = completed.get(nodeId); + return child?.node.status === "complete" ? child.lessons : []; + }); + if (plan.kind === "synthesis" && childLessons.length === 0) { + completed.set( + plan.nodeId, + await this.#checkpointEmptySynthesis(input, producer, plan, unavailableChildNodeIds), + ); return; } - const nodeDeadline = new ActiveComputeDeadline(this.#nodeDeadlineMs, `Reflection node ${plan.nodeId}`); - const runtime = newNodeRuntime(); + const pending = retryPending.get(plan.nodeId); + let nodeAttempt = pending === undefined ? 1 : 2; + let runtime = newNodeRuntime(pending?.runtime); let result: CompletedNode; - try { - result = await this.#executeNode( - { ...input, signal: combinedSignal(stageSignal, nodeDeadline.signal) }, - producer, - registry, - plan, - completed, - () => { - modelCalls += 1; - if (modelCalls > this.#maxModelCalls) { - throw new EvolutionExecutionError("Reflector model-call budget is exhausted", { - reasonCode: "reflection-budget-exhausted", - retryable: false, - }); - } - return modelCalls; - }, - stageDeadline, - nodeDeadline, - runtime, + while (true) { + const priorAttemptModelCalls = priorEvents.filter( + (event) => + event.type === "reflector-model-call" && + event.nodeId === plan.nodeId && + event.attempt === nodeAttempt, + ).length; + runtime.providerCalls += priorAttemptModelCalls; + const nodeDeadline = new ActiveComputeDeadline( + this.#nodeDeadlineMs, + `Reflection node ${plan.nodeId} attempt ${nodeAttempt}`, ); - } catch (error) { - input.signal?.throwIfAborted(); - if (stageDeadline.expired) { - throw new EvolutionExecutionError("Reflector stage active-compute deadline was exhausted", { - reasonCode: "reflection-budget-exhausted", - retryable: true, - cause: error, - }); + const attemptController = new AbortController(); + try { + result = await this.#executeNode( + { ...input, signal: combinedSignal(stageSignal, nodeDeadline.signal, attemptController.signal) }, + producer, + registry, + plan, + completed, + unavailableChildNodeIds, + nodeAttempt, + priorAttemptModelCalls, + () => { + if (modelCalls >= this.#maxModelCalls) { + throw new EvolutionExecutionError("Reflector model-call budget is exhausted", { + reasonCode: "reflection-budget-exhausted", + retryable: false, + }); + } + modelCalls += 1; + }, + stageDeadline, + nodeDeadline, + runtime, + ); + break; + } catch (error) { + input.signal?.throwIfAborted(); + if (stageDeadline.expired) { + throw new EvolutionExecutionError("Reflector stage active-compute deadline was exhausted", { + reasonCode: "reflection-budget-exhausted", + retryable: true, + cause: error, + }); + } + const timedOut = nodeDeadline.expired; + const failure = this.#nodeFailure(error, timedOut); + if (plan.kind === "leaf" && nodeAttempt === 1 && (timedOut || this.#isRetryableLeafFailure(error))) { + const retry = await this.#checkpointRetryPending(input, producer, plan, failure, runtime); + retryPending.set(plan.nodeId, retry); + runtime = newNodeRuntime(retry.runtime); + nodeAttempt = 2; + continue; + } + if (plan.kind === "leaf" && !timedOut && !this.#isRetryableLeafFailure(error)) throw error; + if (plan.kind === "leaf") { + result = await this.#checkpointUnavailable( + input, + producer, + plan, + timedOut ? "timed-out" : "failed", + failure, + runtime, + ); + break; + } + if (!timedOut) throw error; + result = await this.#checkpointUnavailable(input, producer, plan, "timed-out", failure, runtime); + break; + } finally { + attemptController.abort(new Error(`Reflection node ${plan.nodeId} attempt ended`)); + nodeDeadline.dispose(); } - if (!nodeDeadline.expired) throw error; - result = await this.#checkpointTimedOut(input, producer, plan, runtime); - } finally { - nodeDeadline.dispose(); } completed.set(plan.nodeId, result); }; @@ -711,7 +751,9 @@ export class RecursiveReflector implements Reflector { if ( !isReflectionNode(nodeArtifact) || nodeArtifact.nodeId !== event.nodeId || - (nodeArtifact.status !== "complete" && nodeArtifact.status !== "timed-out") + (nodeArtifact.status !== "complete" && + nodeArtifact.status !== "failed" && + nodeArtifact.status !== "timed-out") ) { throw new EvolutionExecutionError(`Reflection checkpoint "${event.nodeId}" is corrupt`, { reasonCode: "artifact-corrupt", @@ -729,7 +771,10 @@ export class RecursiveReflector implements Reflector { } lessons.push(artifact); } - if (lessons.length !== nodeArtifact.lessonIds.length) { + if ( + lessons.length !== nodeArtifact.lessonIds.length || + (nodeArtifact.status !== "complete" && lessons.length > 0) + ) { throw new EvolutionExecutionError(`Reflection checkpoint "${event.nodeId}" is incomplete`, { reasonCode: "artifact-corrupt", retryable: false, @@ -740,13 +785,37 @@ export class RecursiveReflector implements Reflector { return result; } + #loadRetryPending( + input: ReflectorInput, + plans: readonly NodePlan[], + events: readonly EvolutionLedgerEvent[], + completed: ReadonlyMap, + ): Map { + const leafIds = new Set(plans.filter((plan) => plan.kind === "leaf").map((plan) => plan.nodeId)); + const result = new Map(); + for (const event of events) { + if ( + event.type === "reflection-leaf-retry-pending" && + event.manifestDigest === input.corpus.metadata.contentDigest && + leafIds.has(event.nodeId) && + !completed.has(event.nodeId) + ) { + result.set(event.nodeId, event); + } + } + return result; + } + async #executeNode( input: ReflectorInput, producer: EvolutionArtifactProducer, registry: ReflectorToolRegistry, plan: NodePlan, completed: ReadonlyMap, - chargeModelCall: () => number, + unavailableChildNodeIds: readonly string[], + nodeAttempt: number, + priorAttemptModelCalls: number, + chargeModelCall: () => void, stageDeadline: ActiveComputeDeadline, nodeDeadline: ActiveComputeDeadline, runtime: MutableNodeRuntime, @@ -756,8 +825,8 @@ export class RecursiveReflector implements Reflector { const lessonReferences = new EvolutionModelReferenceCatalog("lesson", childLessons); const tools = plan.kind === "leaf" - ? registry.toolsForEvidence(nodeReferences, this.#maxEvidenceToolCallsPerLeaf) - : registry.toolsForLessons(lessonReferences, this.#maxLessonToolCallsPerSynthesisNode); + ? registry.toolsForEvidence(nodeReferences, this.#maxEvidenceToolCallsPerLeaf, input.signal) + : registry.toolsForLessons(lessonReferences, this.#maxLessonToolCallsPerSynthesisNode, input.signal); const maxLessons = plan.kind === "leaf" ? this.#maxLessonsPerLeaf @@ -770,155 +839,120 @@ export class RecursiveReflector implements Reflector { maxLessons, ); const denialStart = registry.recoverableDenials.length; - let response: EvolutionModelResponse | null = null; - let parsed: ReturnType | null = null; - let lastError: unknown = null; - let lastFailure: "model" | "response" | null = null; - let responseCorrection: string | null = null; - const rejectedResponses: JsonObject[] = []; - const invalidReferences = new Set(); - const permittedReferences = new Set(); - let nodeModelCallSequence = 0; - for (let round = 1; round <= this.#maxReflectionRounds && parsed?.complete !== true; round += 1) { - for (let attempt = 1; attempt <= this.#maxResponseRetries; attempt += 1) { - if (attempt > 1) runtime.retries += 1; - let providerCalls = 0; - const beforeProviderCall = async () => { - chargeModelCall(); - providerCalls += 1; - runtime.providerCalls += 1; - nodeModelCallSequence += 1; - const modelCallId = stableId("reflector-model-call", { - idempotencyKey: input.idempotencyKey, - nodeId: plan.nodeId, - round, - nodeModelCallSequence, - }); - await this.#dependencies.ledger.append( - this.#modelCallEvent(input, producer, modelCallId, plan.nodeId, round, attempt), - ); - }; - try { - response = await abortable( - input.model.execute({ - stage: "reflector", - model: input.input.modelConfiguration, - promptVersion: input.input.promptVersions.reflector, - idempotencyKey: `${input.idempotencyKey}:${plan.nodeId}:${round}`, - input: this.#modelInput(input, plan, nodeReferences, lessonReferences, round, responseCorrection), - outputSchema, - tools, - beforeProviderCall, - onProviderCallComplete: (metrics) => { - runtime.providerLatencyMs += metrics.durationMs; - runtime.outputTokens += metrics.outputTokens; - if (metrics.retry) runtime.retries += 1; - }, - onInternalToolRound: () => { - runtime.internalToolRounds += 1; - }, - onForegroundWaitStart: () => { - stageDeadline.pause(); - nodeDeadline.pause(); - }, - onForegroundWaitEnd: (durationMs) => { - runtime.foregroundWaitMs += durationMs; - nodeDeadline.resume(); - stageDeadline.resume(); - }, - signal: input.signal, - }), - input.signal, - ); - if (providerCalls === 0) await beforeProviderCall(); - } catch (error) { - input.signal?.throwIfAborted(); - if (error instanceof EvolutionExecutionError) throw error; - if (providerCalls === 0) await beforeProviderCall(); - lastError = error; - lastFailure = "model"; - continue; - } - if (registry.budgetExhausted) { - throw new EvolutionExecutionError("Reflector tool-call budget is exhausted", { - reasonCode: "reflection-budget-exhausted", - retryable: false, - }); - } - try { - parsed = parseResponse( - response, - plan.kind === "leaf" - ? { kind: "leaf", evidence: nodeReferences.evidence } - : { kind: "synthesis", lessons: lessonReferences }, - maxLessons, - ); - lastError = null; - lastFailure = null; - responseCorrection = null; - break; - } catch (error) { - lastError = error; - lastFailure = "response"; - responseCorrection = error instanceof Error ? error.message : "Response validation failed"; - if (error instanceof EvolutionModelReferenceError) { - for (const reference of error.invalidReferences) invalidReferences.add(reference); - for (const reference of error.permittedReferences) permittedReferences.add(reference); - } - rejectedResponses.push({ - reflectionRound: round, - attempt, - model: response.model, - submissionMethod: response.submissionMethod ?? "unspecified", - validationError: responseCorrection, - output: response.output, - }); - } - } - if (lastError !== null) { - throw new EvolutionExecutionError( - lastFailure === "model" - ? `Reflector model call failed for "${plan.nodeId}"` - : `Reflector returned an invalid response for "${plan.nodeId}"`, - { - reasonCode: lastFailure === "model" ? "model-call-failed" : "reflection-invalid-response", - retryable: lastFailure === "model", - cause: lastError, - diagnostics: { - node: { - nodeId: plan.nodeId, - kind: plan.kind, - depth: plan.depth, - parentNodeId: plan.parentNodeId, - childNodeIds: plan.childNodeIds, - sliceIds: plan.sliceIds, - evidenceIds: plan.evidenceIds, - }, - invalidAliases: [...invalidReferences], - permittedAliases: [...permittedReferences], - attemptDetails: rejectedResponses, - aliasResolutionMaps: { - documents: nodeReferences.documents.resolutionMap((document) => document.descriptor.documentId), - evidence: nodeReferences.evidence.resolutionMap((entry) => entry.evidence.evidenceId), - lessons: lessonReferences.resolutionMap((lesson) => lesson.lessonId), - }, - recoverableToolDenials: registry.recoverableDenials.slice(denialStart).map((denial) => ({ - toolName: denial.toolName, - denialCode: denial.denialCode, - requestedDocumentRef: denial.requestedDocumentRef, - requestedSliceRef: denial.requestedSliceRef, - })), - }, + let providerCallHookInvoked = false; + let providerCalls = 0; + let nodeModelCallSequence = priorAttemptModelCalls; + const beforeProviderCall = async () => { + providerCallHookInvoked = true; + chargeModelCall(); + providerCalls += 1; + runtime.providerCalls += 1; + nodeModelCallSequence += 1; + const modelCallId = stableId("reflector-model-call", { + idempotencyKey: input.idempotencyKey, + nodeId: plan.nodeId, + nodeAttempt, + nodeModelCallSequence, + }); + await this.#dependencies.ledger.append( + this.#modelCallEvent(input, producer, modelCallId, plan.nodeId, nodeAttempt, nodeModelCallSequence), + ); + }; + let response: EvolutionModelResponse; + try { + response = await abortable( + input.model.execute({ + stage: "reflector", + model: input.input.modelConfiguration, + promptVersion: input.input.promptVersions.reflector, + idempotencyKey: `${input.idempotencyKey}:${plan.nodeId}:attempt-${nodeAttempt}`, + input: this.#modelInput(input, plan, nodeReferences, lessonReferences, unavailableChildNodeIds), + outputSchema, + tools, + validateOutput: (output) => { + parseResponse( + { output, model: { provider: "validation", model: "validation" } }, + plan.kind === "leaf" + ? { kind: "leaf", evidence: nodeReferences.evidence } + : { kind: "synthesis", lessons: lessonReferences }, + maxLessons, + ); }, - ); - } + beforeProviderCall, + onProviderCallComplete: (metrics) => { + runtime.providerLatencyMs += metrics.durationMs; + runtime.outputTokens += metrics.outputTokens; + if (metrics.retry) runtime.retries += 1; + }, + onInternalToolRound: () => { + runtime.internalToolRounds += 1; + }, + onForegroundWaitStart: () => { + stageDeadline.pause(); + nodeDeadline.pause(); + }, + onForegroundWaitEnd: (durationMs) => { + runtime.foregroundWaitMs += durationMs; + nodeDeadline.resume(); + stageDeadline.resume(); + }, + signal: input.signal, + }), + input.signal, + ); + if (providerCalls === 0) await beforeProviderCall(); + } catch (error) { + input.signal?.throwIfAborted(); + if (!providerCallHookInvoked) await beforeProviderCall(); + if (error instanceof EvolutionExecutionError) throw error; + throw new EvolutionExecutionError(`Reflector model call failed for "${plan.nodeId}"`, { + reasonCode: "model-call-failed", + retryable: true, + cause: error, + diagnostics: { + nodeId: plan.nodeId, + nodeAttempt, + recoverableToolDenials: registry.recoverableDenials.slice(denialStart).map((denial) => ({ + toolName: denial.toolName, + denialCode: denial.denialCode, + requestedDocumentRef: denial.requestedDocumentRef, + requestedSliceRef: denial.requestedSliceRef, + })), + }, + }); } - if (response === null || parsed === null || !parsed.complete) { - throw new EvolutionExecutionError(`Reflector did not complete "${plan.nodeId}" within the refinement budget`, { + if (registry.budgetExhausted || registry.toolBudgetExhausted(tools)) { + throw new EvolutionExecutionError("Reflector tool-call budget is exhausted", { reasonCode: "reflection-budget-exhausted", retryable: false, }); } + let parsed: ReturnType; + try { + parsed = parseResponse( + response, + plan.kind === "leaf" + ? { kind: "leaf", evidence: nodeReferences.evidence } + : { kind: "synthesis", lessons: lessonReferences }, + maxLessons, + ); + } catch (error) { + const invalidReferences = error instanceof EvolutionModelReferenceError ? error.invalidReferences : []; + const permittedReferences = error instanceof EvolutionModelReferenceError ? error.permittedReferences : []; + throw new EvolutionExecutionError(`Reflector returned an invalid response for "${plan.nodeId}"`, { + reasonCode: "reflection-invalid-response", + retryable: false, + cause: error, + diagnostics: { + nodeId: plan.nodeId, + nodeAttempt, + invalidAliases: invalidReferences, + permittedAliases: permittedReferences, + submissionMethod: response.submissionMethod ?? "unspecified", + output: response.output, + }, + }); + } const modelProducer: EvolutionArtifactProducer = { ...producer, model: { @@ -992,10 +1026,62 @@ export class RecursiveReflector implements Reflector { return { node, lessons }; } - async #checkpointTimedOut( + #nodeFailure(error: unknown, timedOut: boolean): ReflectionNodeFailure { + if (timedOut) { + return { + diagnosticCode: "node-timeout", + message: error instanceof Error ? error.message : "Reflection node execution timed out", + }; + } + return { + diagnosticCode: error instanceof EvolutionExecutionError ? error.reasonCode : "model-call-failed", + message: error instanceof Error ? error.message : String(error), + }; + } + + #isRetryableLeafFailure(error: unknown): boolean { + return ( + error instanceof EvolutionExecutionError && + (error.reasonCode === "model-call-failed" || error.reasonCode === "reflection-invalid-response") + ); + } + + async #checkpointRetryPending( input: ReflectorInput, producer: EvolutionArtifactProducer, plan: NodePlan, + failure: ReflectionNodeFailure, + runtime: MutableNodeRuntime, + ): Promise { + const content = { + nodeId: plan.nodeId, + manifestDigest: input.corpus.metadata.contentDigest, + failure, + runtime: nodeRuntime(runtime), + }; + const eventId = stableId("ledger-event", { + jobId: input.jobId, + type: "reflection-leaf-retry-pending", + nodeId: plan.nodeId, + manifestDigest: input.corpus.metadata.contentDigest, + }); + const event: ReflectionLeafRetryPendingLedgerEvent = { + metadata: this.#metadata(input, producer, eventId, [plan.nodeId], content), + eventId, + jobId: input.jobId, + type: "reflection-leaf-retry-pending", + ...content, + }; + await this.#dependencies.ledger.append(event); + return event; + } + + async #checkpointUnavailable( + input: ReflectorInput, + producer: EvolutionArtifactProducer, + plan: NodePlan, + status: "failed" | "timed-out", + failure: ReflectionNodeFailure, runtime: MutableNodeRuntime, ): Promise { const nodeContent = { @@ -1007,8 +1093,9 @@ export class RecursiveReflector implements Reflector { sliceIds: plan.sliceIds, evidenceIds: plan.evidenceIds, lessonIds: [], - resultDigest: computeEvolutionContentDigest({ reason: "node-timeout" }), - status: "timed-out" as const, + resultDigest: computeEvolutionContentDigest({ status, failure }), + status, + failure, runtime: nodeRuntime(runtime), }; const node: ReflectionNode = { @@ -1039,13 +1126,59 @@ export class RecursiveReflector implements Reflector { return { node, lessons: [] }; } + async #checkpointEmptySynthesis( + input: ReflectorInput, + producer: EvolutionArtifactProducer, + plan: NodePlan, + unavailableChildNodeIds: readonly string[], + ): Promise { + const nodeContent = { + nodeId: plan.nodeId, + kind: plan.kind, + depth: plan.depth, + parentNodeId: plan.parentNodeId, + childNodeIds: plan.childNodeIds, + sliceIds: plan.sliceIds, + evidenceIds: plan.evidenceIds, + lessonIds: [], + resultDigest: computeEvolutionContentDigest({ empty: true, unavailableChildNodeIds }), + status: "complete" as const, + runtime: nodeRuntime(newNodeRuntime()), + }; + const node: ReflectionNode = { + metadata: this.#metadata(input, producer, plan.nodeId, plan.evidenceIds, nodeContent), + ...nodeContent, + }; + const nodeReference = await this.#dependencies.artifacts.put(node); + const checkpointId = stableId("ledger-event", { + jobId: input.jobId, + type: "reflection-node-checkpointed", + nodeId: node.nodeId, + manifestDigest: input.corpus.metadata.contentDigest, + }); + const checkpointContent = { + nodeId: node.nodeId, + manifestDigest: input.corpus.metadata.contentDigest, + node: nodeReference, + lessons: [], + referenceNormalizations: [], + }; + await this.#dependencies.ledger.append({ + metadata: this.#metadata(input, producer, checkpointId, [node.nodeId], checkpointContent), + eventId: checkpointId, + jobId: input.jobId, + type: "reflection-node-checkpointed", + ...checkpointContent, + }); + return { node, lessons: [] }; + } + #modelInput( input: ReflectorInput, plan: NodePlan, nodeReferences: ReflectorNodeReferences, lessonReferences: EvolutionModelReferenceCatalog, - round: number, - responseCorrection: string | null, + unavailableChildNodeIds: readonly string[], ): JsonObject { const instructionReferences = new EvolutionModelReferenceCatalog("instruction", input.instructions.documents); const evidenceReferences = new Map( @@ -1082,22 +1215,11 @@ export class RecursiveReflector implements Reflector { consolidateOverlappingLessons: plan.kind === "synthesis", maximumToolCalls: plan.kind === "leaf" ? this.#maxEvidenceToolCallsPerLeaf : this.#maxLessonToolCallsPerSynthesisNode, - maximumRefinementRounds: this.#maxReflectionRounds, - structuredResponseRetries: this.#maxResponseRetries, - ...(responseCorrection === null - ? {} - : { - responseCorrection: { - reason: responseCorrection, - instruction: "Correct the structured response and submit it again.", - }, - }), }, node: { kind: plan.kind, depth: plan.depth, }, - reflectionRound: round, instructions: instructionReferences.entries.map(({ ref, value: document }, index) => ({ instructionRef: ref, label: `Instruction ${index + 1}`, @@ -1112,6 +1234,7 @@ export class RecursiveReflector implements Reflector { slices: nodeReferences.sliceCatalog, } : { + unavailableChildNodeIds, childLessons: lessonReferences.entries.map(({ ref, value: lesson }) => ({ lessonRef: ref, kind: lesson.kind, @@ -1133,11 +1256,20 @@ export class RecursiveReflector implements Reflector { completed: ReadonlyMap, ): EvidenceCoverageMap { const entries: EvidenceCoverageEntry[] = []; - const timedOutEvidenceIds = new Set( - [...completed.values()] - .filter((entry) => entry.node.status === "timed-out") - .flatMap((entry) => entry.node.evidenceIds), - ); + const unavailableEvidenceDiagnostics = new Map(); + for (const entry of completed.values()) { + if (entry.node.kind !== "leaf" || (entry.node.status !== "failed" && entry.node.status !== "timed-out")) { + continue; + } + for (const evidenceId of entry.node.evidenceIds) { + const diagnostics = unavailableEvidenceDiagnostics.get(evidenceId) ?? []; + diagnostics.push( + entry.node.failure?.diagnosticCode ?? + (entry.node.status === "timed-out" ? "node-timeout" : "model-call-failed"), + ); + unavailableEvidenceDiagnostics.set(evidenceId, diagnostics); + } + } const analyzedEvidenceIds = new Set( [...completed.values()] .filter((entry) => entry.node.kind === "leaf" && entry.node.status === "complete") @@ -1153,6 +1285,7 @@ export class RecursiveReflector implements Reflector { ), ), ); + const unavailableDiagnostics = unavailableEvidenceDiagnostics.get(evidenceId) ?? []; entries.push({ documentId: document.documentId, evidenceId, @@ -1162,7 +1295,7 @@ export class RecursiveReflector implements Reflector { : document.integrity === "partial" ? "corrupt" : matching.length === 0 || - (timedOutEvidenceIds.has(evidenceId) && !analyzedEvidenceIds.has(evidenceId)) + (unavailableDiagnostics.length > 0 && !analyzedEvidenceIds.has(evidenceId)) ? "skipped" : "analyzed", sliceIds: matching.map((slice) => slice.sliceId), @@ -1170,9 +1303,7 @@ export class RecursiveReflector implements Reflector { continuationSliceIds, diagnosticCodes: unique([ ...document.diagnostics.map((diagnostic) => diagnostic.code), - ...(timedOutEvidenceIds.has(evidenceId) && !analyzedEvidenceIds.has(evidenceId) - ? ["node-timeout"] - : []), + ...(!analyzedEvidenceIds.has(evidenceId) ? unavailableDiagnostics : []), ]), }); } @@ -1243,17 +1374,17 @@ export class RecursiveReflector implements Reflector { producer: EvolutionArtifactProducer, modelCallId: string, nodeId: string, - reflectionRound: number, - attempt: number, + nodeAttempt: number, + providerRound: number, ): ReflectorModelCallLedgerEvent { const content = { modelCallId, nodeId, - attempt, - reflectionRound, + attempt: nodeAttempt, + reflectionRound: providerRound, manifestDigest: input.corpus.metadata.contentDigest, - idempotencyKey: `${input.idempotencyKey}:${nodeId}:${reflectionRound}`, - budgetCharge: { modelCalls: 1, reflectionRounds: attempt === 1 ? 1 : 0, toolCalls: 0 }, + idempotencyKey: `${input.idempotencyKey}:${nodeId}:attempt-${nodeAttempt}`, + budgetCharge: { modelCalls: 1, reflectionRounds: providerRound === 1 ? 1 : 0, toolCalls: 0 }, }; return { metadata: this.#metadata(input, producer, modelCallId, [nodeId], content), diff --git a/packages/evolution/test/reflector.test.ts b/packages/evolution/test/reflector.test.ts index cbce1fa98..9ed2c3f6a 100644 --- a/packages/evolution/test/reflector.test.ts +++ b/packages/evolution/test/reflector.test.ts @@ -400,12 +400,19 @@ describe("recursive reflector", () => { { version: "1.0.0", artifacts: persistence, ledger: persistence }, { maxUnitsPerSlice: 2, synthesisFanout: 8, nodeDeadlineMs: 10 }, ); - let calls = 0; + let timedOutLeafCalls = 0; + let unavailableChildNodeIds: readonly string[] = []; const input = reflectInput({ async execute(request) { if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); - calls += 1; - if (calls === 1) return new Promise(() => {}); + const node = jsonObject(request.input.node, "node"); + if (node.kind === "leaf" && references(request.input, "evidence", "evidenceRef").includes("E1")) { + timedOutLeafCalls += 1; + return new Promise(() => {}); + } + if (node.kind === "synthesis") { + unavailableChildNodeIds = request.input.unavailableChildNodeIds as readonly string[]; + } return { output: { complete: true, lessons: [lessonForRequest(request.input)] }, model: { provider: "faux", model: "fixture" }, @@ -415,16 +422,55 @@ describe("recursive reflector", () => { const result = await reflector.reflect(input); - expect(result.nodes.map((node) => node.status)).toEqual(["timed-out", "complete", "complete", "timed-out"]); - expect(result.finalLessonIds).toEqual([]); + expect(result.nodes.map((node) => node.status)).toEqual(["timed-out", "complete", "complete", "complete"]); + expect(result.finalLessonIds).toHaveLength(1); + expect(timedOutLeafCalls).toBe(2); + expect(unavailableChildNodeIds).toHaveLength(1); expect(result.coverage.entries.slice(0, 2)).toEqual([ expect.objectContaining({ disposition: "skipped", diagnosticCodes: expect.arrayContaining(["node-timeout"]) }), expect.objectContaining({ disposition: "skipped", diagnosticCodes: expect.arrayContaining(["node-timeout"]) }), ]); expect(result.coverage.entries.slice(2).every((entry) => entry.disposition === "analyzed")).toBe(true); + const events = await persistence.read("job-1"); + expect(events.filter((event) => event.type === "reflection-leaf-retry-pending")).toHaveLength(1); + expect(events.filter((event) => event.type === "reflection-node-checkpointed")).toHaveLength(4); + }); + + it("checkpoints an empty synthesis branch without blocking successful root lessons", async () => { + const { persistence } = await reflectorFixture(); + const reflector = new RecursiveReflector( + { version: "1.0.0", artifacts: persistence, ledger: persistence }, + { maxUnitsPerSlice: 1, synthesisFanout: 2 }, + ); + const failedEvidenceRefs = new Set(["E1", "E2"]); + const input = reflectInput({ + async execute(request) { + if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + const node = jsonObject(request.input.node, "node"); + if ( + node.kind === "leaf" && + references(request.input, "evidence", "evidenceRef").some((ref) => failedEvidenceRefs.has(ref)) + ) { + throw new Error("simulated leaf failure"); + } + return { + output: { complete: true, lessons: [lessonForRequest(request.input)] }, + model: { provider: "faux", model: "fixture" }, + }; + }, + }); + + const result = await reflector.reflect(input); + + expect(result.nodes.filter((node) => node.kind === "leaf" && node.status === "failed")).toHaveLength(2); expect( - (await persistence.read("job-1")).filter((event) => event.type === "reflection-node-checkpointed"), - ).toHaveLength(4); + result.nodes.some( + (node) => node.kind === "synthesis" && node.status === "complete" && node.runtime?.providerCalls === 0, + ), + ).toBe(true); + expect(result.finalLessonIds).toHaveLength(1); + expect(result.coverage.entries.slice(0, 2).every((entry) => entry.disposition === "skipped")).toBe(true); + expect(result.coverage.entries.slice(2).every((entry) => entry.disposition === "analyzed")).toBe(true); }); it("runs independent leaves with deterministic concurrency of two", async () => { @@ -717,6 +763,63 @@ describe("recursive reflector", () => { expect(resumedCatalog).toBe(interruptedCatalog); }); + it("resumes a durable retry-pending leaf with only its final attempt remaining", async () => { + const { persistence, reflector } = await reflectorFixture(); + const controller = new AbortController(); + let retryStarted: (() => void) | undefined; + const started = new Promise((resolveStarted) => { + retryStarted = resolveStarted; + }); + const firstRun = reflectInput({ + async execute(request) { + if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + const node = jsonObject(request.input.node, "node"); + const target = node.kind === "leaf" && references(request.input, "evidence", "evidenceRef").includes("E1"); + if (!target) { + return { + output: { complete: true, lessons: [lessonForRequest(request.input)] }, + model: { provider: "faux", model: "fixture" }, + }; + } + if (request.idempotencyKey.endsWith("attempt-1")) throw new Error("retryable leaf failure"); + retryStarted?.(); + return new Promise(() => {}); + }, + }); + const interrupted = reflector.reflect({ ...firstRun, signal: controller.signal }); + await started; + controller.abort(new Error("simulated retry interruption")); + await expect(interrupted).rejects.toThrow("simulated retry interruption"); + expect( + (await persistence.read("job-1")).filter((event) => event.type === "reflection-leaf-retry-pending"), + ).toHaveLength(1); + const targetAttempts: string[] = []; + const resumed = reflectInput({ + async execute(request) { + if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + const node = jsonObject(request.input.node, "node"); + if (node.kind === "leaf" && references(request.input, "evidence", "evidenceRef").includes("E1")) { + targetAttempts.push(request.idempotencyKey); + throw new Error("final leaf failure"); + } + return { + output: { complete: true, lessons: [lessonForRequest(request.input)] }, + model: { provider: "faux", model: "fixture" }, + }; + }, + }); + + const result = await reflector.reflect(resumed); + + expect(targetAttempts).toHaveLength(1); + expect(targetAttempts[0]).toContain("attempt-2"); + expect(result.nodes.find((node) => node.evidenceIds.includes("evidence-user-1"))).toMatchObject({ + status: "failed", + lessonIds: [], + }); + expect(result.finalLessonIds).toHaveLength(1); + }); + it.each(["model", "slice-tool"] as const)( "resumes with the same logical call after a %s interruption", async (interruptionPoint) => { @@ -771,7 +874,7 @@ describe("recursive reflector", () => { }, ); - it("retries invalid structured output within the model-call budget", async () => { + it("checkpoints a leaf after two invalid executor results", async () => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( { version: "1.0.0", artifacts: persistence, ledger: persistence }, @@ -786,32 +889,24 @@ describe("recursive reflector", () => { }, }); - await expect(reflector.reflect(input)).rejects.toMatchObject({ - reasonCode: "reflection-invalid-response", - retryable: false, - diagnostics: { - node: { kind: "leaf", evidenceIds: expect.any(Array) }, - invalidAliases: ["evidence-user-l"], - permittedAliases: ["E1", "E2", "E3", "E4"], - attemptDetails: [ - { - submissionMethod: "unspecified", - validationError: - "supportingEvidenceRefs contains an unknown reference. Permitted references: E1, E2, E3, E4", - output: { complete: true, lessons: [leafLesson(["evidence-user-l"])] }, - }, - { - submissionMethod: "unspecified", - validationError: - "supportingEvidenceRefs contains an unknown reference. Permitted references: E1, E2, E3, E4", - output: { complete: true, lessons: [leafLesson(["evidence-user-l"])] }, - }, - ], - }, - }); + const result = await reflector.reflect(input); + + expect(result.nodes.filter((node) => node.kind === "leaf")).toHaveLength(2); + expect(result.nodes.filter((node) => node.kind === "leaf").every((node) => node.status === "failed")).toBe(true); + expect(result.nodes.at(-1)).toMatchObject({ kind: "synthesis", status: "complete", lessonIds: [] }); const events = await persistence.read("job-1"); expect(events.filter((event) => event.type === "reflector-model-call")).toHaveLength(4); - expect(events.filter((event) => event.type === "reflection-node-checkpointed")).toHaveLength(0); + expect(events.filter((event) => event.type === "reflection-leaf-retry-pending")).toHaveLength(2); + expect(events.filter((event) => event.type === "reflection-node-checkpointed")).toHaveLength(3); + + const resumed = await reflector.reflect( + reflectInput({ + execute() { + throw new Error("terminal leaf checkpoints must not execute again"); + }, + }), + ); + expect(resumed.nodes.map((node) => node.status)).toEqual(result.nodes.map((node) => node.status)); }); it("normalizes permitted bare evidence aliases and audits each correction", async () => { @@ -875,34 +970,37 @@ describe("recursive reflector", () => { validationError: 'supportingEvidenceRefs contains an unknown reference; "3" omitted the required type prefix and would be "E3", which is not permitted. Permitted references: E1, E2', }, - ])("continues rejecting $caseName", async ({ references: invalidReferences, validationError }) => { + ])("checkpoints $caseName after the fresh leaf attempt", async ({ references: invalidReferences }) => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( { version: "1.0.0", artifacts: persistence, ledger: persistence }, - { maxResponseRetries: 1, maxUnitsPerSlice: 2, synthesisFanout: 8 }, + { maxUnitsPerSlice: 2, synthesisFanout: 8 }, ); const input = reflectInput({ - execute() { + execute(request) { + const node = jsonObject(request.input.node, "node"); return Promise.resolve({ - output: { complete: true, lessons: [leafLesson(invalidReferences)] }, + output: { + complete: true, + lessons: [node.kind === "leaf" ? leafLesson(invalidReferences) : lessonForRequest(request.input)], + }, model: { provider: "faux", model: "fixture" }, }); }, }); - await expect(reflector.reflect(input)).rejects.toMatchObject({ - reasonCode: "reflection-invalid-response", - diagnostics: { attemptDetails: [{ validationError }] }, - }); + const result = await reflector.reflect(input); + expect(result.nodes.some((node) => node.kind === "leaf" && node.status === "failed")).toBe(true); }); - it("explains an invalid response when retrying the same reflection node", async () => { + it("starts a fresh transcript for a leaf retry after executor validation is exhausted", async () => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( { version: "1.0.0", artifacts: persistence, ledger: persistence }, { maxUnitsPerSlice: 1_000 }, ); let targetCalls = 0; + let staleAttemptTool: EvolutionModelTool | null = null; const input = reflectInput({ async execute(request) { if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); @@ -911,21 +1009,19 @@ describe("recursive reflector", () => { if (evidenceRefs.includes("E1")) targetCalls += 1; if (targetCalls === 1 && evidenceRefs.includes("E1")) { expect(request.input.control).not.toHaveProperty("responseCorrection"); + staleAttemptTool = request.tools.find((tool) => tool.name === "search_evidence") ?? null; return { output: { complete: true, lessons: [leafLesson(["99"])] }, model: { provider: "faux", model: "fixture" }, }; } if (targetCalls === 2 && evidenceRefs.includes("E1")) { - expect(request.input.control).toMatchObject({ - responseCorrection: { - reason: - 'supportingEvidenceRefs contains an unknown reference; "99" omitted the required type prefix and would be "E99", which is not permitted. Permitted references: E1, E2, E3, E4', - }, - }); - } else { - expect(request.input.control).not.toHaveProperty("responseCorrection"); + expect(request.idempotencyKey).toContain("attempt-2"); + const stale = staleAttemptTool as EvolutionModelTool | null; + if (stale === null) throw new Error("Expected the first attempt tool"); + await expect(stale.execute({ query: "evidence" })).rejects.toThrow("attempt ended"); } + expect(request.input.control).not.toHaveProperty("responseCorrection"); return { output: { complete: true, lessons: [lessonForRequest(request.input)] }, model: { provider: "faux", model: "fixture" }, @@ -934,17 +1030,19 @@ describe("recursive reflector", () => { }); await expect(reflector.reflect(input)).resolves.toMatchObject({ nodes: expect.any(Array) }); + expect(targetCalls).toBe(2); }); - it("stops after two incomplete refinement rounds by default", async () => { + it("publishes a complete-only schema and checkpoints executors that violate it", async () => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( { version: "1.0.0", artifacts: persistence, ledger: persistence }, - { maxUnitsPerSlice: 1_000, maxResponseRetries: 1 }, + { maxUnitsPerSlice: 1_000 }, ); const input = reflectInput({ async execute(request) { if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + expect(jsonObject(request.outputSchema.properties, "schema properties").complete).toEqual({ enum: [true] }); return { output: { complete: false, lessons: [lessonForRequest(request.input)] }, model: { provider: "faux", model: "fixture" }, @@ -952,8 +1050,11 @@ describe("recursive reflector", () => { }, }); - await expect(reflector.reflect(input)).rejects.toMatchObject({ - reasonCode: "reflection-budget-exhausted", + const result = await reflector.reflect(input); + + expect(result.nodes[0]).toMatchObject({ + status: "failed", + failure: { diagnosticCode: "reflection-invalid-response" }, }); expect((await persistence.read("job-1")).filter((event) => event.type === "reflector-model-call")).toHaveLength( 4, @@ -971,11 +1072,9 @@ describe("recursive reflector", () => { if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); const node = jsonObject(request.input.node, "node"); if (node.kind === "leaf") { - expect(request.input.control).toMatchObject({ - maximumToolCalls: 6, - maximumRefinementRounds: 2, - structuredResponseRetries: 2, - }); + expect(request.input.control).toMatchObject({ maximumToolCalls: 6 }); + expect(request.input.control).not.toHaveProperty("maximumRefinementRounds"); + expect(request.input.control).not.toHaveProperty("structuredResponseRetries"); const search = request.tools.find((tool) => tool.name === "search_evidence"); for (let count = 0; count < 6; count += 1) await search?.execute({ query: "evidence" }); await expect(search?.execute({ query: "evidence" })).rejects.toThrow("node tool-call budget"); From 1a125cfef8f073ac27bb6120ebac3880647d85eb Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 24 Aug 2026 19:36:16 -0400 Subject: [PATCH 4/8] feat(agent): expose evolution provider call allowance - Publish each executor's bounded provider-call capacity. - Cover the coding-agent executor allowance in runtime tests. --- packages/coding-agent/src/core/evolution-runtime.ts | 1 + packages/coding-agent/test/evolution-runtime.test.ts | 1 + packages/evolution/src/ports.ts | 1 + packages/evolution/test/control-plane.test.ts | 1 + packages/evolution/test/curator.test.ts | 2 +- packages/evolution/test/proposal-ranking.test.ts | 5 ++++- packages/evolution/test/reflector.test.ts | 7 +++++-- 7 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 6d41aca01..a081789c0 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -176,6 +176,7 @@ class SessionPriorityGate implements InteractivePriorityGate { } export class CodingAgentEvolutionModelExecutor implements EvolutionModelExecutor { + readonly maxProviderCallsPerExecution = MAX_MODEL_ROUNDS * 2; readonly #session: AgentSession; readonly #modelRuntime: ModelRuntime; readonly #priority: InteractivePriorityGate; diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index 5f1ade0d4..d69740c6b 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -541,6 +541,7 @@ describe("coding-agent evolution runtime", () => { const model = harness.getModel(); let providerCalls = 0; + expect(executor.maxProviderCallsPerExecution).toBe(20); await expect( executor.execute({ stage: "reflector", diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 846632b36..1345698bf 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -110,6 +110,7 @@ export interface EvolutionModelResponse { } export interface EvolutionModelExecutor { + readonly maxProviderCallsPerExecution: number; execute(request: EvolutionModelRequest): Promise; } diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index 52dd28138..349ae7d86 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -516,6 +516,7 @@ describe("checkpointed evolution pipeline", () => { const pipeline = new EvolutionPipeline({ version: "1.0.0", model: { + maxProviderCallsPerExecution: 20, async execute(request) { modelKeys.push(request.idempotencyKey); return { output: { accepted: true }, model: { provider: "faux", model: "fixture" } }; diff --git a/packages/evolution/test/curator.test.ts b/packages/evolution/test/curator.test.ts index 4fac1d906..4a82d7a27 100644 --- a/packages/evolution/test/curator.test.ts +++ b/packages/evolution/test/curator.test.ts @@ -249,7 +249,7 @@ function curatorInput(execute: CuratorInput["model"]["execute"]): CuratorInput { corpus: corpus(), instructions: instructions(input), reflection: reflection(), - model: { execute }, + model: { maxProviderCallsPerExecution: 20, execute }, idempotencyKey: "job-1:curating", }; } diff --git a/packages/evolution/test/proposal-ranking.test.ts b/packages/evolution/test/proposal-ranking.test.ts index b980190a4..001b7eb36 100644 --- a/packages/evolution/test/proposal-ranking.test.ts +++ b/packages/evolution/test/proposal-ranking.test.ts @@ -585,7 +585,10 @@ describe("deterministic proposal evidence ranking", () => { const ranker = new DeterministicProposalRanker({ version: "1" }, { eligibilityThreshold: 100 }); const pipeline = new EvolutionPipeline({ version: "1", - model: { execute: () => Promise.reject(new Error("Model execution is not expected")) }, + model: { + maxProviderCallsPerExecution: 20, + execute: () => Promise.reject(new Error("Model execution is not expected")), + }, reflector: { reflect: () => Promise.reject(new Error("Reflection is not expected")) }, curator: { curate: () => Promise.reject(new Error("Curation is not expected")) }, validator: ranker, diff --git a/packages/evolution/test/reflector.test.ts b/packages/evolution/test/reflector.test.ts index 9ed2c3f6a..1cea50ca1 100644 --- a/packages/evolution/test/reflector.test.ts +++ b/packages/evolution/test/reflector.test.ts @@ -193,7 +193,10 @@ async function reflectorFixture() { }; } -function reflectInput(model: ReflectorInput["model"]): ReflectorInput { +function reflectInput( + model: Pick & + Partial>, +): ReflectorInput { const { input, corpus } = fixtureInput(); return { jobId: "job-1", @@ -201,7 +204,7 @@ function reflectInput(model: ReflectorInput["model"]): ReflectorInput { corpus, counterReport: null, instructions: instructionSnapshot(), - model, + model: { maxProviderCallsPerExecution: 20, ...model }, idempotencyKey: "job-1:reflecting", }; } From 30193723f74a53446a21121b1acf51f19b1b54c4 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 24 Aug 2026 19:42:44 -0400 Subject: [PATCH 5/8] feat(agent): derive reflector budgets from remaining work - Reserve model and tool capacity for unfinished leaf attempts and synthesis nodes. - Rebuild envelopes from durable checkpoints and fail fast on undersized ceilings. --- packages/coding-agent/CHANGELOG.md | 1 + packages/evolution/src/reflector-tools.ts | 15 ++- packages/evolution/src/reflector.ts | 101 +++++++++++++-- packages/evolution/test/reflector.test.ts | 149 +++++++++++++++++++++- 4 files changed, 244 insertions(+), 22 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index b1cead607..96ba78601 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,6 +13,7 @@ ### Fixed +- Fixed evolution reflection budgets to reserve bounded model and tool capacity for every unfinished leaf attempt and synthesis node. - Fixed evolution reflection nodes losing tool and denial history between response corrections, and continued synthesis past terminal leaf failures. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). diff --git a/packages/evolution/src/reflector-tools.ts b/packages/evolution/src/reflector-tools.ts index c5cd010ca..ac77b599b 100644 --- a/packages/evolution/src/reflector-tools.ts +++ b/packages/evolution/src/reflector-tools.ts @@ -204,7 +204,7 @@ export class ReflectorToolRegistry { readonly #slices = new Map(); readonly #sliceReferences = new Map(); readonly #sensitiveReplacements = new Map(); - readonly #maxToolCalls: number; + #maxToolCalls: number; readonly #signal?: AbortSignal; readonly #audit: (event: ReflectorToolAudit) => Promise; readonly #recoverableDenials: ReflectorRecoverableToolDenial[] = []; @@ -297,6 +297,13 @@ export class ReflectorToolRegistry { return this.#recoverableDenials; } + setMaxToolCalls(maxToolCalls: number): void { + if (!Number.isSafeInteger(maxToolCalls) || maxToolCalls < this.#toolCalls) { + throw new Error("maxToolCalls must be a safe integer no lower than consumed tool calls"); + } + this.#maxToolCalls = maxToolCalls; + } + toolBudgetExhausted(tools: readonly EvolutionModelTool[]): boolean { const budget = this.#toolBudgets.get(tools); return budget !== undefined && budget.toolCalls > budget.maxToolCalls; @@ -535,15 +542,15 @@ export class ReflectorToolRegistry { if (this.#revoked) throw new EvolutionToolDeniedError("registry-revoked", "Reflector tool registry is revoked"); this.#signal?.throwIfAborted(); - if (this.#toolCalls > this.#maxToolCalls) { - throw new EvolutionToolDeniedError("tool-budget-exhausted", "Reflector tool-call budget is exhausted"); - } if (budget.toolCalls > budget.maxToolCalls) { throw new EvolutionToolDeniedError( "tool-budget-exhausted", "Reflection node tool-call budget is exhausted", ); } + if (this.#toolCalls > this.#maxToolCalls) { + throw new EvolutionToolDeniedError("tool-budget-exhausted", "Reflector tool-call budget is exhausted"); + } const result = run(); resolvedDocumentIds = result.resolvedDocumentIds; resolvedEvidenceIds = result.resolvedEvidenceIds; diff --git a/packages/evolution/src/reflector.ts b/packages/evolution/src/reflector.ts index 554124996..ff4d030c8 100644 --- a/packages/evolution/src/reflector.ts +++ b/packages/evolution/src/reflector.ts @@ -31,8 +31,6 @@ import type { import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; import { type ReflectorNodeReferences, type ReflectorToolAudit, ReflectorToolRegistry } from "./reflector-tools.ts"; -const DEFAULT_MAX_TOOL_CALLS = 200; -const DEFAULT_MAX_MODEL_CALLS = 100; const DEFAULT_MAX_RECURSION_DEPTH = 8; const DEFAULT_MAX_EVIDENCE_TOOL_CALLS_PER_LEAF = 6; const DEFAULT_MAX_LESSON_TOOL_CALLS_PER_SYNTHESIS_NODE = 2; @@ -95,6 +93,11 @@ interface CompletedNode { readonly lessons: readonly Lesson[]; } +interface RemainingReflectionBudget { + readonly modelCalls: number; + readonly toolCalls: number; +} + interface MutableNodeRuntime { providerCalls: number; providerLatencyMs: number; @@ -460,8 +463,8 @@ async function runBounded( export class RecursiveReflector implements Reflector { readonly #dependencies: RecursiveReflectorDependencies; - readonly #maxToolCalls: number; - readonly #maxModelCalls: number; + readonly #maxToolCalls: number | null; + readonly #maxModelCalls: number | null; readonly #maxRecursionDepth: number; readonly #maxEvidenceToolCallsPerLeaf: number; readonly #maxLessonToolCallsPerSynthesisNode: number; @@ -475,8 +478,10 @@ export class RecursiveReflector implements Reflector { constructor(dependencies: RecursiveReflectorDependencies, options: RecursiveReflectorOptions = {}) { this.#dependencies = dependencies; - this.#maxToolCalls = positiveInteger(options.maxToolCalls, DEFAULT_MAX_TOOL_CALLS, "maxToolCalls"); - this.#maxModelCalls = positiveInteger(options.maxModelCalls, DEFAULT_MAX_MODEL_CALLS, "maxModelCalls"); + this.#maxToolCalls = + options.maxToolCalls === undefined ? null : positiveInteger(options.maxToolCalls, 1, "maxToolCalls"); + this.#maxModelCalls = + options.maxModelCalls === undefined ? null : positiveInteger(options.maxModelCalls, 1, "maxModelCalls"); this.#maxRecursionDepth = positiveInteger( options.maxRecursionDepth, DEFAULT_MAX_RECURSION_DEPTH, @@ -525,12 +530,11 @@ export class RecursiveReflector implements Reflector { const priorEvents = await this.#dependencies.ledger.read(input.jobId); const priorToolCalls = priorEvents.filter((event) => event.type === "reflector-tool-call").length; const priorModelCalls = priorEvents.filter((event) => event.type === "reflector-model-call").length; - if (priorToolCalls >= this.#maxToolCalls || priorModelCalls >= this.#maxModelCalls) { - throw new EvolutionExecutionError("Reflector budget was exhausted by prior attempts", { - reasonCode: "reflection-budget-exhausted", - retryable: false, - }); - } + const providerCallAllowance = positiveInteger( + input.model.maxProviderCallsPerExecution, + 1, + "maxProviderCallsPerExecution", + ); const producer: EvolutionArtifactProducer = { name: "diffex-evolution-reflector", version: this.#dependencies.version, @@ -541,7 +545,7 @@ export class RecursiveReflector implements Reflector { jobId: input.jobId, input: input.input, corpus: input.corpus, - maxToolCalls: this.#maxToolCalls - priorToolCalls, + maxToolCalls: Number.MAX_SAFE_INTEGER, signal: stageSignal, audit: (audit) => this.#auditTool(input, producer, audit), }); @@ -553,6 +557,16 @@ export class RecursiveReflector implements Reflector { const completed = await this.#loadCompleted(input, plans, priorEvents); const retryPending = this.#loadRetryPending(input, plans, priorEvents, completed); let modelCalls = priorModelCalls; + let effectiveModelCallLimit = priorModelCalls; + const refreshBudgetEnvelope = () => { + const remaining = this.#remainingBudget(plans, completed, retryPending, providerCallAllowance); + const toolCalls = priorToolCalls + registry.toolCalls; + this.#assertBudgetFits(remaining, modelCalls, toolCalls); + effectiveModelCallLimit = this.#maxModelCalls ?? modelCalls + remaining.modelCalls; + const effectiveToolCallLimit = this.#maxToolCalls ?? toolCalls + remaining.toolCalls; + registry.setMaxToolCalls(Math.max(registry.toolCalls, effectiveToolCallLimit - priorToolCalls)); + }; + refreshBudgetEnvelope(); const executePlan = async (plan: NodePlan) => { if (stageDeadline.expired) { throw new EvolutionExecutionError("Reflector stage active-compute deadline was exhausted", { @@ -574,6 +588,7 @@ export class RecursiveReflector implements Reflector { plan.nodeId, await this.#checkpointEmptySynthesis(input, producer, plan, unavailableChildNodeIds), ); + refreshBudgetEnvelope(); return; } const pending = retryPending.get(plan.nodeId); @@ -604,7 +619,7 @@ export class RecursiveReflector implements Reflector { nodeAttempt, priorAttemptModelCalls, () => { - if (modelCalls >= this.#maxModelCalls) { + if (modelCalls >= effectiveModelCallLimit) { throw new EvolutionExecutionError("Reflector model-call budget is exhausted", { reasonCode: "reflection-budget-exhausted", retryable: false, @@ -631,6 +646,7 @@ export class RecursiveReflector implements Reflector { if (plan.kind === "leaf" && nodeAttempt === 1 && (timedOut || this.#isRetryableLeafFailure(error))) { const retry = await this.#checkpointRetryPending(input, producer, plan, failure, runtime); retryPending.set(plan.nodeId, retry); + refreshBudgetEnvelope(); runtime = newNodeRuntime(retry.runtime); nodeAttempt = 2; continue; @@ -656,6 +672,7 @@ export class RecursiveReflector implements Reflector { } } completed.set(plan.nodeId, result); + refreshBudgetEnvelope(); }; await runBounded( plans.filter((plan) => plan.kind === "leaf"), @@ -731,6 +748,62 @@ export class RecursiveReflector implements Reflector { return plans; } + #remainingBudget( + plans: readonly NodePlan[], + completed: ReadonlyMap, + retryPending: ReadonlyMap, + providerCallAllowance: number, + ): RemainingReflectionBudget { + let modelCalls = 0; + let toolCalls = 0; + for (const plan of plans) { + if (completed.has(plan.nodeId)) continue; + let conversations: number; + let toolCallAllowance: number; + if (plan.kind === "leaf") { + conversations = retryPending.has(plan.nodeId) ? 1 : 2; + toolCallAllowance = this.#maxEvidenceToolCallsPerLeaf; + } else { + const childrenTerminal = plan.childNodeIds.every((nodeId) => completed.has(nodeId)); + const hasSuccessfulChildLessons = plan.childNodeIds.some( + (nodeId) => (completed.get(nodeId)?.lessons.length ?? 0) > 0, + ); + conversations = childrenTerminal && !hasSuccessfulChildLessons ? 0 : 1; + toolCallAllowance = this.#maxLessonToolCallsPerSynthesisNode; + } + modelCalls += conversations * providerCallAllowance; + toolCalls += conversations * toolCallAllowance; + if (!Number.isSafeInteger(modelCalls) || !Number.isSafeInteger(toolCalls)) { + throw new EvolutionExecutionError("Reflector remaining-plan budget exceeds safe integer limits", { + reasonCode: "reflection-budget-exhausted", + retryable: false, + }); + } + } + return { modelCalls, toolCalls }; + } + + #assertBudgetFits(remaining: RemainingReflectionBudget, modelCalls: number, toolCalls: number): void { + const availableModelCalls = + this.#maxModelCalls === null ? remaining.modelCalls : Math.max(0, this.#maxModelCalls - modelCalls); + const availableToolCalls = + this.#maxToolCalls === null ? remaining.toolCalls : Math.max(0, this.#maxToolCalls - toolCalls); + if (availableModelCalls >= remaining.modelCalls && availableToolCalls >= remaining.toolCalls) return; + throw new EvolutionExecutionError( + `Reflector remaining plan requires ${remaining.modelCalls} model calls and ${remaining.toolCalls} tool calls, but only ${availableModelCalls} model calls and ${availableToolCalls} tool calls are available`, + { + reasonCode: "reflection-budget-exhausted", + retryable: false, + diagnostics: { + required: { modelCalls: remaining.modelCalls, toolCalls: remaining.toolCalls }, + available: { modelCalls: availableModelCalls, toolCalls: availableToolCalls }, + consumed: { modelCalls, toolCalls }, + ceilings: { modelCalls: this.#maxModelCalls, toolCalls: this.#maxToolCalls }, + }, + }, + ); + } + async #loadCompleted( input: ReflectorInput, plans: readonly NodePlan[], diff --git a/packages/evolution/test/reflector.test.ts b/packages/evolution/test/reflector.test.ts index 1cea50ca1..1776a36b2 100644 --- a/packages/evolution/test/reflector.test.ts +++ b/packages/evolution/test/reflector.test.ts @@ -62,7 +62,7 @@ function sessionEvidence( }; } -function invocation(): MaterializedInvocation { +function invocation(evidenceCount = 6): MaterializedInvocation { const evidence: MaterializedEvidence[] = [ sessionEvidence("evidence-user-1", 0, { role: "user", @@ -83,6 +83,14 @@ function invocation(): MaterializedInvocation { sessionEvidence("evidence-user-2", 4, { role: "user", content: "Verify the result." }), sessionEvidence("evidence-verified", 5, { role: "assistant", content: "The focused test passed." }), ]; + for (let position = evidence.length; position < evidenceCount; position += 1) { + evidence.push( + sessionEvidence(`evidence-extra-${position}`, position, { + role: "assistant", + content: `Additional verified result ${position}`, + }), + ); + } return { version: 1, tracePath: "/private/trace.jsonl", @@ -106,8 +114,11 @@ function invocation(): MaterializedInvocation { }; } -function fixtureInput(): { readonly input: EvolutionInputSnapshot; readonly corpus: EvidenceCorpusManifest } { - const materialized = invocation(); +function fixtureInput(evidenceCount = 6): { + readonly input: EvolutionInputSnapshot; + readonly corpus: EvidenceCorpusManifest; +} { + const materialized = invocation(evidenceCount); const contentDigest = computeEvolutionContentDigest(materialized); const source = { sourceId: "source-1", @@ -196,8 +207,9 @@ async function reflectorFixture() { function reflectInput( model: Pick & Partial>, + evidenceCount = 6, ): ReflectorInput { - const { input, corpus } = fixtureInput(); + const { input, corpus } = fixtureInput(evidenceCount); return { jobId: "job-1", input, @@ -397,6 +409,74 @@ describe("recursive reflector", () => { expect(result.nodes.every((node) => (node.runtime?.nodeDurationMs ?? -1) >= 0)).toBe(true); }); + it("derives default envelopes for every remaining leaf attempt and synthesis node", async () => { + const { persistence } = await reflectorFixture(); + const reflector = new RecursiveReflector( + { version: "1.0.0", artifacts: persistence, ledger: persistence }, + { maxUnitsPerSlice: 1, synthesisFanout: 8 }, + ); + let providerCalls = 0; + const input = reflectInput( + { + maxProviderCallsPerExecution: 10, + async execute(request) { + if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + for (let call = 0; call < 3; call += 1) { + await request.beforeProviderCall?.(); + providerCalls += 1; + } + return { + output: { complete: true, lessons: [lessonForRequest(request.input)] }, + model: { provider: "faux", model: "fixture" }, + }; + }, + }, + 40, + ); + + const result = await reflector.reflect(input); + + expect(result.nodes.filter((node) => node.kind === "leaf")).toHaveLength(40); + expect(result.nodes.filter((node) => node.kind === "synthesis")).toHaveLength(6); + expect(providerCalls).toBe(138); + expect((await persistence.read("job-1")).filter((event) => event.type === "reflector-model-call")).toHaveLength( + 138, + ); + }); + + it.each([ + { maxModelCalls: 859, maxToolCalls: 492 }, + { maxModelCalls: 860, maxToolCalls: 491 }, + ])("fails before work when explicit ceilings cannot fit the remaining plan", async (ceilings) => { + const { persistence } = await reflectorFixture(); + const reflector = new RecursiveReflector( + { version: "1.0.0", artifacts: persistence, ledger: persistence }, + { ...ceilings, maxUnitsPerSlice: 1, synthesisFanout: 8 }, + ); + let executions = 0; + const input = reflectInput( + { + maxProviderCallsPerExecution: 10, + execute() { + executions += 1; + throw new Error("Model execution must not start"); + }, + }, + 40, + ); + + await expect(reflector.reflect(input)).rejects.toMatchObject({ + reasonCode: "reflection-budget-exhausted", + diagnostics: { + required: { modelCalls: 860, toolCalls: 492 }, + available: { modelCalls: ceilings.maxModelCalls, toolCalls: ceilings.maxToolCalls }, + consumed: { modelCalls: 0, toolCalls: 0 }, + }, + }); + expect(executions).toBe(0); + expect(await persistence.read("job-1")).toEqual([]); + }); + it("checkpoints timed-out nodes and continues independent leaves", async () => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( @@ -823,6 +903,67 @@ describe("recursive reflector", () => { expect(result.finalLessonIds).toHaveLength(1); }); + it("reconstructs a tight envelope from durable retry state after restart", async () => { + const { persistence } = await reflectorFixture(); + const firstReflector = new RecursiveReflector( + { version: "1.0.0", artifacts: persistence, ledger: persistence }, + { maxUnitsPerSlice: 2, synthesisFanout: 8 }, + ); + const controller = new AbortController(); + let hangingExecutions = 0; + let signalAllHanging: (() => void) | undefined; + const allHanging = new Promise((resolveHanging) => { + signalAllHanging = resolveHanging; + }); + const firstRun = reflectInput({ + maxProviderCallsPerExecution: 10, + async execute(request) { + if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + const node = jsonObject(request.input.node, "node"); + const target = node.kind === "leaf" && references(request.input, "evidence", "evidenceRef").includes("E1"); + if (target && request.idempotencyKey.endsWith("attempt-1")) { + throw new Error("retryable leaf failure"); + } + hangingExecutions += 1; + if (hangingExecutions === 2) signalAllHanging?.(); + return new Promise(() => {}); + }, + }); + const interrupted = firstReflector.reflect({ ...firstRun, signal: controller.signal }); + await allHanging; + controller.abort(new Error("simulated worker restart")); + await expect(interrupted).rejects.toThrow("simulated worker restart"); + expect( + (await persistence.read("job-1")).filter((event) => event.type === "reflection-leaf-retry-pending"), + ).toHaveLength(1); + + const resumedReflector = new RecursiveReflector( + { version: "1.0.0", artifacts: persistence, ledger: persistence }, + { maxModelCalls: 61, maxToolCalls: 32, maxUnitsPerSlice: 2, synthesisFanout: 8 }, + ); + const targetAttempts: string[] = []; + const resumed = reflectInput({ + maxProviderCallsPerExecution: 10, + async execute(request) { + if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + const node = jsonObject(request.input.node, "node"); + if (node.kind === "leaf" && references(request.input, "evidence", "evidenceRef").includes("E1")) { + targetAttempts.push(request.idempotencyKey); + } + return { + output: { complete: true, lessons: [lessonForRequest(request.input)] }, + model: { provider: "faux", model: "fixture" }, + }; + }, + }); + + const result = await resumedReflector.reflect(resumed); + + expect(result.nodes).toHaveLength(4); + expect(targetAttempts).toHaveLength(1); + expect(targetAttempts[0]).toContain("attempt-2"); + }); + it.each(["model", "slice-tool"] as const)( "resumes with the same logical call after a %s interruption", async (interruptionPoint) => { From 96570ad5c30ba51feed66e624b4e6b106f2d9476 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 24 Aug 2026 22:19:24 -0400 Subject: [PATCH 6/8] fix(agent): leave changelog updates to release workflow - Remove evolution entries added by this pull request. - Keep generated release records unchanged. --- packages/coding-agent/CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 96ba78601..8a05e8c9d 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,8 +13,6 @@ ### Fixed -- Fixed evolution reflection budgets to reserve bounded model and tool capacity for every unfinished leaf attempt and synthesis node. -- Fixed evolution reflection nodes losing tool and denial history between response corrections, and continued synthesis past terminal leaf failures. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). - Fixed right-click not pasting clipboard text in fullscreen mode on Windows. From c86827434137be7e7ebf5c1151a5c023380e2392 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 24 Aug 2026 22:58:15 -0400 Subject: [PATCH 7/8] fix(agent): preserve reflection availability and budgets --- packages/evolution/src/reflector-tools.ts | 8 +- packages/evolution/src/reflector.ts | 63 +++++++++- packages/evolution/test/reflector.test.ts | 135 +++++++++++++++++----- 3 files changed, 171 insertions(+), 35 deletions(-) diff --git a/packages/evolution/src/reflector-tools.ts b/packages/evolution/src/reflector-tools.ts index ac77b599b..971b7e061 100644 --- a/packages/evolution/src/reflector-tools.ts +++ b/packages/evolution/src/reflector-tools.ts @@ -73,6 +73,7 @@ export interface ReflectorRecoverableToolDenial { interface NodeToolBudget { readonly maxToolCalls: number; readonly signal?: AbortSignal; + readonly onToolCall?: () => void; toolCalls: number; } @@ -359,8 +360,9 @@ export class ReflectorToolRegistry { references: ReflectorNodeReferences, maxToolCalls: number, signal?: AbortSignal, + onToolCall?: () => void, ): readonly EvolutionModelTool[] { - const budget: NodeToolBudget = { maxToolCalls, signal, toolCalls: 0 }; + const budget: NodeToolBudget = { maxToolCalls, signal, onToolCall, toolCalls: 0 }; const permittedEvidenceIds = new Set(references.evidence.entries.map((entry) => entry.value.evidence.evidenceId)); const definitions: readonly [Exclude, string, JsonObject][] = [ [ @@ -407,8 +409,9 @@ export class ReflectorToolRegistry { lessons: EvolutionModelReferenceCatalog, maxToolCalls: number, signal?: AbortSignal, + onToolCall?: () => void, ): readonly EvolutionModelTool[] { - const budget: NodeToolBudget = { maxToolCalls, signal, toolCalls: 0 }; + const budget: NodeToolBudget = { maxToolCalls, signal, onToolCall, toolCalls: 0 }; const tools: readonly EvolutionModelTool[] = [ { name: "inspect_child_lessons", @@ -528,6 +531,7 @@ export class ReflectorToolRegistry { if (!this.#revoked) budget.signal?.throwIfAborted(); this.#toolCalls += 1; budget.toolCalls += 1; + budget.onToolCall?.(); const toolCallId = `reflector-tool-${this.#toolCalls}`; const requestedDocumentRef = typeof input.documentRef === "string" ? input.documentRef : null; const requestedSliceRef = typeof input.sliceRef === "string" ? input.sliceRef : null; diff --git a/packages/evolution/src/reflector.ts b/packages/evolution/src/reflector.ts index ff4d030c8..948121b32 100644 --- a/packages/evolution/src/reflector.ts +++ b/packages/evolution/src/reflector.ts @@ -98,6 +98,11 @@ interface RemainingReflectionBudget { readonly toolCalls: number; } +interface MutableAttemptConsumption { + modelCalls: number; + toolCalls: number; +} + interface MutableNodeRuntime { providerCalls: number; providerLatencyMs: number; @@ -556,10 +561,17 @@ export class RecursiveReflector implements Reflector { const plans = this.#buildPlan(input, slices); const completed = await this.#loadCompleted(input, plans, priorEvents); const retryPending = this.#loadRetryPending(input, plans, priorEvents, completed); + const attemptConsumption = new Map(); let modelCalls = priorModelCalls; let effectiveModelCallLimit = priorModelCalls; const refreshBudgetEnvelope = () => { - const remaining = this.#remainingBudget(plans, completed, retryPending, providerCallAllowance); + const remaining = this.#remainingBudget( + plans, + completed, + retryPending, + attemptConsumption, + providerCallAllowance, + ); const toolCalls = priorToolCalls + registry.toolCalls; this.#assertBudgetFits(remaining, modelCalls, toolCalls); effectiveModelCallLimit = this.#maxModelCalls ?? modelCalls + remaining.modelCalls; @@ -586,7 +598,19 @@ export class RecursiveReflector implements Reflector { if (plan.kind === "synthesis" && childLessons.length === 0) { completed.set( plan.nodeId, - await this.#checkpointEmptySynthesis(input, producer, plan, unavailableChildNodeIds), + unavailableChildNodeIds.length === 0 + ? await this.#checkpointEmptySynthesis(input, producer, plan, unavailableChildNodeIds) + : await this.#checkpointUnavailable( + input, + producer, + plan, + "failed", + { + diagnosticCode: "child-nodes-unavailable", + message: "All child reflection lessons are unavailable", + }, + newNodeRuntime(), + ), ); refreshBudgetEnvelope(); return; @@ -603,6 +627,8 @@ export class RecursiveReflector implements Reflector { event.attempt === nodeAttempt, ).length; runtime.providerCalls += priorAttemptModelCalls; + const consumption = { modelCalls: priorAttemptModelCalls, toolCalls: 0 }; + attemptConsumption.set(plan.nodeId, consumption); const nodeDeadline = new ActiveComputeDeadline( this.#nodeDeadlineMs, `Reflection node ${plan.nodeId} attempt ${nodeAttempt}`, @@ -630,6 +656,7 @@ export class RecursiveReflector implements Reflector { stageDeadline, nodeDeadline, runtime, + consumption, ); break; } catch (error) { @@ -645,6 +672,7 @@ export class RecursiveReflector implements Reflector { const failure = this.#nodeFailure(error, timedOut); if (plan.kind === "leaf" && nodeAttempt === 1 && (timedOut || this.#isRetryableLeafFailure(error))) { const retry = await this.#checkpointRetryPending(input, producer, plan, failure, runtime); + attemptConsumption.delete(plan.nodeId); retryPending.set(plan.nodeId, retry); refreshBudgetEnvelope(); runtime = newNodeRuntime(retry.runtime); @@ -671,6 +699,7 @@ export class RecursiveReflector implements Reflector { nodeDeadline.dispose(); } } + attemptConsumption.delete(plan.nodeId); completed.set(plan.nodeId, result); refreshBudgetEnvelope(); }; @@ -687,6 +716,16 @@ export class RecursiveReflector implements Reflector { .filter((node): node is ReflectionNode => node !== undefined); const lessons = plans.flatMap((plan) => completed.get(plan.nodeId)?.lessons ?? []); const rootPlans = plans.filter((plan) => plan.parentNodeId === null); + const unavailableRootNodeIds = rootPlans + .filter((plan) => completed.get(plan.nodeId)?.node.status !== "complete") + .map((plan) => plan.nodeId); + if (unavailableRootNodeIds.length > 0) { + throw new EvolutionExecutionError("Reflector root is unavailable", { + reasonCode: "model-call-failed", + retryable: true, + diagnostics: { unavailableRootNodeIds }, + }); + } const finalLessonIds = rootPlans.flatMap( (plan) => completed.get(plan.nodeId)?.lessons.map((lesson) => lesson.lessonId) ?? [], ); @@ -752,6 +791,7 @@ export class RecursiveReflector implements Reflector { plans: readonly NodePlan[], completed: ReadonlyMap, retryPending: ReadonlyMap, + attemptConsumption: ReadonlyMap, providerCallAllowance: number, ): RemainingReflectionBudget { let modelCalls = 0; @@ -771,8 +811,9 @@ export class RecursiveReflector implements Reflector { conversations = childrenTerminal && !hasSuccessfulChildLessons ? 0 : 1; toolCallAllowance = this.#maxLessonToolCallsPerSynthesisNode; } - modelCalls += conversations * providerCallAllowance; - toolCalls += conversations * toolCallAllowance; + const consumption = attemptConsumption.get(plan.nodeId); + modelCalls += Math.max(0, conversations * providerCallAllowance - (consumption?.modelCalls ?? 0)); + toolCalls += Math.max(0, conversations * toolCallAllowance - (consumption?.toolCalls ?? 0)); if (!Number.isSafeInteger(modelCalls) || !Number.isSafeInteger(toolCalls)) { throw new EvolutionExecutionError("Reflector remaining-plan budget exceeds safe integer limits", { reasonCode: "reflection-budget-exhausted", @@ -892,14 +933,23 @@ export class RecursiveReflector implements Reflector { stageDeadline: ActiveComputeDeadline, nodeDeadline: ActiveComputeDeadline, runtime: MutableNodeRuntime, + consumption: MutableAttemptConsumption, ): Promise { const nodeReferences = registry.nodeReferences(plan.evidenceIds); const childLessons = plan.childNodeIds.flatMap((nodeId) => completed.get(nodeId)?.lessons ?? []); const lessonReferences = new EvolutionModelReferenceCatalog("lesson", childLessons); + const chargeToolCall = () => { + consumption.toolCalls += 1; + }; const tools = plan.kind === "leaf" - ? registry.toolsForEvidence(nodeReferences, this.#maxEvidenceToolCallsPerLeaf, input.signal) - : registry.toolsForLessons(lessonReferences, this.#maxLessonToolCallsPerSynthesisNode, input.signal); + ? registry.toolsForEvidence(nodeReferences, this.#maxEvidenceToolCallsPerLeaf, input.signal, chargeToolCall) + : registry.toolsForLessons( + lessonReferences, + this.#maxLessonToolCallsPerSynthesisNode, + input.signal, + chargeToolCall, + ); const maxLessons = plan.kind === "leaf" ? this.#maxLessonsPerLeaf @@ -918,6 +968,7 @@ export class RecursiveReflector implements Reflector { const beforeProviderCall = async () => { providerCallHookInvoked = true; chargeModelCall(); + consumption.modelCalls += 1; providerCalls += 1; runtime.providerCalls += 1; nodeModelCallSequence += 1; diff --git a/packages/evolution/test/reflector.test.ts b/packages/evolution/test/reflector.test.ts index 1776a36b2..aff83da90 100644 --- a/packages/evolution/test/reflector.test.ts +++ b/packages/evolution/test/reflector.test.ts @@ -477,6 +477,57 @@ describe("recursive reflector", () => { expect(await persistence.read("job-1")).toEqual([]); }); + it("subtracts concurrent attempt consumption when refreshing explicit ceilings", async () => { + const { persistence } = await reflectorFixture(); + const reflector = new RecursiveReflector( + { version: "1.0.0", artifacts: persistence, ledger: persistence }, + { + maxModelCalls: 5, + maxToolCalls: 5, + maxEvidenceToolCallsPerLeaf: 1, + maxLessonToolCallsPerSynthesisNode: 1, + maxUnitsPerSlice: 1_000, + }, + ); + let firstAttemptsStarted = 0; + let releaseFirstAttempts: (() => void) | undefined; + const bothFirstAttemptsStarted = new Promise((resolve) => { + releaseFirstAttempts = resolve; + }); + const input = reflectInput({ + maxProviderCallsPerExecution: 1, + async execute(request) { + if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); + await request.beforeProviderCall?.(); + const node = jsonObject(request.input.node, "node"); + if (node.kind === "leaf") { + await request.tools[0]?.execute({ query: "evidence" }); + if (request.idempotencyKey.endsWith("attempt-1")) { + firstAttemptsStarted += 1; + if (firstAttemptsStarted === 2) releaseFirstAttempts?.(); + await bothFirstAttemptsStarted; + throw new Error("retry both concurrent leaves"); + } + } else { + await request.tools[0]?.execute({ + lessonRefs: references(request.input, "childLessons", "lessonRef"), + }); + } + return { + output: { complete: true, lessons: [lessonForRequest(request.input)] }, + model: { provider: "faux", model: "fixture" }, + }; + }, + }); + + const result = await reflector.reflect(input); + + expect(result.finalLessonIds).toHaveLength(1); + const events = await persistence.read("job-1"); + expect(events.filter((event) => event.type === "reflector-model-call")).toHaveLength(5); + expect(events.filter((event) => event.type === "reflector-tool-call")).toHaveLength(5); + }); + it("checkpoints timed-out nodes and continues independent leaves", async () => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( @@ -519,13 +570,14 @@ describe("recursive reflector", () => { expect(events.filter((event) => event.type === "reflection-node-checkpointed")).toHaveLength(4); }); - it("checkpoints an empty synthesis branch without blocking successful root lessons", async () => { + it("preserves an unavailable synthesis branch without blocking successful root lessons", async () => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( { version: "1.0.0", artifacts: persistence, ledger: persistence }, { maxUnitsPerSlice: 1, synthesisFanout: 2 }, ); const failedEvidenceRefs = new Set(["E1", "E2"]); + const unavailableChildNodeIdSets: (readonly string[])[] = []; const input = reflectInput({ async execute(request) { if (request.stage !== "reflector") throw new Error("Unexpected Curator request"); @@ -536,6 +588,9 @@ describe("recursive reflector", () => { ) { throw new Error("simulated leaf failure"); } + if (node.kind === "synthesis") { + unavailableChildNodeIdSets.push(request.input.unavailableChildNodeIds as readonly string[]); + } return { output: { complete: true, lessons: [lessonForRequest(request.input)] }, model: { provider: "faux", model: "fixture" }, @@ -546,11 +601,15 @@ describe("recursive reflector", () => { const result = await reflector.reflect(input); expect(result.nodes.filter((node) => node.kind === "leaf" && node.status === "failed")).toHaveLength(2); - expect( - result.nodes.some( - (node) => node.kind === "synthesis" && node.status === "complete" && node.runtime?.providerCalls === 0, - ), - ).toBe(true); + expect(result.nodes).toContainEqual( + expect.objectContaining({ + kind: "synthesis", + status: "failed", + failure: expect.objectContaining({ diagnosticCode: "child-nodes-unavailable" }), + runtime: expect.objectContaining({ providerCalls: 0 }), + }), + ); + expect(unavailableChildNodeIdSets.some((nodeIds) => nodeIds.length === 1)).toBe(true); expect(result.finalLessonIds).toHaveLength(1); expect(result.coverage.entries.slice(0, 2).every((entry) => entry.disposition === "skipped")).toBe(true); expect(result.coverage.entries.slice(2).every((entry) => entry.disposition === "analyzed")).toBe(true); @@ -1018,7 +1077,7 @@ describe("recursive reflector", () => { }, ); - it("checkpoints a leaf after two invalid executor results", async () => { + it("fails reflection after every leaf returns two invalid executor results", async () => { const { persistence } = await reflectorFixture(); const reflector = new RecursiveReflector( { version: "1.0.0", artifacts: persistence, ledger: persistence }, @@ -1033,24 +1092,32 @@ describe("recursive reflector", () => { }, }); - const result = await reflector.reflect(input); + await expect(reflector.reflect(input)).rejects.toMatchObject({ + reasonCode: "model-call-failed", + diagnostics: { unavailableRootNodeIds: [expect.any(String)] }, + }); - expect(result.nodes.filter((node) => node.kind === "leaf")).toHaveLength(2); - expect(result.nodes.filter((node) => node.kind === "leaf").every((node) => node.status === "failed")).toBe(true); - expect(result.nodes.at(-1)).toMatchObject({ kind: "synthesis", status: "complete", lessonIds: [] }); const events = await persistence.read("job-1"); expect(events.filter((event) => event.type === "reflector-model-call")).toHaveLength(4); expect(events.filter((event) => event.type === "reflection-leaf-retry-pending")).toHaveLength(2); - expect(events.filter((event) => event.type === "reflection-node-checkpointed")).toHaveLength(3); + const checkpoints = events.filter((event) => event.type === "reflection-node-checkpointed"); + expect(checkpoints).toHaveLength(3); + const checkpointNodes = await Promise.all(checkpoints.map((event) => persistence.get(event.node))); + expect(checkpointNodes.map((artifact) => ("status" in artifact ? artifact.status : null))).toEqual([ + "failed", + "failed", + "failed", + ]); - const resumed = await reflector.reflect( - reflectInput({ - execute() { - throw new Error("terminal leaf checkpoints must not execute again"); - }, - }), - ); - expect(resumed.nodes.map((node) => node.status)).toEqual(result.nodes.map((node) => node.status)); + await expect( + reflector.reflect( + reflectInput({ + execute() { + throw new Error("terminal leaf checkpoints must not execute again"); + }, + }), + ), + ).rejects.toMatchObject({ reasonCode: "model-call-failed" }); }); it("normalizes permitted bare evidence aliases and audits each correction", async () => { @@ -1133,8 +1200,21 @@ describe("recursive reflector", () => { }, }); - const result = await reflector.reflect(input); - expect(result.nodes.some((node) => node.kind === "leaf" && node.status === "failed")).toBe(true); + try { + const result = await reflector.reflect(input); + expect(result.nodes.some((node) => node.kind === "leaf" && node.status === "failed")).toBe(true); + } catch (error) { + expect(error).toMatchObject({ reasonCode: "model-call-failed" }); + const checkpoints = (await persistence.read("job-1")).filter( + (event) => event.type === "reflection-node-checkpointed", + ); + const checkpointNodes = await Promise.all(checkpoints.map((event) => persistence.get(event.node))); + expect( + checkpointNodes.some( + (artifact) => "kind" in artifact && artifact.kind === "leaf" && artifact.status === "failed", + ), + ).toBe(true); + } }); it("starts a fresh transcript for a leaf retry after executor validation is exhausted", async () => { @@ -1194,15 +1274,16 @@ describe("recursive reflector", () => { }, }); - const result = await reflector.reflect(input); + await expect(reflector.reflect(input)).rejects.toMatchObject({ reasonCode: "model-call-failed" }); - expect(result.nodes[0]).toMatchObject({ + const events = await persistence.read("job-1"); + const checkpoint = events.find((event) => event.type === "reflection-node-checkpointed"); + if (checkpoint === undefined) throw new Error("Expected a reflection node checkpoint"); + await expect(persistence.get(checkpoint.node)).resolves.toMatchObject({ status: "failed", failure: { diagnosticCode: "reflection-invalid-response" }, }); - expect((await persistence.read("job-1")).filter((event) => event.type === "reflector-model-call")).toHaveLength( - 4, - ); + expect(events.filter((event) => event.type === "reflector-model-call")).toHaveLength(4); }); it("limits each leaf to six evidence-tool calls", async () => { From e700fa0a6204dce72f434ca1837afa11b23aac4c Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 24 Aug 2026 23:03:18 -0400 Subject: [PATCH 8/8] fix(ai): stabilize Cloudflare gateway model generation --- packages/ai/scripts/generate-models.ts | 18 ++++++++++++++++++ .../ai/src/providers/cloudflare-ai-gateway.ts | 8 ++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 38bfdf31a..6f6c0a900 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -2269,6 +2269,24 @@ async function generateModels() { !((model.provider === "opencode" || model.provider === "opencode-go") && model.id === "gpt-5.3-codex-spark"), ); + // Cloudflare AI Gateway exposes Workers AI models through its OpenAI-compatible + // endpoint even when models.dev omits the duplicated gateway catalog entries. + for (const model of allModels.filter((candidate) => candidate.provider === "cloudflare-workers-ai")) { + const id = `workers-ai/${model.id}`; + if (allModels.some((candidate) => candidate.provider === "cloudflare-ai-gateway" && candidate.id === id)) continue; + const gatewayModel = { + ...model, + id, + provider: "cloudflare-ai-gateway", + baseUrl: CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, + cost: { ...model.cost }, + compat: { ...model.compat, sendSessionAffinityHeaders: true }, + }; + allModels.push(gatewayModel); + const reasoningOptions = modelsDevReasoningOptions.get(getModelKey(model)); + if (reasoningOptions) modelsDevReasoningOptions.set(getModelKey(gatewayModel), reasoningOptions); + } + // Temporary overrides until upstream model metadata is corrected. for (const candidate of allModels) { if (candidate.provider === "github-copilot" && GITHUB_COPILOT_EXTENDED_CONTEXT_MODELS.has(candidate.id)) { diff --git a/packages/ai/src/providers/cloudflare-ai-gateway.ts b/packages/ai/src/providers/cloudflare-ai-gateway.ts index 50c10569f..7c1cc3b78 100644 --- a/packages/ai/src/providers/cloudflare-ai-gateway.ts +++ b/packages/ai/src/providers/cloudflare-ai-gateway.ts @@ -6,10 +6,10 @@ import { CLOUDFLARE_AI_GATEWAY_MODELS } from "./cloudflare-ai-gateway.models.ts" import { cloudflareAIGatewayAuth } from "./cloudflare-auth.ts"; import { cloudflareStreams } from "./cloudflare-stream.ts"; -export function cloudflareAIGatewayProvider(): Provider< - "anthropic-messages" | "openai-completions" | "openai-responses" -> { - return createProvider({ +type CloudflareAIGatewayApi = "anthropic-messages" | "openai-completions" | "openai-responses"; + +export function cloudflareAIGatewayProvider(): Provider { + return createProvider({ id: "cloudflare-ai-gateway", name: "Cloudflare AI Gateway", auth: { apiKey: cloudflareAIGatewayAuth() },