fix(agent): reconcile and classify skill candidate retries

packages/ai/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+- Added a reusable classifier for transient provider and transport error messages.
+
 ## [0.84.0] - 2026-08-06
 
 ### Breaking Changes

packages/ai/src/utils/retry.ts

@@ -210,6 +210,12 @@ export async function retryAssistantCall(
 	}
 }
 
+/** Classifies whether provider or transport error text describes a transient failure. */
+export function isRetryableProviderErrorMessage(errorMessage: string): boolean {
+	if (NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN.test(errorMessage)) return false;
+	return RETRYABLE_PROVIDER_ERROR_PATTERN.test(errorMessage);
+}
+
 /**
  * Classifies whether a failed assistant message looks like a transient provider
  * or transport error, so callers can decide if the last assistant turn should be
@@ -220,8 +226,9 @@ export async function retryAssistantCall(
  * before restarting the assistant turn.
  */
 export function isRetryableAssistantError(message: AssistantMessage): boolean {
-	if (message.stopReason !== "error" || !message.errorMessage) return false;
-	const errorMessage = message.errorMessage;
-	if (NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN.test(errorMessage)) return false;
-	return RETRYABLE_PROVIDER_ERROR_PATTERN.test(errorMessage);
+	return (
+		message.stopReason === "error" &&
+		message.errorMessage !== undefined &&
+		isRetryableProviderErrorMessage(message.errorMessage)
+	);
 }

packages/ai/test/retry.test.ts

@@ -1,6 +1,11 @@
 import { describe, expect, it, vi } from "vitest";
 import { fauxAssistantMessage } from "../src/providers/faux.ts";
-import { isRetryableAssistantError, type RetryPolicy, retryAssistantCall } from "../src/utils/retry.ts";
+import {
+	isRetryableAssistantError,
+	isRetryableProviderErrorMessage,
+	type RetryPolicy,
+	retryAssistantCall,
+} from "../src/utils/retry.ts";
 
 const openAIExplicitRetryMessage =
 	"An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message.";
@@ -55,6 +60,7 @@ describe("provider retry classification", () => {
 				fauxAssistantMessage("", { stopReason: "error", errorMessage: openAIResponsesEarlyEofMessage }),
 			),
 		).toBe(true);
+		expect(isRetryableProviderErrorMessage(openAIResponsesEarlyEofMessage)).toBe(true);
 	});
 
 	it("keeps provider limit errors non-retryable", () => {

packages/coding-agent/CHANGELOG.md

@@ -23,7 +23,9 @@
 - Fixed an open, idle `/evolve` menu retaining enabled actions after another Diffex process disables evolution.
 - Fixed one failed skill candidate synthesis preventing later qualified candidates from being drafted, validated, and evaluated.
 - Fixed exhausted skill synthesis being retried by unrelated later evolution jobs instead of remaining terminal until new evidence revises the candidate.
+- Fixed skill synthesis retrying deterministic failures and resetting transient retry budgets between source evolution jobs instead of tracking each synthesis identity.
 - Fixed persisted skill candidate failures discarding the affected candidate group, work stage, provider or schema error, stop reason, and structured diagnostics.
+- Fixed startup leaving legacy exhausted skill synthesis and unevaluated structurally valid candidates permanently stranded.
 - Fixed completed evolution analysis hiding asynchronous candidate-generation progress or failure, candidate lifecycle states and reasons, and distinct stage outcomes.
 - Fixed skill candidate review leading with unreadable provenance IDs instead of an `EVOLVE.md`-style summary and materialized supporting and contradicting evidence.
 - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes.

packages/coding-agent/src/core/evolution-runtime.ts

@@ -3,7 +3,15 @@ import { randomUUID } from "node:crypto";
 import { mkdir, readdir, readFile, rename, stat, unlink, writeFile } from "node:fs/promises";
 import { basename, join } from "node:path";
 import { promisify } from "node:util";
-import { type AssistantMessage, contentText, type Message, type Model, type Tool } from "@earendil-works/diffex-ai";
+import {
+	type AssistantMessage,
+	contentText,
+	isRetryableAssistantError,
+	isRetryableProviderErrorMessage,
+	type Message,
+	type Model,
+	type Tool,
+} from "@earendil-works/diffex-ai";
 import type { HarnessStateStore, HarnessVersion } from "@earendil-works/diffex-harness-state";
 import lockfile from "proper-lockfile";
 import type { TSchema } from "typebox";
@@ -268,15 +276,33 @@ export class CodingAgentEvolutionModelExecutor implements EvolutionModelExecutor
 						});
 					}
 					request.signal?.throwIfAborted();
-					if (!(error instanceof EvolutionProviderTimeoutError) || providerAttempt >= 2) throw error;
+					if (error instanceof EvolutionProviderTimeoutError && providerAttempt < 2) continue;
+					if (error instanceof EvolutionProviderTimeoutError) {
+						throw new EvolutionExecutionError(error.message, {
+							reasonCode: "model-call-failed",
+							retryable: true,
+							diagnostics: { providerTimeout: error.deadline },
+						});
+					}
+					const errorMessage = error instanceof Error ? error.message : String(error);
+					if (isRetryableProviderErrorMessage(errorMessage)) {
+						throw new EvolutionExecutionError(errorMessage, {
+							reasonCode: "model-call-failed",
+							retryable: true,
+							cause: error,
+							diagnostics: { transportError: errorMessage },
+						});
+					}
+					throw error;
 				}
 			}
+			request.signal?.throwIfAborted();
 			if (assistant.stopReason === "error" || assistant.stopReason === "aborted") {
 				throw new EvolutionExecutionError(
 					assistant.errorMessage ?? `Evolution model stopped with ${assistant.stopReason}`,
 					{
 						reasonCode: "model-call-failed",
-						retryable: true,
+						retryable: isRetryableAssistantError(assistant),
 						diagnostics: {
 							stopReason: assistant.stopReason,
 							providerError: assistant.errorMessage ?? null,
@@ -972,15 +998,16 @@ export async function createCodingAgentEvolutionRuntime(
 			resourceLoader: options.resourceLoader,
 		}),
 	});
+	const candidateSource = new PollingSkillCandidateWorkSource({
+		clock,
+		jobs: persistence,
+		store: persistence,
+		registry: candidateRegistry,
+		evaluatorAvailable: options.skillEvaluator !== undefined,
+	});
 	const candidateWorker = new SkillCandidateBackgroundWorker({
 		clock,
-		source: new PollingSkillCandidateWorkSource({
-			clock,
-			jobs: persistence,
-			store: persistence,
-			registry: candidateRegistry,
-			evaluatorAvailable: options.skillEvaluator !== undefined,
-		}),
+		source: candidateSource,
 		processor: new SkillCandidateRegistryWorker({
 			version: VERSION,
 			clock,
@@ -1052,6 +1079,7 @@ export async function createCodingAgentEvolutionRuntime(
 	return {
 		control,
 		async start() {
+			await candidateSource.reconcileLegacyWork();
 			const abandoned = (await persistence.list(scope)).filter((job) => job.outcome === "active");
 			for (const job of abandoned) {
 				abandonedJobIds.add(job.jobId);

packages/coding-agent/test/evolution-runtime.test.ts

@@ -659,13 +659,13 @@ describe("coding-agent evolution runtime", () => {
 		});
 	});
 
-	it("preserves provider stop diagnostics for evolution failures", async () => {
+	it("preserves transient provider stop diagnostics for evolution failures", async () => {
 		const harness = await createHarness();
 		harnesses.push(harness);
 		harness.setResponses([
 			fauxAssistantMessage("", {
 				stopReason: "error",
-				errorMessage: "Provider rejected the structured response",
+				errorMessage: "Provider returned error: 503 service unavailable",
 			}),
 		]);
 		const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, {
@@ -689,16 +689,89 @@ describe("coding-agent evolution runtime", () => {
 			}),
 		).rejects.toMatchObject({
 			name: "EvolutionExecutionError",
-			message: "Provider rejected the structured response",
+			message: "Provider returned error: 503 service unavailable",
 			reasonCode: "model-call-failed",
 			retryable: true,
+			diagnostics: {
+				stopReason: "error",
+				providerError: "Provider returned error: 503 service unavailable",
+			},
+		});
+	});
+
+	it("classifies deterministic provider stops as non-retryable", async () => {
+		const harness = await createHarness();
+		harnesses.push(harness);
+		harness.setResponses([
+			fauxAssistantMessage("", {
+				stopReason: "error",
+				errorMessage: "Provider rejected the structured response",
+			}),
+		]);
+		const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, {
+			async waitUntilBackgroundAllowed() {},
+		});
+		const model = harness.getModel();
+
+		await expect(
+			executor.execute({
+				stage: "skill-synthesizer",
+				model: {
+					provider: model.provider,
+					model: model.id,
+					configurationDigest: `sha256:${"1".repeat(64)}`,
+				},
+				promptVersion: "skill-synthesizer-v1",
+				idempotencyKey: "deterministic-provider-stop",
+				input: {},
+				outputSchema: { type: "object" },
+				tools: [],
+			}),
+		).rejects.toMatchObject({
+			name: "EvolutionExecutionError",
+			reasonCode: "model-call-failed",
+			retryable: false,
 			diagnostics: {
 				stopReason: "error",
 				providerError: "Provider rejected the structured response",
 			},
 		});
 	});
 
+	it("classifies thrown provider transport failures as retryable", async () => {
+		const harness = await createHarness();
+		harnesses.push(harness);
+		harness.setResponses([
+			() => {
+				throw new Error("fetch failed: connection reset before headers");
+			},
+		]);
+		const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, {
+			async waitUntilBackgroundAllowed() {},
+		});
+		const model = harness.getModel();
+
+		await expect(
+			executor.execute({
+				stage: "skill-synthesizer",
+				model: {
+					provider: model.provider,
+					model: model.id,
+					configurationDigest: `sha256:${"1".repeat(64)}`,
+				},
+				promptVersion: "skill-synthesizer-v1",
+				idempotencyKey: "provider-transport-failure",
+				input: {},
+				outputSchema: { type: "object" },
+				tools: [],
+			}),
+		).rejects.toMatchObject({
+			name: "EvolutionExecutionError",
+			reasonCode: "model-call-failed",
+			retryable: true,
+		});
+	});
+
 	it("corrects an invalid response-tool submission within the active transcript", async () => {
 		const harness = await createHarness();
 		harnesses.push(harness);

packages/evolution/RECOVERY.md

@@ -26,7 +26,7 @@ Recovery never treats partial output as completed work and never feeds evolution
 | Successful rollback selection | The recorded immutable parent is still present and the evolved revision is current. | Select the parent with compare-and-swap and refresh on the next prompt boundary. | Existing in-flight prompts stay pinned to their admitted revision. |
 | Unsupported skill or orchestration target | The target registry has no enabled writer route. | Preserve the finding as deferred evidence with no executable payload. | Disabled target kinds cannot reach filesystem mutation or activation. |
 | Interrupted skill-candidate consolidation | A completed evolution job has checkpointed candidates but no completed candidate-work event. | Resume from append-only registry events with the same source-job idempotency key; repair missing scores or transitions before completion. | Candidate proposal revisions remain immutable, retries do not add recurrence, and partial registry appends cannot activate a skill. |
-| Interrupted or failed skill synthesis, validation, evaluation, write, or activation | A qualified group is missing the next immutable artifact or transition, an accepted candidate has a durable write without a matching activation, or synthesis exhausted its bounded attempts. | Resume recoverable work with revision-bound idempotency keys; reuse recorded artifacts and receipts, repair the missing transition, and record exhausted synthesis as `synthesis-failed` until a new proposal revision changes its synthesis identity. | Only evidence-qualified groups invoke synthesis, unrelated later jobs cannot retry an exhausted group, only deterministically valid drafts enter evaluation, only accepted evaluations reach the writer, and retries cannot stage or activate a skill twice. |
+| Interrupted or failed skill synthesis, validation, evaluation, write, or activation | A qualified group is missing the next immutable artifact or transition, an accepted candidate has a durable write without a matching activation, or synthesis returned a non-retryable error or exhausted its identity-scoped attempts. | Resume recoverable work with revision-bound idempotency keys; reuse recorded artifacts and receipts, repair the missing transition, and record terminal synthesis as `synthesis-failed` until a new proposal revision changes its synthesis identity. | Only transient provider or transport synthesis failures are retried, attempts follow the synthesis identity across source jobs, terminal groups do not block later qualified groups, only deterministically valid drafts enter evaluation, only accepted evaluations reach the writer, and retries cannot stage or activate a skill twice. |
 
 ## Operational observability
 

packages/evolution/src/contracts.ts

@@ -992,6 +992,8 @@ export interface SkillCandidateWorkErrorDiagnostic {
 
 export interface SkillCandidateGroupFailureDiagnostic {
 	readonly groupId: string | null;
+	/** Present for synthesis failures so retry budgets follow the immutable synthesis identity across jobs. */
+	readonly synthesisIdempotencyKey?: string;
 	readonly error: SkillCandidateWorkErrorDiagnostic;
 }
 

packages/evolution/src/filesystem-store.ts

@@ -324,6 +324,7 @@ function isCandidateGroupFailureDiagnostic(value: unknown): boolean {
 	return (
 		isRecord(value) &&
 		(value.groupId === null || typeof value.groupId === "string") &&
+		(value.synthesisIdempotencyKey === undefined || typeof value.synthesisIdempotencyKey === "string") &&
 		isCandidateWorkErrorDiagnostic(value.error)
 	);
 }

packages/evolution/src/ports.ts

@@ -341,6 +341,7 @@ export interface SkillCandidateSynthesisFailureRecordingInput {
 		readonly groupId: string;
 		readonly proposalRevisionIds: readonly string[];
 		readonly idempotencyKey: string;
+		readonly reasonCode: "skill-synthesis-attempts-exhausted" | "skill-synthesis-non-retryable";
 	}[];
 	readonly createdAt: string;
 }

packages/evolution/src/skill-candidate-registry.ts

@@ -865,7 +865,7 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry
 					failure.groupId,
 					"evidence-qualified",
 					"synthesis-failed",
-					["skill-synthesis-attempts-exhausted"],
+					[failure.reasonCode],
 					nextSequence(snapshot.events, planned.length),
 					failure.idempotencyKey,
 					this.#producer,