From f95b7f14f2eec39d27c91804b7f667e3be1568ee Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 30 Aug 2026 14:53:53 -0400 Subject: [PATCH 1/2] fix(agent): reconcile legacy skill candidate work --- packages/coding-agent/CHANGELOG.md | 1 + .../src/core/evolution-runtime.ts | 16 +- .../evolution/src/skill-candidate-worker.ts | 254 ++++++++++++++---- .../test/skill-candidate-worker.test.ts | 170 ++++++++++++ 4 files changed, 380 insertions(+), 61 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 525e0d669..9825b590c 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -24,6 +24,7 @@ - Fixed one failed skill candidate synthesis preventing later qualified candidates from being drafted, validated, and evaluated. - Fixed exhausted skill synthesis being retried by unrelated later evolution jobs instead of remaining terminal until new evidence revises the candidate. - Fixed persisted skill candidate failures discarding the affected candidate group, work stage, provider or schema error, stop reason, and structured diagnostics. +- Fixed startup leaving legacy exhausted skill synthesis and unevaluated structurally valid candidates permanently stranded. - Fixed completed evolution analysis hiding asynchronous candidate-generation progress or failure, candidate lifecycle states and reasons, and distinct stage outcomes. - Fixed skill candidate review leading with unreadable provenance IDs instead of an `EVOLVE.md`-style summary and materialized supporting and contradicting evidence. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 84ceaf625..34c784bd3 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -972,15 +972,16 @@ export async function createCodingAgentEvolutionRuntime( resourceLoader: options.resourceLoader, }), }); + const candidateSource = new PollingSkillCandidateWorkSource({ + clock, + jobs: persistence, + store: persistence, + registry: candidateRegistry, + evaluatorAvailable: options.skillEvaluator !== undefined, + }); const candidateWorker = new SkillCandidateBackgroundWorker({ clock, - source: new PollingSkillCandidateWorkSource({ - clock, - jobs: persistence, - store: persistence, - registry: candidateRegistry, - evaluatorAvailable: options.skillEvaluator !== undefined, - }), + source: candidateSource, processor: new SkillCandidateRegistryWorker({ version: VERSION, clock, @@ -1052,6 +1053,7 @@ export async function createCodingAgentEvolutionRuntime( return { control, async start() { + await candidateSource.reconcileLegacyWork(); const abandoned = (await persistence.list(scope)).filter((job) => job.outcome === "active"); for (const job of abandoned) { abandonedJobIds.add(job.jobId); diff --git a/packages/evolution/src/skill-candidate-worker.ts b/packages/evolution/src/skill-candidate-worker.ts index bc662c19d..194144711 100644 --- a/packages/evolution/src/skill-candidate-worker.ts +++ b/packages/evolution/src/skill-candidate-worker.ts @@ -6,6 +6,7 @@ import type { EvolutionLeaseToken, EvolutionScope, SkillCandidate, + SkillCandidateGroup, SkillCandidateGroupFailureDiagnostic, SkillCandidateRegistryEvent, SkillCandidateWorkCheckpointedEvent, @@ -160,17 +161,88 @@ function hasSkillCandidates(job: EvolutionJob): boolean { ); } +type SkillCandidateWorkEvent = SkillCandidateWorkCheckpointedEvent | SkillCandidateWorkFailedEvent; + function workEvents( events: readonly SkillCandidateRegistryEvent[], sourceEvolutionJobId: string, -): readonly (SkillCandidateWorkCheckpointedEvent | SkillCandidateWorkFailedEvent)[] { +): readonly SkillCandidateWorkEvent[] { return events.filter( - (event): event is SkillCandidateWorkCheckpointedEvent | SkillCandidateWorkFailedEvent => + (event): event is SkillCandidateWorkEvent => (event.type === "skill-candidate-work-checkpointed" || event.type === "skill-candidate-work-failed") && event.sourceEvolutionJobId === sourceEvolutionJobId, ); } +function latestWorkEvent(events: readonly SkillCandidateWorkEvent[]): SkillCandidateWorkEvent | null { + return events.reduce( + (latest, event) => (latest === null || event.registrySequence > latest.registrySequence ? event : latest), + null, + ); +} + +function latestLegacySynthesisFailure( + events: readonly SkillCandidateWorkEvent[], +): SkillCandidateWorkFailedEvent | null { + const latestCompletedSequence = events.reduce( + (latest, event) => + event.type === "skill-candidate-work-checkpointed" && event.stage === "completed" + ? Math.max(latest, event.registrySequence) + : latest, + 0, + ); + return events.reduce((latest, event) => { + if ( + event.type !== "skill-candidate-work-failed" || + event.retryNotBefore !== null || + event.diagnostics !== undefined || + event.registrySequence < latestCompletedSequence + ) { + return latest; + } + return latest === null || event.registrySequence > latest.registrySequence ? event : latest; + }, null); +} + +function legacySynthesisGroupIds(events: readonly SkillCandidateWorkEvent[]): ReadonlySet | null { + const failure = latestLegacySynthesisFailure(events); + if (failure === null) return null; + const checkpoint = events.reduce((latest, event) => { + if ( + event.type !== "skill-candidate-work-checkpointed" || + event.stage !== "synthesizing" || + event.registrySequence >= failure.registrySequence + ) { + return latest; + } + return latest === null || event.registrySequence > latest.registrySequence ? event : latest; + }, null); + return checkpoint === null ? null : new Set(checkpoint.groupIds); +} + +function evaluationGroupIds(events: readonly SkillCandidateWorkEvent[]): ReadonlySet { + return new Set( + events.flatMap((event) => + event.type === "skill-candidate-work-checkpointed" && event.stage === "evaluating" ? event.groupIds : [], + ), + ); +} + +function hasAwaitingEvaluation( + events: readonly SkillCandidateWorkEvent[], + groups: readonly SkillCandidateGroup[], +): boolean { + const groupIds = evaluationGroupIds(events); + return ( + groupIds.size > 0 && + groups.some( + (group) => + groupIds.has(group.groupId) && + (group.state === "structurally-valid" || group.state === "evaluation-pending"), + ) + ); +} + function isInputSnapshot(artifact: EvolutionArtifact): artifact is EvolutionInputSnapshot { return "evidenceSources" in artifact && "repository" in artifact && "harness" in artifact && "trigger" in artifact; } @@ -233,6 +305,8 @@ class SkillCandidateWorkStageError extends AggregateError { export class PollingSkillCandidateWorkSource { readonly #dependencies: PollingSkillCandidateWorkSourceDependencies; readonly #pollIntervalMs: number; + readonly #legacySynthesisJobIds = new Set(); + readonly #legacyEvaluationJobIds = new Set(); constructor( dependencies: PollingSkillCandidateWorkSourceDependencies, @@ -242,41 +316,92 @@ export class PollingSkillCandidateWorkSource { this.#pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS, "pollIntervalMs"); } - async nextRunnableJob(now: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted(); - const nowMs = Date.parse(now); - if (!Number.isFinite(nowMs)) throw new Error("Skill candidate worker time must be a valid ISO timestamp"); - const jobs = (await this.#dependencies.jobs.list()) + async #candidateJobs(): Promise { + return (await this.#dependencies.jobs.list()) .filter((job) => job.outcome === "completed" && job.stage === "completed" && hasSkillCandidates(job)) .sort( (left, right) => (left.terminalAt ?? left.updatedAt).localeCompare(right.terminalAt ?? right.updatedAt) || left.jobId.localeCompare(right.jobId), ); - for (const job of jobs) { + } + + async reconcileLegacyWork(signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + this.#legacySynthesisJobIds.clear(); + this.#legacyEvaluationJobIds.clear(); + const scopeStates = new Map< + string, + { + readonly events: readonly SkillCandidateRegistryEvent[]; + readonly groups: readonly SkillCandidateGroup[]; + } + >(); + for (const job of await this.#candidateJobs()) { + signal?.throwIfAborted(); + const scopeKey = `${job.metadata.scope.repositoryId}\u0000${job.metadata.scope.scopeId}`; + let scopeState = scopeStates.get(scopeKey); + if (scopeState === undefined) { + const [snapshot, groups] = await Promise.all([ + this.#dependencies.store.loadSkillCandidateRegistry(job.metadata.scope), + this.#dependencies.registry.list(job.metadata.scope), + ]); + scopeState = { events: snapshot.events, groups }; + scopeStates.set(scopeKey, scopeState); + } + const events = workEvents(scopeState.events, job.jobId); + const terminal = latestWorkEvent(events); + const recoveryGroupIds = legacySynthesisGroupIds(events); + if ( + terminal?.type === "skill-candidate-work-failed" && + terminal === latestLegacySynthesisFailure(events) && + recoveryGroupIds !== null && + scopeState.groups.some( + (group) => recoveryGroupIds.has(group.groupId) && group.state === "evidence-qualified", + ) + ) { + this.#legacySynthesisJobIds.add(job.jobId); + } + if ( + !this.#dependencies.evaluatorAvailable && + terminal?.type === "skill-candidate-work-checkpointed" && + terminal.stage === "completed" && + hasAwaitingEvaluation(events, scopeState.groups) + ) { + this.#legacyEvaluationJobIds.add(job.jobId); + } + } + } + + async nextRunnableJob(now: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const nowMs = Date.parse(now); + if (!Number.isFinite(nowMs)) throw new Error("Skill candidate worker time must be a valid ISO timestamp"); + for (const job of await this.#candidateJobs()) { const snapshot = await this.#dependencies.store.loadSkillCandidateRegistry(job.metadata.scope); const events = workEvents(snapshot.events, job.jobId); - const evaluationGroupIds = new Set( - events.flatMap((event) => - event.type === "skill-candidate-work-checkpointed" && event.stage === "evaluating" ? event.groupIds : [], - ), - ); - const awaitingEvaluation = - evaluationGroupIds.size > 0 && - (await this.#dependencies.registry.list(job.metadata.scope)).some( - (group) => - evaluationGroupIds.has(group.groupId) && - (group.state === "structurally-valid" || group.state === "evaluation-pending"), + const terminal = latestWorkEvent(events); + if (terminal?.type === "skill-candidate-work-checkpointed" && terminal.stage === "completed") { + const awaitingEvaluation = hasAwaitingEvaluation( + events, + await this.#dependencies.registry.list(job.metadata.scope), ); - const completed = events.some( - (event) => event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", - ); - if (completed && !(this.#dependencies.evaluatorAvailable && awaitingEvaluation)) continue; - const latestFailure = events - .filter((event): event is SkillCandidateWorkFailedEvent => event.type === "skill-candidate-work-failed") - .sort((left, right) => right.attempt - left.attempt)[0]; - if (latestFailure?.retryNotBefore === null) continue; - if (latestFailure !== undefined && Date.parse(latestFailure.retryNotBefore) > nowMs) continue; + if ( + !awaitingEvaluation || + (!this.#dependencies.evaluatorAvailable && !this.#legacyEvaluationJobIds.has(job.jobId)) + ) { + continue; + } + } + if (terminal?.type === "skill-candidate-work-failed") { + if ( + terminal.retryNotBefore === null && + !(terminal.diagnostics === undefined && this.#legacySynthesisJobIds.has(job.jobId)) + ) { + continue; + } + if (terminal.retryNotBefore !== null && Date.parse(terminal.retryNotBefore) > nowMs) continue; + } signal?.throwIfAborted(); return job.jobId; } @@ -361,13 +486,12 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor ): Promise { const idempotencyKey = stableId("skill-candidate-work-stage", { sourceEvolutionJobId: job.jobId, stage }); await this.#appendEvent(job.metadata.scope, lease, (registrySequence, existing) => { + const history = workEvents(existing, job.jobId); + const terminal = latestWorkEvent(history); if ( - existing.some( - (event) => - event.type === "skill-candidate-work-checkpointed" && - event.sourceEvolutionJobId === job.jobId && - event.stage === stage, - ) + stage === "completed" + ? terminal?.type === "skill-candidate-work-checkpointed" && terminal.stage === "completed" + : history.some((event) => event.type === "skill-candidate-work-checkpointed" && event.stage === stage) ) { return null; } @@ -400,6 +524,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor async #appendFailure( job: EvolutionJob, attempt: number, + retryAttempt: number, stage: SkillCandidateWorkStage, diagnostics: readonly SkillCandidateGroupFailureDiagnostic[], error: unknown, @@ -410,16 +535,11 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor attempt, }); await this.#appendEvent(job.metadata.scope, lease, (registrySequence, existing) => { + const history = workEvents(existing, job.jobId); + const terminal = latestWorkEvent(history); if ( - existing.some( - (event) => - (event.type === "skill-candidate-work-checkpointed" && - event.sourceEvolutionJobId === job.jobId && - event.stage === "completed") || - (event.type === "skill-candidate-work-failed" && - event.sourceEvolutionJobId === job.jobId && - event.attempt === attempt), - ) + (terminal?.type === "skill-candidate-work-checkpointed" && terminal.stage === "completed") || + history.some((event) => event.type === "skill-candidate-work-failed" && event.attempt === attempt) ) { return null; } @@ -435,10 +555,10 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor attempt, message: error instanceof Error ? error.message : String(error), retryNotBefore: - attempt >= this.#maxAttempts + retryAttempt >= this.#maxAttempts ? null : new Date( - Date.parse(now) + retryDelay(attempt, this.#retryBaseDelayMs, this.#retryMaxDelayMs), + Date.parse(now) + retryDelay(retryAttempt, this.#retryBaseDelayMs, this.#retryMaxDelayMs), ).toISOString(), stage, diagnostics, @@ -540,13 +660,35 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor (await this.#dependencies.store.loadSkillCandidateRegistry(job.metadata.scope)).events, job.jobId, ); - const completedCheckpoint = history.find( - (event): event is SkillCandidateWorkCheckpointedEvent => - event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", + const terminal = latestWorkEvent(history); + if ( + terminal?.type === "skill-candidate-work-checkpointed" && + terminal.stage === "completed" && + this.#dependencies.evaluator === undefined && + !hasAwaitingEvaluation(history, await this.#dependencies.registry.list(job.metadata.scope)) + ) { + return; + } + const failures = history.filter( + (event): event is SkillCandidateWorkFailedEvent => event.type === "skill-candidate-work-failed", + ); + const latestLegacyFailure = failures.reduce( + (latest, failure) => + failure.diagnostics === undefined && (latest === null || failure.registrySequence > latest.registrySequence) + ? failure + : latest, + null, ); - if (completedCheckpoint !== undefined && this.#dependencies.evaluator === undefined) return; - const attempt = history.filter((event) => event.type === "skill-candidate-work-failed").length + 1; - if (attempt > this.#maxAttempts) throw new Error(`Skill candidate work for "${job.jobId}" exhausted retries`); + const retryAttempt = + failures.filter( + (failure) => + latestLegacyFailure === null || failure.registrySequence > latestLegacyFailure.registrySequence, + ).length + 1; + const attempt = failures.reduce((highest, failure) => Math.max(highest, failure.attempt), 0) + 1; + if (retryAttempt > this.#maxAttempts) { + throw new Error(`Skill candidate work for "${job.jobId}" exhausted retries`); + } + const recoveryGroupIds = legacySynthesisGroupIds(history); let activeStage: SkillCandidateWorkStage = "consolidating"; let activeGroupId: string | null = null; @@ -596,7 +738,11 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor ]; activeStage = "scoring"; await this.#appendCheckpoint(job, activeStage, attempt, processedGroupIds, claim()); - const synthesisTargets = result.groups.filter((group) => group.state === "evidence-qualified"); + const synthesisTargets = result.groups.filter( + (group) => + group.state === "evidence-qualified" && + (recoveryGroupIds === null || recoveryGroupIds.has(group.groupId)), + ); activeStage = "synthesizing"; await this.#appendCheckpoint( job, @@ -651,7 +797,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor ); } activeGroupId = null; - if (synthesisFailures.length > 0 && attempt >= this.#maxAttempts) { + if (synthesisFailures.length > 0 && retryAttempt >= this.#maxAttempts) { await this.#dependencies.registry.recordSynthesisFailures({ scope: job.metadata.scope, lease: claim(), @@ -806,7 +952,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor error: errorDiagnostic(failure.error), })) : [{ groupId: activeGroupId, error: errorDiagnostic(error) }]; - await this.#appendFailure(job, attempt, stage, diagnostics, error, claim()); + await this.#appendFailure(job, attempt, retryAttempt, stage, diagnostics, error, claim()); throw error; } } diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index 89044e8e4..62c2723ce 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -262,6 +262,43 @@ class MemoryRegistryStore implements SkillCandidateRegistryStore { } } +function workCheckpoint( + registrySequence: number, + stage: "synthesizing" | "evaluating" | "completed", + groupIds: readonly string[], + attempt = 3, +): SkillCandidateRegistryEvent { + const content = { + schemaVersion: 1 as const, + eventId: `work-checkpoint-${registrySequence}`, + registrySequence, + type: "skill-candidate-work-checkpointed" as const, + workId: "work-job-1", + sourceEvolutionJobId: "job-1", + stage, + idempotencyKey: `work-checkpoint-${stage}`, + attempt, + groupIds, + }; + return { metadata: metadata(content.eventId, [content.sourceEvolutionJobId, ...groupIds], content), ...content }; +} + +function legacyWorkFailure(registrySequence: number, attempt = 3): SkillCandidateRegistryEvent { + const content = { + schemaVersion: 1 as const, + eventId: `legacy-work-failure-${registrySequence}`, + registrySequence, + type: "skill-candidate-work-failed" as const, + workId: "work-job-1", + sourceEvolutionJobId: "job-1", + idempotencyKey: `legacy-work-failure-${attempt}`, + attempt, + message: "legacy synthesis failed", + retryNotBefore: null, + }; + return { metadata: metadata(content.eventId, [content.sourceEvolutionJobId], content), ...content }; +} + class CompletionAcknowledgementLostStore extends MemoryRegistryStore { interrupted = false; @@ -836,6 +873,106 @@ describe("skill candidate registry worker", () => { ).toBe(true); }); + it("reconciles legacy exhausted synthesis by identity under fresh bounded retries", async () => { + const store = new MemoryRegistryStore([ + workCheckpoint(1, "synthesizing", ["group-failing", "group-succeeding"]), + legacyWorkFailure(2), + ]); + const data = fixture(true, store); + const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; + if (candidateReference === undefined) throw new Error("Expected candidate reference"); + const group = qualifiedGroup(candidateReference); + data.registry.groups.push( + { + ...group, + groupId: "group-failing", + synthesisIdempotencyKey: "skill-synthesis-failing", + validationIdempotencyKey: "skill-validation-failing", + evaluationIdempotencyKey: "skill-evaluation-failing", + }, + { + ...group, + groupId: "group-succeeding", + synthesisIdempotencyKey: "skill-synthesis-succeeding", + validationIdempotencyKey: "skill-validation-succeeding", + evaluationIdempotencyKey: "skill-evaluation-succeeding", + }, + { + ...group, + groupId: "group-unrelated", + synthesisIdempotencyKey: "skill-synthesis-unrelated", + validationIdempotencyKey: "skill-validation-unrelated", + evaluationIdempotencyKey: "skill-evaluation-unrelated", + }, + ); + data.synthesisFailures.set("group-failing", new Error("synthesis failed")); + const source = new PollingSkillCandidateWorkSource({ + clock: data.clock, + jobs: data.jobs, + store, + registry: data.registry, + evaluatorAvailable: true, + }); + + expect(await source.nextRunnableJob(now)).toBeNull(); + await source.reconcileLegacyWork(); + expect(await source.nextRunnableJob(now)).toBe(data.job.jobId); + + for (let retryAttempt = 1; retryAttempt <= 3; retryAttempt += 1) { + await expect(data.worker.process(data.job.jobId)).rejects.toThrow( + "Skill synthesis failed for candidate groups: group-failing", + ); + const failure = store.events.find( + (event) => event.type === "skill-candidate-work-failed" && event.attempt === retryAttempt + 3, + ); + expect(failure).toMatchObject({ retryNotBefore: retryAttempt === 3 ? null : expect.any(String) }); + } + + expect(data.synthesisRequests.map((request) => request.group.groupId)).toEqual([ + "group-failing", + "group-succeeding", + "group-failing", + "group-failing", + ]); + expect(data.registry.groups).toEqual([ + expect.objectContaining({ groupId: "group-failing", state: "synthesis-failed" }), + expect.objectContaining({ groupId: "group-succeeding", state: "accepted" }), + expect.objectContaining({ groupId: "group-unrelated", state: "evidence-qualified" }), + ]); + expect(await source.nextRunnableJob(now)).toBeNull(); + }); + + it("reconciles a completed checkpoint followed by a legacy synthesis failure", async () => { + const store = new MemoryRegistryStore([ + workCheckpoint(1, "synthesizing", ["group-qualified"]), + workCheckpoint(2, "completed", ["group-qualified"]), + legacyWorkFailure(3), + ]); + const data = fixture(false, store); + const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; + if (candidateReference === undefined) throw new Error("Expected candidate reference"); + data.registry.groups.push(qualifiedGroup(candidateReference)); + const source = new PollingSkillCandidateWorkSource({ + clock: data.clock, + jobs: data.jobs, + store, + registry: data.registry, + evaluatorAvailable: false, + }); + + await source.reconcileLegacyWork(); + expect(await source.nextRunnableJob(now)).toBe(data.job.jobId); + await data.worker.process(data.job.jobId); + + const completions = store.events.filter( + (event) => event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", + ); + expect(completions).toHaveLength(2); + expect(completions[1]?.registrySequence).toBeGreaterThan(legacyWorkFailure(3).registrySequence); + expect(data.registry.groups[0]?.state).toBe("accepted"); + expect(await source.nextRunnableJob(now)).toBeNull(); + }); + it("durably exposes exhausted synthesis without blocking a later candidate in the same batch", async () => { const data = fixture(true); const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; @@ -975,6 +1112,39 @@ describe("skill candidate registry worker", () => { expect(await source.nextRunnableJob(now)).toBeNull(); }); + it("reopens legacy completed work to record skipped evaluation when no evaluator is available", async () => { + const data = fixture(); + const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; + if (candidateReference === undefined) throw new Error("Expected candidate reference"); + data.registry.groups.push(qualifiedGroup(candidateReference)); + await data.worker.process(data.job.jobId); + const accepted = data.registry.groups[0]; + if (accepted === undefined) throw new Error("Expected accepted group"); + data.registry.groups[0] = { ...accepted, state: "structurally-valid", evaluation: null }; + const source = new PollingSkillCandidateWorkSource({ + clock: data.clock, + jobs: data.jobs, + store: data.store, + registry: data.registry, + evaluatorAvailable: false, + }); + + expect(await source.nextRunnableJob(now)).toBeNull(); + await source.reconcileLegacyWork(); + expect(await source.nextRunnableJob(now)).toBe(data.job.jobId); + await data.worker.process(data.job.jobId); + + expect(data.registry.evaluationSkipRequests).toHaveLength(2); + expect(data.registry.groups[0]).toMatchObject({ + state: "accepted", + evaluation: null, + dispositionHistory: expect.arrayContaining([ + expect.objectContaining({ reasonCodes: ["retrieval-and-behavior-evaluation-skipped"] }), + ]), + }); + expect(await source.nextRunnableJob(now)).toBeNull(); + }); + it("reopens legacy completed work when its evaluator becomes available", async () => { const data = fixture(true); const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; From 27b2f32036c244d6ce6777e960660a89d66a91d6 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 30 Aug 2026 16:39:07 -0400 Subject: [PATCH 2/2] fix(agent): classify skill synthesis retries --- packages/ai/CHANGELOG.md | 4 + packages/ai/src/utils/retry.ts | 15 ++- packages/ai/test/retry.test.ts | 8 +- packages/coding-agent/CHANGELOG.md | 1 + .../src/core/evolution-runtime.ts | 32 +++++- .../test/evolution-runtime.test.ts | 79 ++++++++++++++- packages/evolution/RECOVERY.md | 2 +- packages/evolution/src/contracts.ts | 2 + packages/evolution/src/filesystem-store.ts | 1 + packages/evolution/src/ports.ts | 1 + .../evolution/src/skill-candidate-registry.ts | 2 +- .../evolution/src/skill-candidate-worker.ts | 97 +++++++++++++++---- .../evolution/test/filesystem-store.test.ts | 1 + .../test/skill-candidate-registry.test.ts | 2 + .../test/skill-candidate-worker.test.ts | 66 +++++++++---- 15 files changed, 260 insertions(+), 53 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 92840d746..a240d7717 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added a reusable classifier for transient provider and transport error messages. + ## [0.84.0] - 2026-08-06 ### Breaking Changes diff --git a/packages/ai/src/utils/retry.ts b/packages/ai/src/utils/retry.ts index 463b90329..907d19504 100644 --- a/packages/ai/src/utils/retry.ts +++ b/packages/ai/src/utils/retry.ts @@ -210,6 +210,12 @@ export async function retryAssistantCall( } } +/** Classifies whether provider or transport error text describes a transient failure. */ +export function isRetryableProviderErrorMessage(errorMessage: string): boolean { + if (NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN.test(errorMessage)) return false; + return RETRYABLE_PROVIDER_ERROR_PATTERN.test(errorMessage); +} + /** * Classifies whether a failed assistant message looks like a transient provider * or transport error, so callers can decide if the last assistant turn should be @@ -220,8 +226,9 @@ export async function retryAssistantCall( * before restarting the assistant turn. */ export function isRetryableAssistantError(message: AssistantMessage): boolean { - if (message.stopReason !== "error" || !message.errorMessage) return false; - const errorMessage = message.errorMessage; - if (NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN.test(errorMessage)) return false; - return RETRYABLE_PROVIDER_ERROR_PATTERN.test(errorMessage); + return ( + message.stopReason === "error" && + message.errorMessage !== undefined && + isRetryableProviderErrorMessage(message.errorMessage) + ); } diff --git a/packages/ai/test/retry.test.ts b/packages/ai/test/retry.test.ts index fc79f6916..4f91f429c 100644 --- a/packages/ai/test/retry.test.ts +++ b/packages/ai/test/retry.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it, vi } from "vitest"; import { fauxAssistantMessage } from "../src/providers/faux.ts"; -import { isRetryableAssistantError, type RetryPolicy, retryAssistantCall } from "../src/utils/retry.ts"; +import { + isRetryableAssistantError, + isRetryableProviderErrorMessage, + type RetryPolicy, + retryAssistantCall, +} from "../src/utils/retry.ts"; const openAIExplicitRetryMessage = "An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message."; @@ -55,6 +60,7 @@ describe("provider retry classification", () => { fauxAssistantMessage("", { stopReason: "error", errorMessage: openAIResponsesEarlyEofMessage }), ), ).toBe(true); + expect(isRetryableProviderErrorMessage(openAIResponsesEarlyEofMessage)).toBe(true); }); it("keeps provider limit errors non-retryable", () => { diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 9825b590c..0b5c0c68f 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -23,6 +23,7 @@ - Fixed an open, idle `/evolve` menu retaining enabled actions after another Diffex process disables evolution. - Fixed one failed skill candidate synthesis preventing later qualified candidates from being drafted, validated, and evaluated. - Fixed exhausted skill synthesis being retried by unrelated later evolution jobs instead of remaining terminal until new evidence revises the candidate. +- Fixed skill synthesis retrying deterministic failures and resetting transient retry budgets between source evolution jobs instead of tracking each synthesis identity. - Fixed persisted skill candidate failures discarding the affected candidate group, work stage, provider or schema error, stop reason, and structured diagnostics. - Fixed startup leaving legacy exhausted skill synthesis and unevaluated structurally valid candidates permanently stranded. - Fixed completed evolution analysis hiding asynchronous candidate-generation progress or failure, candidate lifecycle states and reasons, and distinct stage outcomes. diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 34c784bd3..34f5362be 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -3,7 +3,15 @@ import { randomUUID } from "node:crypto"; import { mkdir, readdir, readFile, rename, stat, unlink, writeFile } from "node:fs/promises"; import { basename, join } from "node:path"; import { promisify } from "node:util"; -import { type AssistantMessage, contentText, type Message, type Model, type Tool } from "@earendil-works/diffex-ai"; +import { + type AssistantMessage, + contentText, + isRetryableAssistantError, + isRetryableProviderErrorMessage, + type Message, + type Model, + type Tool, +} from "@earendil-works/diffex-ai"; import type { HarnessStateStore, HarnessVersion } from "@earendil-works/diffex-harness-state"; import lockfile from "proper-lockfile"; import type { TSchema } from "typebox"; @@ -268,15 +276,33 @@ export class CodingAgentEvolutionModelExecutor implements EvolutionModelExecutor }); } request.signal?.throwIfAborted(); - if (!(error instanceof EvolutionProviderTimeoutError) || providerAttempt >= 2) throw error; + if (error instanceof EvolutionProviderTimeoutError && providerAttempt < 2) continue; + if (error instanceof EvolutionProviderTimeoutError) { + throw new EvolutionExecutionError(error.message, { + reasonCode: "model-call-failed", + retryable: true, + diagnostics: { providerTimeout: error.deadline }, + }); + } + const errorMessage = error instanceof Error ? error.message : String(error); + if (isRetryableProviderErrorMessage(errorMessage)) { + throw new EvolutionExecutionError(errorMessage, { + reasonCode: "model-call-failed", + retryable: true, + cause: error, + diagnostics: { transportError: errorMessage }, + }); + } + throw error; } } + request.signal?.throwIfAborted(); if (assistant.stopReason === "error" || assistant.stopReason === "aborted") { throw new EvolutionExecutionError( assistant.errorMessage ?? `Evolution model stopped with ${assistant.stopReason}`, { reasonCode: "model-call-failed", - retryable: true, + retryable: isRetryableAssistantError(assistant), diagnostics: { stopReason: assistant.stopReason, providerError: assistant.errorMessage ?? null, diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index 56781bca1..9c7f94ea3 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -659,13 +659,13 @@ describe("coding-agent evolution runtime", () => { }); }); - it("preserves provider stop diagnostics for evolution failures", async () => { + it("preserves transient provider stop diagnostics for evolution failures", async () => { const harness = await createHarness(); harnesses.push(harness); harness.setResponses([ fauxAssistantMessage("", { stopReason: "error", - errorMessage: "Provider rejected the structured response", + errorMessage: "Provider returned error: 503 service unavailable", }), ]); const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, { @@ -689,9 +689,48 @@ describe("coding-agent evolution runtime", () => { }), ).rejects.toMatchObject({ name: "EvolutionExecutionError", - message: "Provider rejected the structured response", + message: "Provider returned error: 503 service unavailable", reasonCode: "model-call-failed", retryable: true, + diagnostics: { + stopReason: "error", + providerError: "Provider returned error: 503 service unavailable", + }, + }); + }); + + it("classifies deterministic provider stops as non-retryable", async () => { + const harness = await createHarness(); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage("", { + stopReason: "error", + errorMessage: "Provider rejected the structured response", + }), + ]); + const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, { + async waitUntilBackgroundAllowed() {}, + }); + const model = harness.getModel(); + + await expect( + executor.execute({ + stage: "skill-synthesizer", + model: { + provider: model.provider, + model: model.id, + configurationDigest: `sha256:${"1".repeat(64)}`, + }, + promptVersion: "skill-synthesizer-v1", + idempotencyKey: "deterministic-provider-stop", + input: {}, + outputSchema: { type: "object" }, + tools: [], + }), + ).rejects.toMatchObject({ + name: "EvolutionExecutionError", + reasonCode: "model-call-failed", + retryable: false, diagnostics: { stopReason: "error", providerError: "Provider rejected the structured response", @@ -699,6 +738,40 @@ describe("coding-agent evolution runtime", () => { }); }); + it("classifies thrown provider transport failures as retryable", async () => { + const harness = await createHarness(); + harnesses.push(harness); + harness.setResponses([ + () => { + throw new Error("fetch failed: connection reset before headers"); + }, + ]); + const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, { + async waitUntilBackgroundAllowed() {}, + }); + const model = harness.getModel(); + + await expect( + executor.execute({ + stage: "skill-synthesizer", + model: { + provider: model.provider, + model: model.id, + configurationDigest: `sha256:${"1".repeat(64)}`, + }, + promptVersion: "skill-synthesizer-v1", + idempotencyKey: "provider-transport-failure", + input: {}, + outputSchema: { type: "object" }, + tools: [], + }), + ).rejects.toMatchObject({ + name: "EvolutionExecutionError", + reasonCode: "model-call-failed", + retryable: true, + }); + }); + it("corrects an invalid response-tool submission within the active transcript", async () => { const harness = await createHarness(); harnesses.push(harness); diff --git a/packages/evolution/RECOVERY.md b/packages/evolution/RECOVERY.md index d4b1c5ec3..143be15fc 100644 --- a/packages/evolution/RECOVERY.md +++ b/packages/evolution/RECOVERY.md @@ -26,7 +26,7 @@ Recovery never treats partial output as completed work and never feeds evolution | Successful rollback selection | The recorded immutable parent is still present and the evolved revision is current. | Select the parent with compare-and-swap and refresh on the next prompt boundary. | Existing in-flight prompts stay pinned to their admitted revision. | | Unsupported skill or orchestration target | The target registry has no enabled writer route. | Preserve the finding as deferred evidence with no executable payload. | Disabled target kinds cannot reach filesystem mutation or activation. | | Interrupted skill-candidate consolidation | A completed evolution job has checkpointed candidates but no completed candidate-work event. | Resume from append-only registry events with the same source-job idempotency key; repair missing scores or transitions before completion. | Candidate proposal revisions remain immutable, retries do not add recurrence, and partial registry appends cannot activate a skill. | -| Interrupted or failed skill synthesis, validation, evaluation, write, or activation | A qualified group is missing the next immutable artifact or transition, an accepted candidate has a durable write without a matching activation, or synthesis exhausted its bounded attempts. | Resume recoverable work with revision-bound idempotency keys; reuse recorded artifacts and receipts, repair the missing transition, and record exhausted synthesis as `synthesis-failed` until a new proposal revision changes its synthesis identity. | Only evidence-qualified groups invoke synthesis, unrelated later jobs cannot retry an exhausted group, only deterministically valid drafts enter evaluation, only accepted evaluations reach the writer, and retries cannot stage or activate a skill twice. | +| Interrupted or failed skill synthesis, validation, evaluation, write, or activation | A qualified group is missing the next immutable artifact or transition, an accepted candidate has a durable write without a matching activation, or synthesis returned a non-retryable error or exhausted its identity-scoped attempts. | Resume recoverable work with revision-bound idempotency keys; reuse recorded artifacts and receipts, repair the missing transition, and record terminal synthesis as `synthesis-failed` until a new proposal revision changes its synthesis identity. | Only transient provider or transport synthesis failures are retried, attempts follow the synthesis identity across source jobs, terminal groups do not block later qualified groups, only deterministically valid drafts enter evaluation, only accepted evaluations reach the writer, and retries cannot stage or activate a skill twice. | ## Operational observability diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index c2200527e..e05819b24 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -992,6 +992,8 @@ export interface SkillCandidateWorkErrorDiagnostic { export interface SkillCandidateGroupFailureDiagnostic { readonly groupId: string | null; + /** Present for synthesis failures so retry budgets follow the immutable synthesis identity across jobs. */ + readonly synthesisIdempotencyKey?: string; readonly error: SkillCandidateWorkErrorDiagnostic; } diff --git a/packages/evolution/src/filesystem-store.ts b/packages/evolution/src/filesystem-store.ts index 278850140..bb539fdaf 100644 --- a/packages/evolution/src/filesystem-store.ts +++ b/packages/evolution/src/filesystem-store.ts @@ -324,6 +324,7 @@ function isCandidateGroupFailureDiagnostic(value: unknown): boolean { return ( isRecord(value) && (value.groupId === null || typeof value.groupId === "string") && + (value.synthesisIdempotencyKey === undefined || typeof value.synthesisIdempotencyKey === "string") && isCandidateWorkErrorDiagnostic(value.error) ); } diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 376b2a0e2..f74ec2235 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -341,6 +341,7 @@ export interface SkillCandidateSynthesisFailureRecordingInput { readonly groupId: string; readonly proposalRevisionIds: readonly string[]; readonly idempotencyKey: string; + readonly reasonCode: "skill-synthesis-attempts-exhausted" | "skill-synthesis-non-retryable"; }[]; readonly createdAt: string; } diff --git a/packages/evolution/src/skill-candidate-registry.ts b/packages/evolution/src/skill-candidate-registry.ts index 53fdd9e2d..a97fd2753 100644 --- a/packages/evolution/src/skill-candidate-registry.ts +++ b/packages/evolution/src/skill-candidate-registry.ts @@ -865,7 +865,7 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry failure.groupId, "evidence-qualified", "synthesis-failed", - ["skill-synthesis-attempts-exhausted"], + [failure.reasonCode], nextSequence(snapshot.events, planned.length), failure.idempotencyKey, this.#producer, diff --git a/packages/evolution/src/skill-candidate-worker.ts b/packages/evolution/src/skill-candidate-worker.ts index 194144711..9f4447ede 100644 --- a/packages/evolution/src/skill-candidate-worker.ts +++ b/packages/evolution/src/skill-candidate-worker.ts @@ -283,14 +283,38 @@ function errorDiagnostic(error: unknown): SkillCandidateWorkErrorDiagnostic { }; } +function synthesisAttemptCount( + events: readonly SkillCandidateRegistryEvent[], + synthesisIdempotencyKey: string, +): number { + return events.reduce( + (count, event) => + event.type === "skill-candidate-work-failed" && + event.stage === "synthesizing" && + event.diagnostics?.some((diagnostic) => diagnostic.synthesisIdempotencyKey === synthesisIdempotencyKey) + ? count + 1 + : count, + 0, + ); +} + +interface SkillCandidateWorkStageFailure { + readonly groupId: string; + readonly synthesisIdempotencyKey?: string; + readonly error: unknown; +} + class SkillCandidateWorkStageError extends AggregateError { readonly stage: SkillCandidateWorkStage; - readonly failures: readonly { readonly groupId: string; readonly error: unknown }[]; + readonly failures: readonly SkillCandidateWorkStageFailure[]; + readonly retryable: boolean; + readonly retryAttempt: number; constructor( stage: SkillCandidateWorkStage, - failures: readonly { readonly groupId: string; readonly error: unknown }[], + failures: readonly SkillCandidateWorkStageFailure[], message: string, + options: { readonly retryable: boolean; readonly retryAttempt: number }, ) { super( failures.map((failure) => failure.error), @@ -299,6 +323,8 @@ class SkillCandidateWorkStageError extends AggregateError { this.name = "SkillCandidateWorkStageError"; this.stage = stage; this.failures = failures; + this.retryable = options.retryable; + this.retryAttempt = options.retryAttempt; } } @@ -525,6 +551,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor job: EvolutionJob, attempt: number, retryAttempt: number, + retryable: boolean, stage: SkillCandidateWorkStage, diagnostics: readonly SkillCandidateGroupFailureDiagnostic[], error: unknown, @@ -554,12 +581,11 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor idempotencyKey, attempt, message: error instanceof Error ? error.message : String(error), - retryNotBefore: - retryAttempt >= this.#maxAttempts - ? null - : new Date( - Date.parse(now) + retryDelay(retryAttempt, this.#retryBaseDelayMs, this.#retryMaxDelayMs), - ).toISOString(), + retryNotBefore: retryable + ? new Date( + Date.parse(now) + retryDelay(retryAttempt, this.#retryBaseDelayMs, this.#retryMaxDelayMs), + ).toISOString() + : null, stage, diagnostics, }; @@ -656,10 +682,8 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor if (skillCheckpoint === undefined || !hasSkillCandidates(job)) { throw new Error(`Evolution job "${sourceEvolutionJobId}" has no checkpointed skill candidates`); } - const history = workEvents( - (await this.#dependencies.store.loadSkillCandidateRegistry(job.metadata.scope)).events, - job.jobId, - ); + const registryEvents = (await this.#dependencies.store.loadSkillCandidateRegistry(job.metadata.scope)).events; + const history = workEvents(registryEvents, job.jobId); const terminal = latestWorkEvent(history); if ( terminal?.type === "skill-candidate-work-checkpointed" && @@ -685,9 +709,6 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor latestLegacyFailure === null || failure.registrySequence > latestLegacyFailure.registrySequence, ).length + 1; const attempt = failures.reduce((highest, failure) => Math.max(highest, failure.attempt), 0) + 1; - if (retryAttempt > this.#maxAttempts) { - throw new Error(`Skill candidate work for "${job.jobId}" exhausted retries`); - } const recoveryGroupIds = legacySynthesisGroupIds(history); let activeStage: SkillCandidateWorkStage = "consolidating"; @@ -757,6 +778,9 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor readonly proposalRevisionIds: readonly string[]; readonly idempotencyKey: string; readonly error: unknown; + readonly attempt: number; + readonly retryPending: boolean; + readonly reasonCode: "skill-synthesis-attempts-exhausted" | "skill-synthesis-non-retryable"; }[] = []; for (const group of synthesisTargets) { activeGroupId = group.groupId; @@ -774,11 +798,16 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor }); } catch (error) { signal.throwIfAborted(); + const synthesisAttempt = synthesisAttemptCount(registryEvents, group.synthesisIdempotencyKey) + 1; + const retryable = error instanceof EvolutionExecutionError && error.retryable; synthesisFailures.push({ groupId: group.groupId, proposalRevisionIds: group.proposalRevisions.map((revision) => revision.revisionId), idempotencyKey: group.synthesisIdempotencyKey, error, + attempt: synthesisAttempt, + retryPending: retryable && synthesisAttempt < this.#maxAttempts, + reasonCode: retryable ? "skill-synthesis-attempts-exhausted" : "skill-synthesis-non-retryable", }); continue; } @@ -797,11 +826,17 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor ); } activeGroupId = null; - if (synthesisFailures.length > 0 && retryAttempt >= this.#maxAttempts) { + const terminalSynthesisFailures = synthesisFailures.filter((failure) => !failure.retryPending); + if (terminalSynthesisFailures.length > 0) { await this.#dependencies.registry.recordSynthesisFailures({ scope: job.metadata.scope, lease: claim(), - failures: synthesisFailures.map(({ error: _error, ...failure }) => failure), + failures: terminalSynthesisFailures.map((failure) => ({ + groupId: failure.groupId, + proposalRevisionIds: failure.proposalRevisionIds, + idempotencyKey: failure.idempotencyKey, + reasonCode: failure.reasonCode, + })), createdAt: this.#dependencies.clock.now(), }); } @@ -928,10 +963,19 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor } activeGroupId = null; if (synthesisFailures.length > 0) { + const retryableFailures = synthesisFailures.filter((failure) => failure.retryPending); throw new SkillCandidateWorkStageError( "synthesizing", - synthesisFailures, + synthesisFailures.map((failure) => ({ + groupId: failure.groupId, + synthesisIdempotencyKey: failure.idempotencyKey, + error: failure.error, + })), `Skill synthesis failed for candidate groups: ${synthesisFailures.map((failure) => failure.groupId).join(", ")}`, + { + retryable: retryableFailures.length > 0, + retryAttempt: Math.max(1, ...retryableFailures.map((failure) => failure.attempt)), + }, ); } activeStage = "completed"; @@ -949,10 +993,25 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor error instanceof SkillCandidateWorkStageError ? error.failures.map((failure) => ({ groupId: failure.groupId, + ...(failure.synthesisIdempotencyKey === undefined + ? {} + : { synthesisIdempotencyKey: failure.synthesisIdempotencyKey }), error: errorDiagnostic(failure.error), })) : [{ groupId: activeGroupId, error: errorDiagnostic(error) }]; - await this.#appendFailure(job, attempt, retryAttempt, stage, diagnostics, error, claim()); + const failureRetryAttempt = error instanceof SkillCandidateWorkStageError ? error.retryAttempt : retryAttempt; + const failureRetryable = + error instanceof SkillCandidateWorkStageError ? error.retryable : retryAttempt < this.#maxAttempts; + await this.#appendFailure( + job, + attempt, + failureRetryAttempt, + failureRetryable, + stage, + diagnostics, + error, + claim(), + ); throw error; } } diff --git a/packages/evolution/test/filesystem-store.test.ts b/packages/evolution/test/filesystem-store.test.ts index 625071b4e..0a221c5f6 100644 --- a/packages/evolution/test/filesystem-store.test.ts +++ b/packages/evolution/test/filesystem-store.test.ts @@ -155,6 +155,7 @@ function candidateWorkFailureEvent(eventId: string, registrySequence: number): S diagnostics: [ { groupId: "group-1", + synthesisIdempotencyKey: "skill-synthesis-group-1", error: { name: "EvolutionExecutionError", message: "Skill synthesizer failed to produce a draft", diff --git a/packages/evolution/test/skill-candidate-registry.test.ts b/packages/evolution/test/skill-candidate-registry.test.ts index 126f2b7be..57d5c6027 100644 --- a/packages/evolution/test/skill-candidate-registry.test.ts +++ b/packages/evolution/test/skill-candidate-registry.test.ts @@ -370,6 +370,7 @@ describe("persistent skill candidate registry", () => { groupId: group.groupId, proposalRevisionIds: group.proposalRevisions.map((revision) => revision.revisionId), idempotencyKey: group.synthesisIdempotencyKey, + reasonCode: "skill-synthesis-attempts-exhausted", }, ], createdAt: "2026-08-26T12:01:00.000Z", @@ -394,6 +395,7 @@ describe("persistent skill candidate registry", () => { groupId: group.groupId, proposalRevisionIds: group.proposalRevisions.map((revision) => revision.revisionId), idempotencyKey: group.synthesisIdempotencyKey, + reasonCode: "skill-synthesis-attempts-exhausted", }, ], createdAt: "2026-08-26T12:01:00.000Z", diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index 62c2723ce..50435ff61 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -153,6 +153,13 @@ function completedJob( }; } +function transientSynthesisError(message = "temporary provider failure"): EvolutionExecutionError { + return new EvolutionExecutionError(message, { + reasonCode: "model-call-failed", + retryable: true, + }); +} + function qualifiedGroup(candidateReference: EvolutionArtifactReference): SkillCandidateGroup { return { groupId: "group-qualified", @@ -401,7 +408,7 @@ class FixtureRegistry implements SkillCandidateRegistry { eventId: `synthesis-failed-${failure.groupId}`, from: group.state, to: "synthesis-failed" as const, - reasonCodes: ["skill-synthesis-attempts-exhausted"], + reasonCodes: [failure.reasonCode], createdAt: input.createdAt, }, ], @@ -828,9 +835,10 @@ describe("skill candidate registry worker", () => { ]); expect(data.validationRequests.map((request) => request.group.groupId)).toEqual(["group-succeeding"]); expect(data.evaluationRequests.map((request) => request.group.groupId)).toEqual(["group-succeeding"]); - expect(data.registry.groups.find((candidateGroup) => candidateGroup.groupId === "group-failing")?.state).toBe( - "evidence-qualified", - ); + expect(data.registry.groups.find((candidateGroup) => candidateGroup.groupId === "group-failing")).toMatchObject({ + state: "synthesis-failed", + dispositionHistory: [expect.objectContaining({ reasonCodes: ["skill-synthesis-non-retryable"] })], + }); expect(data.registry.groups.find((candidateGroup) => candidateGroup.groupId === "group-succeeding")?.state).toBe( "accepted", ); @@ -840,6 +848,7 @@ describe("skill candidate registry worker", () => { diagnostics: [ { groupId: "group-failing", + synthesisIdempotencyKey: "skill-synthesis-failing", error: { name: "EvolutionExecutionError", message: "Skill synthesizer failed to produce a draft", @@ -850,27 +859,13 @@ describe("skill candidate registry worker", () => { }, }, ], + retryNotBefore: null, }); expect( data.store.events.some( (event) => event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", ), ).toBe(false); - - data.synthesisFailures.clear(); - await data.worker.process(data.job.jobId); - - expect(data.synthesisRequests.map((request) => request.group.groupId)).toEqual([ - "group-failing", - "group-succeeding", - "group-failing", - ]); - expect(data.registry.groups.map((candidateGroup) => candidateGroup.state)).toEqual(["accepted", "accepted"]); - expect( - data.store.events.some( - (event) => event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", - ), - ).toBe(true); }); it("reconciles legacy exhausted synthesis by identity under fresh bounded retries", async () => { @@ -905,7 +900,7 @@ describe("skill candidate registry worker", () => { evaluationIdempotencyKey: "skill-evaluation-unrelated", }, ); - data.synthesisFailures.set("group-failing", new Error("synthesis failed")); + data.synthesisFailures.set("group-failing", transientSynthesisError()); const source = new PollingSkillCandidateWorkSource({ clock: data.clock, jobs: data.jobs, @@ -973,6 +968,35 @@ describe("skill candidate registry worker", () => { expect(await source.nextRunnableJob(now)).toBeNull(); }); + it("tracks transient synthesis attempts by identity across source evolution jobs", async () => { + const data = fixture(true); + const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; + if (candidateReference === undefined) throw new Error("Expected candidate reference"); + data.registry.groups.push(qualifiedGroup(candidateReference)); + data.synthesisFailures.set("group-qualified", transientSynthesisError()); + + for (let attempt = 1; attempt <= 3; attempt += 1) { + const job = attempt === 1 ? data.job : { ...data.job, jobId: `job-${attempt}` }; + if (attempt > 1) data.jobs.jobs.push(job); + await expect(data.worker.process(job.jobId)).rejects.toThrow( + "Skill synthesis failed for candidate groups: group-qualified", + ); + const failure = data.store.events.find( + (event) => event.type === "skill-candidate-work-failed" && event.sourceEvolutionJobId === job.jobId, + ); + expect(failure).toMatchObject({ + retryNotBefore: attempt === 3 ? null : expect.any(String), + diagnostics: [expect.objectContaining({ synthesisIdempotencyKey: "skill-synthesis-qualified" })], + }); + } + + expect(data.synthesisRequests).toHaveLength(3); + expect(data.registry.groups[0]).toMatchObject({ + state: "synthesis-failed", + dispositionHistory: [expect.objectContaining({ reasonCodes: ["skill-synthesis-attempts-exhausted"] })], + }); + }); + it("durably exposes exhausted synthesis without blocking a later candidate in the same batch", async () => { const data = fixture(true); const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; @@ -994,7 +1018,7 @@ describe("skill candidate registry worker", () => { evaluationIdempotencyKey: "skill-evaluation-succeeding", }, ); - data.synthesisFailures.set("group-failing", new Error("synthesis failed")); + data.synthesisFailures.set("group-failing", transientSynthesisError()); for (let attempt = 1; attempt <= 3; attempt += 1) { await expect(data.worker.process(data.job.jobId)).rejects.toThrow(