From 91580310c7e7423f32458d1bf370ccccc7f3ebf4 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sat, 29 Aug 2026 22:48:40 -0400 Subject: [PATCH 01/10] feat(agent): skip unavailable skill evaluation --- packages/evolution/src/contracts.ts | 2 +- packages/evolution/src/ports.ts | 1 + .../evolution/src/skill-candidate-registry.ts | 73 +++++++++++++++++++ .../evolution/src/skill-candidate-worker.ts | 30 +++++++- packages/evolution/src/skill-writer.ts | 69 +++++++++++------- .../test/skill-candidate-registry.test.ts | 36 +++++++++ .../test/skill-candidate-worker.test.ts | 54 ++++++++++---- packages/evolution/test/skill-writer.test.ts | 47 +++++++++++- 8 files changed, 266 insertions(+), 46 deletions(-) diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 3a37b0d08..4dc934edd 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -1235,7 +1235,7 @@ export interface SkillMutation { readonly proposalRevisionIds: readonly string[]; readonly draft: EvolutionArtifactReference; readonly validation: EvolutionArtifactReference; - readonly evaluation: EvolutionArtifactReference; + readonly evaluation: EvolutionArtifactReference | null; readonly memoryDisposition: SkillMutationMemoryDisposition; readonly existingSkills: readonly SkillMutationExistingArtifact[]; readonly skill: SkillMutationArtifact | null; diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 7b9f761ab..efd4b807e 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -382,6 +382,7 @@ export interface SkillCandidateRegistry { consolidate(input: SkillCandidateRegistryConsolidationInput): Promise; recordDraft(input: SkillCandidateDraftRecordingInput): Promise; recordValidation(input: SkillCandidateValidationRecordingInput): Promise; + skipEvaluation(input: SkillCandidateEvaluationStartInput): Promise; startEvaluation(input: SkillCandidateEvaluationStartInput): Promise; recordEvaluation(input: SkillCandidateEvaluationRecordingInput): Promise; list(scope: EvolutionScope): Promise; diff --git a/packages/evolution/src/skill-candidate-registry.ts b/packages/evolution/src/skill-candidate-registry.ts index 0440de03f..2baa1c6d1 100644 --- a/packages/evolution/src/skill-candidate-registry.ts +++ b/packages/evolution/src/skill-candidate-registry.ts @@ -1079,6 +1079,79 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry throw new Error("Skill candidate registry changed too often to start evaluation"); } + async skipEvaluation(input: SkillCandidateEvaluationStartInput): Promise { + for (let attempt = 0; attempt < this.#maxCommitAttempts; attempt += 1) { + const snapshot = await this.#store.loadSkillCandidateRegistry(input.scope); + const mutable = materializeGroups(snapshot.events).get(input.groupId); + if (mutable === undefined) throw new Error(`Unknown candidate group "${input.groupId}"`); + const group = immutableGroup(mutable); + if ( + !sameValues( + input.proposalRevisionIds, + group.proposalRevisions.map((revision) => revision.revisionId), + ) || + group.draft?.artifactId !== input.draft.artifactId || + group.draft.contentDigest !== input.draft.contentDigest || + group.validation?.artifactId !== input.validation.artifactId || + group.validation.contentDigest !== input.validation.contentDigest || + input.idempotencyKey !== group.evaluationIdempotencyKey + ) { + throw new Error(`Candidate group "${input.groupId}" changed before evaluation was skipped`); + } + if ( + group.state === "accepted" && + group.evaluation === null && + group.dispositionHistory.at(-1)?.reasonCodes.includes("retrieval-and-behavior-evaluation-skipped") + ) { + return group; + } + if ( + (group.state !== "structurally-valid" && group.state !== "evaluation-pending") || + group.evaluation !== null + ) { + throw new Error(`Candidate group "${input.groupId}" is not eligible to skip evaluation`); + } + const validation = await this.#artifacts.get(input.validation); + if ( + !("artifactKind" in validation) || + validation.artifactKind !== "skill-draft-validation" || + validation.disposition !== "accepted" || + computeEvolutionContentDigest(validation) !== input.validation.contentDigest + ) { + throw new Error(`Candidate group "${input.groupId}" does not have accepted validation`); + } + const transition = transitionEvent( + input.groupId, + group.state, + "accepted", + ["retrieval-and-behavior-evaluation-skipped"], + nextSequence(snapshot.events, 0), + input.idempotencyKey, + this.#producer, + input.scope, + input.createdAt, + ); + const committed = await this.#store.appendSkillCandidateRegistryEvents( + input.scope, + snapshot.revision, + [transition], + input.lease, + ); + if (committed === "committed") { + mutable.state = "accepted"; + mutable.dispositionHistory.push({ + eventId: transition.eventId, + from: transition.from, + to: transition.to, + reasonCodes: transition.reasonCodes, + createdAt: input.createdAt, + }); + return immutableGroup(mutable); + } + } + throw new Error("Skill candidate registry changed too often to skip evaluation"); + } + async recordEvaluation(input: SkillCandidateEvaluationRecordingInput): Promise { for (let attempt = 0; attempt < this.#maxCommitAttempts; attempt += 1) { const snapshot = await this.#store.loadSkillCandidateRegistry(input.scope); diff --git a/packages/evolution/src/skill-candidate-worker.ts b/packages/evolution/src/skill-candidate-worker.ts index cb05afd3b..3977de8bc 100644 --- a/packages/evolution/src/skill-candidate-worker.ts +++ b/packages/evolution/src/skill-candidate-worker.ts @@ -231,7 +231,6 @@ export class PollingSkillCandidateWorkSource { (event) => event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", ); if (completed && !(this.#dependencies.evaluatorAvailable && awaitingEvaluation)) continue; - if (awaitingEvaluation && !this.#dependencies.evaluatorAvailable) continue; const latestFailure = events .filter((event): event is SkillCandidateWorkFailedEvent => event.type === "skill-candidate-work-failed") .sort((left, right) => right.attempt - left.attempt)[0]; @@ -619,8 +618,33 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor } await this.#appendCheckpoint(job, "evaluating", attempt, [...evaluationGroups.keys()], claim()); const evaluatedGroupIds: string[] = []; - if (this.#dependencies.evaluator === undefined && evaluationGroups.size > 0) return; - if (this.#dependencies.evaluator !== undefined) { + if (this.#dependencies.evaluator === undefined) { + for (const evaluationGroup of evaluationGroups.values()) { + signal.throwIfAborted(); + if (evaluationGroup.draft === null || evaluationGroup.validation === null) { + throw new Error(`Candidate group "${evaluationGroup.groupId}" lacks evaluation prerequisites`); + } + const common = { + scope: job.metadata.scope, + lease: claim(), + groupId: evaluationGroup.groupId, + draft: evaluationGroup.draft, + validation: evaluationGroup.validation, + proposalRevisionIds: evaluationGroup.proposalRevisions.map((revision) => revision.revisionId), + idempotencyKey: evaluationGroup.evaluationIdempotencyKey, + createdAt: this.#dependencies.clock.now(), + }; + if (evaluationGroup.evaluation === null) { + await this.#dependencies.registry.skipEvaluation(common); + } else { + await this.#dependencies.registry.recordEvaluation({ + ...common, + evaluation: evaluationGroup.evaluation, + }); + } + evaluatedGroupIds.push(evaluationGroup.groupId); + } + } else { for (const evaluationGroup of evaluationGroups.values()) { signal.throwIfAborted(); if (evaluationGroup.draft === null || evaluationGroup.validation === null) { diff --git a/packages/evolution/src/skill-writer.ts b/packages/evolution/src/skill-writer.ts index 0c6b56ca9..6d32ec7a1 100644 --- a/packages/evolution/src/skill-writer.ts +++ b/packages/evolution/src/skill-writer.ts @@ -83,7 +83,7 @@ interface AcceptedSkillMutationSource { readonly candidates: readonly SkillCandidate[]; readonly draft: SkillDraft; readonly validation: SkillDraftValidationReport; - readonly evaluation: SkillEvaluationReport; + readonly evaluation: SkillEvaluationReport | null; } export interface SkillWriterBuildInput { @@ -399,12 +399,20 @@ export class SkillWriter { async #acceptedSource(scope: EvolutionScope, groupId: string): Promise { const group = (await this.#dependencies.registry.list(scope)).find((candidate) => candidate.groupId === groupId); + const evaluationSkipTransition = group?.dispositionHistory.at(-1); + const evaluationSkipped = + group?.evaluation === null && + evaluationSkipTransition?.to === "accepted" && + (evaluationSkipTransition.from === "structurally-valid" || + evaluationSkipTransition.from === "evaluation-pending") && + evaluationSkipTransition.reasonCodes.length === 1 && + evaluationSkipTransition.reasonCodes[0] === "retrieval-and-behavior-evaluation-skipped"; if ( group === undefined || group.state !== "accepted" || group.draft === null || group.validation === null || - group.evaluation === null + (group.evaluation === null && !evaluationSkipped) ) { throw new Error(`Candidate group "${groupId}" is not accepted for skill activation`); } @@ -412,14 +420,14 @@ export class SkillWriter { Promise.all(group.proposalRevisions.map((revision) => this.#dependencies.artifacts.get(revision.candidate))), this.#dependencies.artifacts.get(group.draft), this.#dependencies.artifacts.get(group.validation), - this.#dependencies.artifacts.get(group.evaluation), + group.evaluation === null ? Promise.resolve(null) : this.#dependencies.artifacts.get(group.evaluation), ]); if ( candidateArtifacts.length === 0 || !candidateArtifacts.every(isSkillCandidate) || !isSkillDraft(draftArtifact) || !isSkillValidation(validationArtifact) || - !isSkillEvaluation(evaluationArtifact) + (evaluationArtifact !== null && !isSkillEvaluation(evaluationArtifact)) ) { throw new Error(`Candidate group "${groupId}" has invalid writer prerequisites`); } @@ -433,7 +441,9 @@ export class SkillWriter { } assertArtifactIdentity(draftArtifact, group.draft, "Skill draft"); assertArtifactIdentity(validationArtifact, group.validation, "Skill validation"); - assertArtifactIdentity(evaluationArtifact, group.evaluation, "Skill evaluation"); + if (evaluationArtifact !== null && group.evaluation !== null) { + assertArtifactIdentity(evaluationArtifact, group.evaluation, "Skill evaluation"); + } if ( group.profile.workspace.evolutionScope.repositoryId !== scope.repositoryId || group.profile.workspace.evolutionScope.scopeId !== scope.scopeId || @@ -441,24 +451,25 @@ export class SkillWriter { draftArtifact.metadata.scope.scopeId !== scope.scopeId || validationArtifact.metadata.scope.repositoryId !== scope.repositoryId || validationArtifact.metadata.scope.scopeId !== scope.scopeId || - evaluationArtifact.metadata.scope.repositoryId !== scope.repositoryId || - evaluationArtifact.metadata.scope.scopeId !== scope.scopeId || draftArtifact.groupId !== group.groupId || validationArtifact.groupId !== group.groupId || - evaluationArtifact.groupId !== group.groupId || !sameReference(validationArtifact.draft, group.draft) || - !sameReference(evaluationArtifact.draft, group.draft) || - !sameReference(evaluationArtifact.validation, group.validation) || validationArtifact.disposition !== "accepted" || - evaluationArtifact.disposition !== "accepted" || - !evaluationArtifact.gates.suite.passed || - !evaluationArtifact.gates.retrieval.passed || - !evaluationArtifact.gates.behavior.passed || !sameValues( draftArtifact.proposalRevisionIds, group.proposalRevisions.map((revision) => revision.revisionId), ) || - !sameValues(evaluationArtifact.proposalRevisionIds, draftArtifact.proposalRevisionIds) + (evaluationArtifact !== null && + (evaluationArtifact.metadata.scope.repositoryId !== scope.repositoryId || + evaluationArtifact.metadata.scope.scopeId !== scope.scopeId || + evaluationArtifact.groupId !== group.groupId || + !sameReference(evaluationArtifact.draft, group.draft) || + !sameReference(evaluationArtifact.validation, group.validation) || + evaluationArtifact.disposition !== "accepted" || + !evaluationArtifact.gates.suite.passed || + !evaluationArtifact.gates.retrieval.passed || + !evaluationArtifact.gates.behavior.passed || + !sameValues(evaluationArtifact.proposalRevisionIds, draftArtifact.proposalRevisionIds))) ) { throw new Error(`Candidate group "${groupId}" accepted artifacts are stale or incomplete`); } @@ -532,7 +543,7 @@ export class SkillWriter { draft.workspace.evolutionScope.repositoryId !== group.profile.workspace.evolutionScope.repositoryId || draft.workspace.evolutionScope.scopeId !== group.profile.workspace.evolutionScope.scopeId || validation.validatedHarnessVersion !== validatedBase.manifest.version || - evaluation.harnessVersion !== validatedBase.manifest.version || + (evaluation !== null && evaluation.harnessVersion !== validatedBase.manifest.version) || serializedSkills(base.state.skills) !== serializedSkills(validatedBase.state.skills) ) { throw new EvolutionExecutionError( @@ -553,14 +564,15 @@ export class SkillWriter { throw new Error(`Candidate group "${group.groupId}" has a noncanonical validated inventory`); } if ( - evaluation.candidateSkill.skillId !== draft.draftId || - evaluation.candidateSkill.name !== draft.proposedName || - evaluation.candidateSkill.version !== - computeHarnessSkillPackageVersion({ content: draft.content, files: draft.files }) || - evaluation.candidateSkill.content !== draft.content || - evaluation.candidateSkill.contentDigest !== computeEvolutionTextDigest(draft.content) || - serializeEvolutionValue(sortPackageFiles(evaluation.candidateSkill.files ?? [])) !== - serializeEvolutionValue(sortPackageFiles(draft.files)) + evaluation !== null && + (evaluation.candidateSkill.skillId !== draft.draftId || + evaluation.candidateSkill.name !== draft.proposedName || + evaluation.candidateSkill.version !== + computeHarnessSkillPackageVersion({ content: draft.content, files: draft.files }) || + evaluation.candidateSkill.content !== draft.content || + evaluation.candidateSkill.contentDigest !== computeEvolutionTextDigest(draft.content) || + serializeEvolutionValue(sortPackageFiles(evaluation.candidateSkill.files ?? [])) !== + serializeEvolutionValue(sortPackageFiles(draft.files))) ) { throw new Error(`Candidate group "${group.groupId}" evaluation does not match its draft`); } @@ -593,7 +605,7 @@ export class SkillWriter { } const draftReference = artifactReference(draft); const validationReference = artifactReference(validation); - const evaluationReference = artifactReference(evaluation); + const evaluationReference = evaluation === null ? null : artifactReference(evaluation); const memoryDisposition = this.#memoryDisposition(source); return { mutationId: stableId("skill-mutation", { @@ -638,7 +650,10 @@ export class SkillWriter { source: AcceptedSkillMutationSource, base: HarnessVersionBundle, ): Promise<{ readonly mutation: SkillMutation; readonly validatedBase: HarnessVersionBundle }> { - if (source.validation.validatedHarnessVersion !== source.evaluation.harnessVersion) { + if ( + source.evaluation !== null && + source.validation.validatedHarnessVersion !== source.evaluation.harnessVersion + ) { throw new EvolutionExecutionError("Accepted skill candidate validation and evaluation bases differ", { reasonCode: "harness-base-conflict", retryable: false, @@ -675,7 +690,7 @@ export class SkillWriter { input.groupId, mutation.draft.artifactId, mutation.validation.artifactId, - mutation.evaluation.artifactId, + ...(mutation.evaluation === null ? [] : [mutation.evaluation.artifactId]), ], scope: input.scope, createdAt: input.createdAt, diff --git a/packages/evolution/test/skill-candidate-registry.test.ts b/packages/evolution/test/skill-candidate-registry.test.ts index 02eeb415c..ce77007c6 100644 --- a/packages/evolution/test/skill-candidate-registry.test.ts +++ b/packages/evolution/test/skill-candidate-registry.test.ts @@ -492,10 +492,46 @@ describe("persistent skill candidate registry", () => { idempotencyKey: validated.evaluationIdempotencyKey, createdAt: "2026-08-26T12:03:00.000Z", }; + const skipStore = new MemoryRegistryStore(); + skipStore.events.push(...fixture.store.events); + const skipRegistry = new PersistentSkillCandidateRegistry({ + version: "1", + artifacts: fixture.artifacts, + store: skipStore, + }); + const skipped = await skipRegistry.skipEvaluation(evaluationStart); + const skippedRetry = await skipRegistry.skipEvaluation(evaluationStart); + expect(skipped).toMatchObject({ state: "accepted", evaluation: null }); + expect(skippedRetry).toMatchObject({ state: "accepted", evaluation: null }); + expect(skipped.dispositionHistory.at(-1)).toMatchObject({ + from: "structurally-valid", + to: "accepted", + reasonCodes: ["retrieval-and-behavior-evaluation-skipped"], + }); + const pending = await fixture.registry.startEvaluation(evaluationStart); const pendingRetry = await fixture.registry.startEvaluation(evaluationStart); expect(pending).toMatchObject({ state: "evaluation-pending", evaluation: null }); expect(pendingRetry).toMatchObject({ state: "evaluation-pending", evaluation: null }); + const pendingSkipStore = new MemoryRegistryStore(); + pendingSkipStore.events.push(...fixture.store.events); + const pendingSkipRegistry = new PersistentSkillCandidateRegistry({ + version: "1", + artifacts: fixture.artifacts, + store: pendingSkipStore, + }); + expect(await pendingSkipRegistry.skipEvaluation(evaluationStart)).toMatchObject({ + state: "accepted", + evaluation: null, + dispositionHistory: [ + ...pending.dispositionHistory, + expect.objectContaining({ + from: "evaluation-pending", + to: "accepted", + reasonCodes: ["retrieval-and-behavior-evaluation-skipped"], + }), + ], + }); const configurationDigest = `sha256:${"6".repeat(64)}`; const evaluationContent = { artifactKind: "skill-evaluation" as const, diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index 3edaf6bd8..109cc6195 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -318,6 +318,7 @@ class FixtureRegistry implements SkillCandidateRegistry { readonly requests: SkillCandidateRegistryConsolidationInput[] = []; readonly draftRequests: SkillCandidateDraftRecordingInput[] = []; readonly validationRequests: SkillCandidateValidationRecordingInput[] = []; + readonly evaluationSkipRequests: SkillCandidateEvaluationStartInput[] = []; readonly evaluationStartRequests: SkillCandidateEvaluationStartInput[] = []; readonly evaluationRequests: SkillCandidateEvaluationRecordingInput[] = []; readonly groups: SkillCandidateGroup[] = []; @@ -359,6 +360,31 @@ class FixtureRegistry implements SkillCandidateRegistry { return validated; } + async skipEvaluation(input: SkillCandidateEvaluationStartInput): Promise { + this.evaluationSkipRequests.push(input); + const index = this.groups.findIndex((candidateGroup) => candidateGroup.groupId === input.groupId); + const group = this.groups[index]; + if (group === undefined) throw new Error("Unknown group"); + const accepted = { + ...group, + state: "accepted" as const, + draft: input.draft, + validation: input.validation, + dispositionHistory: [ + ...group.dispositionHistory, + { + eventId: "evaluation-skipped", + from: group.state, + to: "accepted" as const, + reasonCodes: ["retrieval-and-behavior-evaluation-skipped"], + createdAt: input.createdAt, + }, + ], + }; + this.groups[index] = accepted; + return accepted; + } + async startEvaluation(input: SkillCandidateEvaluationStartInput): Promise { this.evaluationStartRequests.push(input); const index = this.groups.findIndex((candidateGroup) => candidateGroup.groupId === input.groupId); @@ -679,7 +705,7 @@ describe("skill candidate registry worker", () => { ]); }); - it("parks structurally valid drafts until an evaluator becomes available", async () => { + it("accepts structurally valid drafts when retrieval and behavior evaluation is unavailable", async () => { const data = fixture(); const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; if (candidateReference === undefined) throw new Error("Expected candidate reference"); @@ -687,28 +713,30 @@ describe("skill candidate registry worker", () => { await data.worker.process(data.job.jobId); - expect(data.registry.groups[0]?.state).toBe("structurally-valid"); + expect(data.registry.groups[0]).toMatchObject({ + state: "accepted", + evaluation: null, + dispositionHistory: [expect.objectContaining({ reasonCodes: ["retrieval-and-behavior-evaluation-skipped"] })], + }); + expect(data.registry.evaluationSkipRequests).toEqual([ + expect.objectContaining({ + groupId: "group-qualified", + idempotencyKey: "skill-evaluation-qualified", + }), + ]); expect( data.store.events.some( (event) => event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", ), - ).toBe(false); - const unavailableSource = new PollingSkillCandidateWorkSource({ + ).toBe(true); + const source = new PollingSkillCandidateWorkSource({ clock: data.clock, jobs: data.jobs, store: data.store, registry: data.registry, evaluatorAvailable: false, }); - const availableSource = new PollingSkillCandidateWorkSource({ - clock: data.clock, - jobs: data.jobs, - store: data.store, - registry: data.registry, - evaluatorAvailable: true, - }); - expect(await unavailableSource.nextRunnableJob(now)).toBeNull(); - expect(await availableSource.nextRunnableJob(now)).toBe(data.job.jobId); + expect(await source.nextRunnableJob(now)).toBeNull(); }); it("reopens legacy completed work when its evaluator becomes available", async () => { diff --git a/packages/evolution/test/skill-writer.test.ts b/packages/evolution/test/skill-writer.test.ts index 67fed8358..cc74e5c72 100644 --- a/packages/evolution/test/skill-writer.test.ts +++ b/packages/evolution/test/skill-writer.test.ts @@ -82,6 +82,7 @@ function registry(groups: readonly SkillCandidateGroup[]): SkillCandidateRegistr consolidate: unsupported, recordDraft: unsupported, recordValidation: unsupported, + skipEvaluation: unsupported, startEvaluation: unsupported, recordEvaluation: unsupported, }; @@ -95,6 +96,7 @@ interface WriterFixtureOptions { readonly retainedNames?: readonly string[]; readonly proposedName: string; readonly files?: SkillDraft["files"]; + readonly evaluationSkipped?: boolean; } async function writerFixture(options: WriterFixtureOptions) { @@ -306,7 +308,17 @@ async function writerFixture(options: WriterFixtureOptions) { }, ], state: "accepted", - dispositionHistory: [], + dispositionHistory: options.evaluationSkipped + ? [ + { + eventId: "evaluation-skipped", + from: "structurally-valid", + to: "accepted", + reasonCodes: ["retrieval-and-behavior-evaluation-skipped"], + createdAt: now, + }, + ] + : [], evidence: { supportingEvidenceIds: ["evidence-1"], contradictingEvidenceIds: [], @@ -318,7 +330,7 @@ async function writerFixture(options: WriterFixtureOptions) { priorityScore: null, draft: draftReference, validation: validationReference, - evaluation: reference(evaluation), + evaluation: options.evaluationSkipped ? null : reference(evaluation), synthesisIdempotencyKey: draft.synthesisIdempotencyKey, validationIdempotencyKey: "validation-1", evaluationIdempotencyKey: evaluation.evaluationIdempotencyKey, @@ -399,6 +411,37 @@ describe("skill writer", () => { expect(current.state.skills.map((skill) => skill.name).sort()).toEqual([...testCase.expectedNames].sort()); }); + it("stages and activates a deterministically validated mutation when evaluation is skipped", async () => { + const data = await writerFixture({ + operation: "CREATE", + proposedName: "created-skill", + evaluationSkipped: true, + }); + + const batch = await data.writer.buildMutationBatch({ scope, groupId: data.group.groupId, createdAt: now }); + expect(batch.mutations[0]?.evaluation).toBeNull(); + const writeReceipt = await data.writer.write(batch, { idempotencyKey: "write-1", lease: data.lease }); + await expect( + data.writer.activate(batch, writeReceipt, { idempotencyKey: "activate-1", lease: data.lease }), + ).resolves.toMatchObject({ result: { status: "activated" } }); + expect((await data.harnessState.loadCurrent()).state.skills).toEqual([ + expect.objectContaining({ name: "created-skill" }), + ]); + }); + + it("rejects unevaluated candidates without an explicit skip transition", async () => { + const data = await writerFixture({ + operation: "CREATE", + proposedName: "created-skill", + evaluationSkipped: true, + }); + data.groups[0] = { ...data.group, dispositionHistory: [] }; + + await expect( + data.writer.buildMutationBatch({ scope, groupId: data.group.groupId, createdAt: now }), + ).rejects.toThrow("is not accepted for skill activation"); + }); + it("persists supporting files in an activated skill package", async () => { const files = [ { path: "assets/config.json", content: '{"mode":"focused"}\n' }, From 8927e12e8c7cebd50fc88fe05cfd441a284ec97c Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sat, 29 Aug 2026 22:48:46 -0400 Subject: [PATCH 02/10] feat(coding-agent): complete skills without evaluator --- packages/coding-agent/docs/evolution.md | 10 +- packages/coding-agent/docs/security.md | 4 +- packages/coding-agent/docs/skills.md | 2 +- .../interactive/components/evolution-menu.ts | 2 +- .../coding-agent/test/evolution-menu.test.ts | 33 ++++++ .../test/evolution-runtime.test.ts | 102 +----------------- 6 files changed, 45 insertions(+), 108 deletions(-) diff --git a/packages/coding-agent/docs/evolution.md b/packages/coding-agent/docs/evolution.md index 09f493e29..d02a8c3ed 100644 --- a/packages/coding-agent/docs/evolution.md +++ b/packages/coding-agent/docs/evolution.md @@ -9,12 +9,12 @@ Use `/evolve` to: - inspect evidence eligibility and running or completed jobs; - start, resume, cancel, or browse evolution work; - inspect memory proposals and their evidence; -- inspect skill-candidate provenance, textual draft diffs, deterministic validation, and retrieval/behavior evaluation; -- activate a skill candidate only after its persisted evaluation is accepted; +- inspect skill-candidate provenance, textual draft diffs, deterministic validation, and retrieval/behavior evaluation when available; +- activate a skill candidate after deterministic validation succeeds and any available persisted evaluation is accepted; - inspect target revisions and roll back an activated evolved skill to its pre-activation revision, even when later descendants preserved it; - enable or disable future tracing and evolution work. -Skill activation always requires explicit confirmation. Automatic skill activation is not enabled. Candidates that have not passed every required gate remain read-only. If harness memory changes after evaluation while the evaluated skill inventory remains identical, activation automatically rebases the candidate onto the selected revision and preserves its memory. Skill-inventory changes remain conflicts because they require fresh retrieval and behavior evaluation. +Skill activation always requires explicit confirmation. Automatic skill activation is not enabled. Retrieval and behavior evaluation is temporarily skipped when no evaluator is configured; the candidate history and UI record that no evaluation ran. In that case, deterministic structure, provenance, scope, security, and inventory validation still must pass before activation becomes available. If harness memory changes after validation while the validated skill inventory remains identical, activation automatically rebases the candidate onto the selected revision and preserves its memory. Skill-inventory changes remain conflicts because they require fresh validation. Use `/version` to inspect compatible revisions for the current workspace and select one. Selection validates the complete bundle. A revision with corrupt, unsupported, or workspace-mismatched artifacts is not activated. @@ -37,9 +37,9 @@ Evolution model calls use the configured provider and may consume provider quota ## Validation and security -Source traces, tool output, files, and generated drafts are treated as untrusted data. Deterministic validation checks structure, provenance, workspace scope, immutable versions, the closed package inventory, portable paths, local-reference resolution, aggregate size limits, and bounded sensitive-data rules. Retrieval and behavioral evaluation are separate gates. +Source traces, tool output, files, and generated drafts are treated as untrusted data. Deterministic validation checks structure, provenance, workspace scope, immutable versions, the closed package inventory, portable paths, local-reference resolution, aggregate size limits, and bounded sensitive-data rules. Retrieval and behavioral evaluation remain separate capabilities but are temporarily skipped when the runtime has no evaluator. -These checks do not prove semantic safety and are not a sandbox. Review candidate provenance, draft diffs, validation issues, and evaluation results before activation. See [Security](security.md). +These checks do not prove semantic safety and are not a sandbox. When evaluation is skipped, Diffex has not verified retrieval quality, behavior improvement, misapplication, or regressions. Review candidate provenance, draft diffs, validation issues, and any available evaluation results before activation. See [Security](security.md). ## Storage and recovery diff --git a/packages/coding-agent/docs/security.md b/packages/coding-agent/docs/security.md index e51ffef8f..20f4029fd 100644 --- a/packages/coding-agent/docs/security.md +++ b/packages/coding-agent/docs/security.md @@ -56,9 +56,9 @@ If you bind-mount a host workspace read/write, writes from inside the container Invocation traces, evolution artifacts and failures, candidate history, lifecycle observations, writer receipts, and harness revisions persist beneath the effective agent directory. They can contain session evidence, generated instructions, skill content, paths, and runtime metadata. Evolution model calls may consume provider quota. -Evolved skills are generated instructions and can direct the model to use tools or referenced assets. Structural, provenance, scope, security, retrieval, and behavioral gates reduce risk but do not prove semantic safety and are not a sandbox. Project trust does not sandbox an already selected harness revision. +Evolved skills are generated instructions and can direct the model to use tools or referenced assets. Structural, provenance, scope, and security validation reduce risk but do not prove semantic safety and are not a sandbox. Retrieval and behavioral evaluation is temporarily skipped when no evaluator is configured, so those candidates have not been checked for retrieval quality, behavior improvement, misapplication, or regressions. Project trust does not sandbox an already selected harness revision. -Review `/evolve` provenance, draft diffs, validation, and evaluation before activation. Use `/version` and rollback controls to select immutable revisions, and use OS or container isolation for unattended work. See [Harness Evolution](evolution.md). +Review `/evolve` provenance, draft diffs, validation, whether evaluation ran or was skipped, and the complete generated package before activation. Use `/version` and rollback controls to select immutable revisions, and use OS or container isolation for unattended work. See [Harness Evolution](evolution.md). ## Reporting Security Issues diff --git a/packages/coding-agent/docs/skills.md b/packages/coding-agent/docs/skills.md index 10543e671..d9e718f4d 100644 --- a/packages/coding-agent/docs/skills.md +++ b/packages/coding-agent/docs/skills.md @@ -76,7 +76,7 @@ This is progressive disclosure: only names, descriptions, and locations are alwa Evolved skills are immutable, workspace-scoped packages in a harness revision. The package version covers `SKILL.md` and every bundled UTF-8 supporting file, while the content digest identifies `SKILL.md` itself. Diffex exposes a package only when the active revision and workspace identity match. Selecting another revision changes `EVOLVE.md` memory and the evolved-skill set together at the next prompt boundary; in-flight and already admitted queued requests remain pinned. -An evolved reference binds its skill ID, content version, digest, workspace, and harness revision. A reference from an inactive revision is rejected. `/reload` refreshes installed skills while preserving the evolved overlay from the active harness. Use `/evolve` to inspect provenance, validation, evaluation, activation, and rollback. See [Harness Evolution](evolution.md). +An evolved reference binds its skill ID, content version, digest, workspace, and harness revision. A reference from an inactive revision is rejected. `/reload` refreshes installed skills while preserving the evolved overlay from the active harness. Use `/evolve` to inspect provenance, validation, whether evaluation ran or was skipped, activation, and rollback. See [Harness Evolution](evolution.md). ## Skill Commands diff --git a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts index 7b3b61420..141aaac34 100644 --- a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts +++ b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts @@ -614,7 +614,7 @@ export class EvolutionMenuComponent extends Container implements Focusable { { label: "Evaluation", enabled: detail.evaluation !== null, - disabledStatus: "pending", + disabledStatus: detail.summary.state === "accepted" ? "skipped" : "pending", select: () => review("evaluation"), }, { diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index 163f4fb7c..b877000a0 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -743,6 +743,39 @@ describe("EvolutionMenuComponent", () => { component.dispose(); }); + it("labels skipped skill evaluation without blocking explicit activation", async () => { + const detail = skillCandidateDetail({ evaluation: null }); + const fixture = client(overview()); + fixture.listJobs.mockResolvedValue([job()]); + fixture.jobDetailCall.mockResolvedValue(jobDetail({ skillCandidates: [detail.summary] })); + fixture.skillCandidateDetailCall.mockResolvedValue(detail); + const component = new EvolutionMenuComponent( + fixture.client, + () => {}, + () => {}, + () => {}, + ); + await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Browse")); + + for (let index = 0; index < 4; index += 1) component.handleInput("\u001b[B"); + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain(formatEvolutionJobLabel(job())), + ); + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain("Skill focused-verification - accepted"), + ); + component.handleInput("\n"); + await vi.waitFor(() => expect(fixture.skillCandidateDetailCall).toHaveBeenCalledOnce()); + + const output = stripAnsi(component.render(180).join("\n")); + expect(output).toContain("Evaluation [skipped]"); + expect(output).toContain("Activate"); + expect(output).not.toContain("Activate [unavailable]"); + component.dispose(); + }); + it("formats create, update, and retirement draft diffs", () => { const createDiff = formatEvolutionSkillDraftDiff(skillCandidateDetail()); expect(createDiff).toContain("+Run checks."); diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index df45bc16a..5b7f751c4 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -2,23 +2,13 @@ import { existsSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { fauxAssistantMessage, fauxToolCall, type Message } from "@earendil-works/diffex-ai"; -import { - computeHarnessSkillId, - computeHarnessSkillPackageVersion, - HarnessStateStore, -} from "@earendil-works/diffex-harness-state"; +import { computeHarnessSkillId, HarnessStateStore } from "@earendil-works/diffex-harness-state"; import { afterEach, describe, expect, it, vi } from "vitest"; import { - computeEvolutionContentDigest, - computeEvolutionTextDigest, - createEvolutionArtifactMetadata, - type EvolutionArtifactProducer, type EvolutionControlService, type EvolutionModelRequest, type EvolutionScope, EvolutionToolDeniedError, - type SkillEvaluationReport, - type SkillEvaluatorInput, } from "#evolution"; import { encodeTraceRecord, type MaterializedInvocation, type TraceRecord } from "#harness-trace"; import { @@ -38,83 +28,6 @@ import { createSkill } from "../src/core/skills.ts"; import { createSyntheticSourceInfo } from "../src/core/source-info.ts"; import { createHarness, type Harness } from "./suite/harness.ts"; -const skillEvaluatorProducer: EvolutionArtifactProducer = { - name: "coding-agent-evolution-runtime-test", - version: "1", - promptVersion: null, - model: null, -}; - -function acceptedSkillEvaluation(input: SkillEvaluatorInput): SkillEvaluationReport { - const draft = { - artifactId: input.draft.draftId, - contentDigest: computeEvolutionContentDigest(input.draft), - }; - const validation = { - artifactId: input.validation.validationId, - contentDigest: computeEvolutionContentDigest(input.validation), - }; - const content = { - artifactKind: "skill-evaluation" as const, - schemaVersion: 1 as const, - evaluationId: `evaluation-${input.group.groupId}`, - groupId: input.group.groupId, - evaluationIdempotencyKey: input.idempotencyKey, - proposalRevisionIds: input.group.proposalRevisions.map((revision) => revision.revisionId), - draft, - validation, - evaluatorVersion: "runtime-test-evaluator-v1", - assurance: "retrieval-and-deterministic-behavior-gates" as const, - disposition: "accepted" as const, - harnessVersion: input.validation.validatedHarnessVersion, - candidateSkill: { - skillId: input.draft.draftId, - name: input.draft.proposedName, - description: "Run focused verification after repository changes.", - source: "candidate" as const, - version: computeHarnessSkillPackageVersion({ content: input.draft.content, files: input.draft.files }), - contentDigest: computeEvolutionTextDigest(input.draft.content), - content: input.draft.content, - files: input.draft.files, - }, - neighboringSkills: [], - configuration: { - model: { provider: "faux", model: "fixture", configurationDigest: `sha256:${"1".repeat(64)}` }, - runtime: { name: "node", version: "24", configurationDigest: `sha256:${"2".repeat(64)}` }, - retriever: { name: "fixture", version: "1", configurationDigest: `sha256:${"3".repeat(64)}` }, - executor: { name: "fixture", version: "1", configurationDigest: `sha256:${"4".repeat(64)}` }, - }, - cases: [], - retrievalResults: [], - behaviorResults: [], - metrics: { - exactRetrievalAccuracy: 1, - sourceReplayReproducibility: 1, - heldOutIntendedBaselineScore: 0, - heldOutIntendedFullPoolScore: 1, - heldOutIntendedImprovement: 1, - negativeRegressionCount: 0, - misapplicationCount: 0, - }, - gates: { - suite: { passed: true, reasonCodes: [] }, - retrieval: { passed: true, reasonCodes: [] }, - behavior: { passed: true, reasonCodes: [] }, - }, - }; - return { - metadata: createEvolutionArtifactMetadata({ - artifactId: content.evaluationId, - producer: skillEvaluatorProducer, - sourceIds: [input.draft.draftId, input.validation.validationId], - scope: input.draft.metadata.scope, - createdAt: input.createdAt, - content, - }), - ...content, - }; -} - function materializedInvocation(traceId: string, sessionPath: string): MaterializedInvocation { return { version: 1, @@ -1164,7 +1077,7 @@ describe("coding-agent evolution runtime", () => { }); }); - it("evaluates and activates evidence-derived skills once across concurrent runtimes before prompt refresh", async () => { + it("validates and activates evidence-derived skills without an evaluator across concurrent runtimes", async () => { const agentDir = join( tmpdir(), `diffex-skill-evolution-flow-${Date.now()}-${Math.random().toString(36).slice(2)}`, @@ -1320,13 +1233,6 @@ describe("coding-agent evolution runtime", () => { { stopReason: "toolUse" }, ), ]); - const evaluationRequests: SkillEvaluatorInput[] = []; - const skillEvaluator = { - async evaluate(input: SkillEvaluatorInput): Promise { - evaluationRequests.push(input); - return acceptedSkillEvaluation(input); - }, - }; const runtimeOptions = { agentDir, cwd: harness.tempDir, @@ -1334,7 +1240,6 @@ describe("coding-agent evolution runtime", () => { harnessState: store, modelRuntime: harness.session.modelRuntime, resourceLoader: harness.session.resourceLoader, - skillEvaluator, }; const runtime = await createCodingAgentEvolutionRuntime(runtimeOptions); const peerRuntime = await createCodingAgentEvolutionRuntime(runtimeOptions); @@ -1364,12 +1269,11 @@ describe("coding-agent evolution runtime", () => { }, { timeout: 8_000, interval: 20 }, ); - expect(evaluationRequests).toHaveLength(1); const candidate = await runtime.control.getSkillCandidate(started.job.jobId, candidateId); expect(candidate).toMatchObject({ summary: { state: "accepted" }, validation: { disposition: "accepted", validatedHarnessVersion: root.manifest.version }, - evaluation: { disposition: "accepted", harnessVersion: root.manifest.version }, + evaluation: null, activationAvailable: true, }); From 3f23385e9e9527561ca7b024e812f439c3b3296c Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sat, 29 Aug 2026 23:27:35 -0400 Subject: [PATCH 03/10] fix(agent): isolate skill candidate synthesis failures --- packages/coding-agent/CHANGELOG.md | 1 + .../evolution/src/skill-candidate-worker.ts | 29 ++++++-- .../test/skill-candidate-worker.test.ts | 67 +++++++++++++++++++ 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 5e1e42c1f..889fca8cd 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -21,6 +21,7 @@ - Fixed the harness footer showing an evolution spinner while self-evolution is disabled. - Fixed evolution setting changes not reaching already-running Diffex processes, which could let stale processes continue tracing and launch automatic work. - Fixed an open, idle `/evolve` menu retaining enabled actions after another Diffex process disables evolution. +- Fixed one failed skill candidate synthesis preventing later qualified candidates from being drafted, validated, and evaluated. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). diff --git a/packages/evolution/src/skill-candidate-worker.ts b/packages/evolution/src/skill-candidate-worker.ts index 3977de8bc..0e912df38 100644 --- a/packages/evolution/src/skill-candidate-worker.ts +++ b/packages/evolution/src/skill-candidate-worker.ts @@ -10,6 +10,7 @@ import type { SkillCandidateWorkCheckpointedEvent, SkillCandidateWorkFailedEvent, SkillCandidateWorkStage, + SkillDraft, SkillDraftValidationReport, } from "./contracts.ts"; import { SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION } from "./contracts.ts"; @@ -551,17 +552,25 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor claim(), ); const draftedGroups = result.groups.filter((group) => group.state === "drafted"); + const synthesisFailures: { readonly groupId: string; readonly error: unknown }[] = []; for (const group of synthesisTargets) { signal.throwIfAborted(); let draftReference = group.draft; if (draftReference === null) { - const draft = await this.#dependencies.synthesizer.synthesize({ - group, - model: this.#dependencies.model, - idempotencyKey: group.synthesisIdempotencyKey, - createdAt: this.#dependencies.clock.now(), - signal, - }); + let draft: SkillDraft; + try { + draft = await this.#dependencies.synthesizer.synthesize({ + group, + model: this.#dependencies.model, + idempotencyKey: group.synthesisIdempotencyKey, + createdAt: this.#dependencies.clock.now(), + signal, + }); + } catch (error) { + signal.throwIfAborted(); + synthesisFailures.push({ groupId: group.groupId, error }); + continue; + } draftReference = await this.#dependencies.artifacts.put(draft); } draftedGroups.push( @@ -691,6 +700,12 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor evaluatedGroupIds.push(pending.groupId); } } + if (synthesisFailures.length > 0) { + throw new AggregateError( + synthesisFailures.map((failure) => failure.error), + `Skill synthesis failed for candidate groups: ${synthesisFailures.map((failure) => failure.groupId).join(", ")}`, + ); + } await this.#appendCheckpoint( job, "completed", diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index 109cc6195..6a06d422d 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -425,6 +425,7 @@ function fixture(withEvaluator = false, store: MemoryRegistryStore = new MemoryR const leases = new MemoryLeaseStore(); const registry = new FixtureRegistry(); const synthesisRequests: SkillSynthesizerInput[] = []; + const synthesisFailures = new Set(); const validationRequests: SkillDraftValidatorInput[] = []; const evaluationRequests: SkillEvaluatorInput[] = []; let clockTime = now; @@ -435,6 +436,7 @@ function fixture(withEvaluator = false, store: MemoryRegistryStore = new MemoryR store, registry, synthesisRequests, + synthesisFailures, validationRequests, evaluationRequests, clock, @@ -453,6 +455,7 @@ function fixture(withEvaluator = false, store: MemoryRegistryStore = new MemoryR synthesizer: { async synthesize(request): Promise { synthesisRequests.push(request); + if (synthesisFailures.has(request.group.groupId)) throw new Error("synthesis failed"); const content = { artifactKind: "skill-draft" as const, draftId: "draft-1", @@ -705,6 +708,70 @@ describe("skill candidate registry worker", () => { ]); }); + it("continues processing later candidates when one synthesis fails", async () => { + const data = fixture(true); + const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; + if (candidateReference === undefined) throw new Error("Expected candidate reference"); + const group = qualifiedGroup(candidateReference); + data.registry.groups.push( + { + ...group, + groupId: "group-failing", + synthesisIdempotencyKey: "skill-synthesis-failing", + validationIdempotencyKey: "skill-validation-failing", + evaluationIdempotencyKey: "skill-evaluation-failing", + }, + { + ...group, + groupId: "group-succeeding", + synthesisIdempotencyKey: "skill-synthesis-succeeding", + validationIdempotencyKey: "skill-validation-succeeding", + evaluationIdempotencyKey: "skill-evaluation-succeeding", + }, + ); + data.synthesisFailures.add("group-failing"); + + await expect(data.worker.process(data.job.jobId)).rejects.toThrow( + "Skill synthesis failed for candidate groups: group-failing", + ); + + expect(data.synthesisRequests.map((request) => request.group.groupId)).toEqual([ + "group-failing", + "group-succeeding", + ]); + expect(data.validationRequests.map((request) => request.group.groupId)).toEqual(["group-succeeding"]); + expect(data.evaluationRequests.map((request) => request.group.groupId)).toEqual(["group-succeeding"]); + expect(data.registry.groups.find((candidateGroup) => candidateGroup.groupId === "group-failing")?.state).toBe( + "evidence-qualified", + ); + expect(data.registry.groups.find((candidateGroup) => candidateGroup.groupId === "group-succeeding")?.state).toBe( + "accepted", + ); + expect(data.store.events.find((event) => event.type === "skill-candidate-work-failed")).toMatchObject({ + message: "Skill synthesis failed for candidate groups: group-failing", + }); + expect( + data.store.events.some( + (event) => event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", + ), + ).toBe(false); + + data.synthesisFailures.clear(); + await data.worker.process(data.job.jobId); + + expect(data.synthesisRequests.map((request) => request.group.groupId)).toEqual([ + "group-failing", + "group-succeeding", + "group-failing", + ]); + expect(data.registry.groups.map((candidateGroup) => candidateGroup.state)).toEqual(["accepted", "accepted"]); + expect( + data.store.events.some( + (event) => event.type === "skill-candidate-work-checkpointed" && event.stage === "completed", + ), + ).toBe(true); + }); + it("accepts structurally valid drafts when retrieval and behavior evaluation is unavailable", async () => { const data = fixture(); const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; From 6ba3599cda8e4bce59f43ef014a1877a8015f63a Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sat, 29 Aug 2026 23:44:35 -0400 Subject: [PATCH 04/10] fix(agent): stop retrying exhausted skill synthesis --- packages/coding-agent/CHANGELOG.md | 1 + packages/evolution/RECOVERY.md | 2 +- packages/evolution/src/contracts.ts | 1 + packages/evolution/src/filesystem-store.ts | 1 + packages/evolution/src/index.ts | 1 + packages/evolution/src/ports.ts | 14 +++ .../evolution/src/skill-candidate-registry.ts | 106 +++++++++++++++++- .../evolution/src/skill-candidate-worker.ts | 22 +++- .../evolution/test/filesystem-store.test.ts | 38 ++++++- .../test/skill-candidate-registry.test.ts | 81 ++++++++++++- .../test/skill-candidate-worker.test.ts | 81 +++++++++++++ packages/evolution/test/skill-writer.test.ts | 1 + 12 files changed, 339 insertions(+), 10 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 889fca8cd..d50d2319e 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -22,6 +22,7 @@ - Fixed evolution setting changes not reaching already-running Diffex processes, which could let stale processes continue tracing and launch automatic work. - Fixed an open, idle `/evolve` menu retaining enabled actions after another Diffex process disables evolution. - Fixed one failed skill candidate synthesis preventing later qualified candidates from being drafted, validated, and evaluated. +- Fixed exhausted skill synthesis being retried by unrelated later evolution jobs instead of remaining terminal until new evidence revises the candidate. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). diff --git a/packages/evolution/RECOVERY.md b/packages/evolution/RECOVERY.md index f0f97e289..d4b1c5ec3 100644 --- a/packages/evolution/RECOVERY.md +++ b/packages/evolution/RECOVERY.md @@ -26,7 +26,7 @@ Recovery never treats partial output as completed work and never feeds evolution | Successful rollback selection | The recorded immutable parent is still present and the evolved revision is current. | Select the parent with compare-and-swap and refresh on the next prompt boundary. | Existing in-flight prompts stay pinned to their admitted revision. | | Unsupported skill or orchestration target | The target registry has no enabled writer route. | Preserve the finding as deferred evidence with no executable payload. | Disabled target kinds cannot reach filesystem mutation or activation. | | Interrupted skill-candidate consolidation | A completed evolution job has checkpointed candidates but no completed candidate-work event. | Resume from append-only registry events with the same source-job idempotency key; repair missing scores or transitions before completion. | Candidate proposal revisions remain immutable, retries do not add recurrence, and partial registry appends cannot activate a skill. | -| Interrupted skill synthesis, validation, evaluation, write, or activation | A qualified group is missing the next immutable artifact or transition, or an accepted candidate has a durable write without a matching activation. | Resume with revision-bound idempotency keys; reuse recorded artifacts and receipts, repair the missing transition, and revalidate the exact accepted group before writing or activation. | Only evidence-qualified groups invoke synthesis, only deterministically valid drafts enter evaluation, only accepted evaluations reach the writer, and retries cannot stage or activate a skill twice. | +| Interrupted or failed skill synthesis, validation, evaluation, write, or activation | A qualified group is missing the next immutable artifact or transition, an accepted candidate has a durable write without a matching activation, or synthesis exhausted its bounded attempts. | Resume recoverable work with revision-bound idempotency keys; reuse recorded artifacts and receipts, repair the missing transition, and record exhausted synthesis as `synthesis-failed` until a new proposal revision changes its synthesis identity. | Only evidence-qualified groups invoke synthesis, unrelated later jobs cannot retry an exhausted group, only deterministically valid drafts enter evaluation, only accepted evaluations reach the writer, and retries cannot stage or activate a skill twice. | ## Operational observability diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 4dc934edd..1e191206e 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -648,6 +648,7 @@ export interface DeferredSkillCandidateFinding { export type SkillCandidateState = | "proposed" | "evidence-qualified" + | "synthesis-failed" | "drafted" | "structurally-valid" | "evaluation-pending" diff --git a/packages/evolution/src/filesystem-store.ts b/packages/evolution/src/filesystem-store.ts index 86fc405ad..0079022ac 100644 --- a/packages/evolution/src/filesystem-store.ts +++ b/packages/evolution/src/filesystem-store.ts @@ -68,6 +68,7 @@ const SKILL_CANDIDATE_REGISTRY_EVENT_TYPES = new Set([ const SKILL_CANDIDATE_STATES = new Set([ "proposed", "evidence-qualified", + "synthesis-failed", "drafted", "structurally-valid", "evaluation-pending", diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 055006789..1959d836a 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -302,6 +302,7 @@ export type { SkillCandidateRegistryConsolidationInput, SkillCandidateRegistryConsolidationResult, SkillCandidateRegistryStore, + SkillCandidateSynthesisFailureRecordingInput, SkillCandidateValidationRecordingInput, SkillCurationResult, SkillCurator, diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index efd4b807e..376b2a0e2 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -334,6 +334,17 @@ export interface SkillCandidateRegistryConsolidationResult { readonly changedGroupIds: readonly string[]; } +export interface SkillCandidateSynthesisFailureRecordingInput { + readonly scope: EvolutionScope; + readonly lease?: EvolutionLeaseToken; + readonly failures: readonly { + readonly groupId: string; + readonly proposalRevisionIds: readonly string[]; + readonly idempotencyKey: string; + }[]; + readonly createdAt: string; +} + export interface SkillCandidateDraftRecordingInput { readonly scope: EvolutionScope; readonly lease?: EvolutionLeaseToken; @@ -380,6 +391,9 @@ export interface SkillCandidateEvaluationRecordingInput { export interface SkillCandidateRegistry { consolidate(input: SkillCandidateRegistryConsolidationInput): Promise; + recordSynthesisFailures( + input: SkillCandidateSynthesisFailureRecordingInput, + ): Promise; recordDraft(input: SkillCandidateDraftRecordingInput): Promise; recordValidation(input: SkillCandidateValidationRecordingInput): Promise; skipEvaluation(input: SkillCandidateEvaluationStartInput): Promise; diff --git a/packages/evolution/src/skill-candidate-registry.ts b/packages/evolution/src/skill-candidate-registry.ts index 2baa1c6d1..53fdd9e2d 100644 --- a/packages/evolution/src/skill-candidate-registry.ts +++ b/packages/evolution/src/skill-candidate-registry.ts @@ -30,6 +30,7 @@ import type { SkillCandidateRegistryConsolidationInput, SkillCandidateRegistryConsolidationResult, SkillCandidateRegistryStore, + SkillCandidateSynthesisFailureRecordingInput, SkillCandidateValidationRecordingInput, } from "./ports.ts"; import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; @@ -63,6 +64,7 @@ interface MutableGroup { readonly proposalRevisions: SkillCandidateProposalRevision[]; state: SkillCandidateState; readonly dispositionHistory: SkillCandidateDispositionRecord[]; + readonly synthesisFailedProposalRevisionIds: string[]; priorityScore: EvolutionArtifactReference | null; draft: EvolutionArtifactReference | null; readonly draftProposalRevisionIds: string[]; @@ -273,6 +275,7 @@ function materializeGroups(events: readonly SkillCandidateRegistryEvent[]): Map< proposalRevisions: [event.proposalRevision], state: "proposed", dispositionHistory: [], + synthesisFailedProposalRevisionIds: [], priorityScore: null, draft: null, draftProposalRevisionIds: [], @@ -393,6 +396,13 @@ function materializeGroups(events: readonly SkillCandidateRegistryEvent[]): Map< throw new Error(`Candidate group "${event.groupId}" disposition history is not contiguous`); } group.state = event.to; + if (event.to === "synthesis-failed") { + group.synthesisFailedProposalRevisionIds.splice( + 0, + group.synthesisFailedProposalRevisionIds.length, + ...group.proposalRevisions.map((revision) => revision.revisionId), + ); + } group.dispositionHistory.push({ eventId: event.eventId, from: event.from, @@ -799,6 +809,97 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry .sort((left, right) => left.groupId.localeCompare(right.groupId)); } + #qualification(group: MutableGroup): { + readonly state: SkillCandidateState; + readonly reasonCodes: readonly string[]; + } { + const immutable = immutableGroup(group); + if (group.state !== "synthesis-failed") return this.#scorer.qualification(immutable); + if ( + sameValues( + group.synthesisFailedProposalRevisionIds, + group.proposalRevisions.map((revision) => revision.revisionId), + ) + ) { + return { state: group.state, reasonCodes: [] }; + } + const qualification = this.#scorer.qualification({ ...immutable, state: "proposed" }); + return qualification.state === "evidence-qualified" + ? { state: qualification.state, reasonCodes: ["proposal-revision-set-changed"] } + : qualification; + } + + async recordSynthesisFailures( + input: SkillCandidateSynthesisFailureRecordingInput, + ): Promise { + const failures = [...input.failures].sort((left, right) => left.groupId.localeCompare(right.groupId)); + if (new Set(failures.map((failure) => failure.groupId)).size !== failures.length) { + throw new Error("Skill synthesis failures must identify unique candidate groups"); + } + for (let attempt = 0; attempt < this.#maxCommitAttempts; attempt += 1) { + const snapshot = await this.#store.loadSkillCandidateRegistry(input.scope); + const groups = materializeGroups(snapshot.events); + const planned: SkillCandidateRegistryEvent[] = []; + const recorded: SkillCandidateGroup[] = []; + for (const failure of failures) { + const mutable = groups.get(failure.groupId); + if (mutable === undefined) throw new Error(`Unknown candidate group "${failure.groupId}"`); + const group = immutableGroup(mutable); + if ( + !sameValues( + failure.proposalRevisionIds, + group.proposalRevisions.map((revision) => revision.revisionId), + ) || + failure.idempotencyKey !== group.synthesisIdempotencyKey + ) { + throw new Error(`Candidate group "${failure.groupId}" changed before synthesis failure was recorded`); + } + if (group.state === "synthesis-failed") { + recorded.push(group); + continue; + } + if (group.state !== "evidence-qualified") { + throw new Error(`Candidate group "${failure.groupId}" is not eligible to fail synthesis`); + } + const transition = transitionEvent( + failure.groupId, + "evidence-qualified", + "synthesis-failed", + ["skill-synthesis-attempts-exhausted"], + nextSequence(snapshot.events, planned.length), + failure.idempotencyKey, + this.#producer, + input.scope, + input.createdAt, + ); + planned.push(transition); + mutable.state = "synthesis-failed"; + mutable.synthesisFailedProposalRevisionIds.splice( + 0, + mutable.synthesisFailedProposalRevisionIds.length, + ...failure.proposalRevisionIds, + ); + mutable.dispositionHistory.push({ + eventId: transition.eventId, + from: transition.from, + to: transition.to, + reasonCodes: transition.reasonCodes, + createdAt: input.createdAt, + }); + recorded.push(immutableGroup(mutable)); + } + if (planned.length === 0) return recorded; + const committed = await this.#store.appendSkillCandidateRegistryEvents( + input.scope, + snapshot.revision, + planned, + input.lease, + ); + if (committed === "committed") return recorded; + } + throw new Error("Skill candidate registry changed too often to record synthesis failures"); + } + async recordDraft(input: SkillCandidateDraftRecordingInput): Promise { for (let attempt = 0; attempt < this.#maxCommitAttempts; attempt += 1) { const snapshot = await this.#store.loadSkillCandidateRegistry(input.scope); @@ -1315,6 +1416,7 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry proposalRevisions: [], state: "proposed", dispositionHistory: [], + synthesisFailedProposalRevisionIds: [], priorityScore: null, draft: null, draftProposalRevisionIds: [], @@ -1367,7 +1469,7 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry for (const group of groups.values()) { const revisionIds = group.proposalRevisions.map((revision) => revision.revisionId); - const qualification = this.#scorer.qualification(immutableGroup(group)); + const qualification = this.#qualification(group); if (!sameValues(group.scoredProposalRevisionIds, revisionIds) || qualification.state !== group.state) { changedGroupIds.add(group.groupId); } @@ -1429,7 +1531,7 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry group.scoredProposalRevisionIds.length, ...scoreContent.proposalRevisionIds, ); - const qualification = this.#scorer.qualification(immutableGroup(group)); + const qualification = this.#qualification(group); if (qualification.state !== group.state) { const transition = transitionEvent( groupId, diff --git a/packages/evolution/src/skill-candidate-worker.ts b/packages/evolution/src/skill-candidate-worker.ts index 0e912df38..9b1284c3f 100644 --- a/packages/evolution/src/skill-candidate-worker.ts +++ b/packages/evolution/src/skill-candidate-worker.ts @@ -552,7 +552,12 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor claim(), ); const draftedGroups = result.groups.filter((group) => group.state === "drafted"); - const synthesisFailures: { readonly groupId: string; readonly error: unknown }[] = []; + const synthesisFailures: { + readonly groupId: string; + readonly proposalRevisionIds: readonly string[]; + readonly idempotencyKey: string; + readonly error: unknown; + }[] = []; for (const group of synthesisTargets) { signal.throwIfAborted(); let draftReference = group.draft; @@ -568,7 +573,12 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor }); } catch (error) { signal.throwIfAborted(); - synthesisFailures.push({ groupId: group.groupId, error }); + synthesisFailures.push({ + groupId: group.groupId, + proposalRevisionIds: group.proposalRevisions.map((revision) => revision.revisionId), + idempotencyKey: group.synthesisIdempotencyKey, + error, + }); continue; } draftReference = await this.#dependencies.artifacts.put(draft); @@ -585,6 +595,14 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor }), ); } + if (synthesisFailures.length > 0 && attempt >= this.#maxAttempts) { + await this.#dependencies.registry.recordSynthesisFailures({ + scope: job.metadata.scope, + lease: claim(), + failures: synthesisFailures.map(({ error: _error, ...failure }) => failure), + createdAt: this.#dependencies.clock.now(), + }); + } await this.#appendCheckpoint( job, "validating", diff --git a/packages/evolution/test/filesystem-store.test.ts b/packages/evolution/test/filesystem-store.test.ts index 1dd488b43..26b396963 100644 --- a/packages/evolution/test/filesystem-store.test.ts +++ b/packages/evolution/test/filesystem-store.test.ts @@ -139,6 +139,30 @@ function candidateWorkEvent(eventId: string, registrySequence: number): SkillCan }; } +function candidateStateEvent(eventId: string, registrySequence: number): SkillCandidateRegistryEvent { + const content = { + schemaVersion: SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, + eventId, + registrySequence, + type: "skill-candidate-state-transitioned" as const, + groupId: "group-1", + from: "evidence-qualified" as const, + to: "synthesis-failed" as const, + reasonCodes: ["skill-synthesis-attempts-exhausted"], + }; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: eventId, + producer, + sourceIds: ["group-1"], + scope, + createdAt: "2026-08-19T12:00:00.000Z", + content, + }), + ...content, + }; +} + function candidateEvaluationEvent(eventId: string, registrySequence: number): SkillCandidateRegistryEvent { const reference = (artifactId: string) => ({ artifactId, contentDigest: `sha256:${"a".repeat(64)}` }); const content = { @@ -270,13 +294,19 @@ describe("filesystem evolution persistence", () => { const first = candidateWorkEvent("candidate-event-1", 1); const evaluation = candidateEvaluationEvent("candidate-event-2", 2); - expect(await store.appendSkillCandidateRegistryEvents(scope, 0, [first, evaluation])).toBe("committed"); - expect(await peer.loadSkillCandidateRegistry(scope)).toEqual({ revision: 2, events: [first, evaluation] }); + const synthesisFailed = candidateStateEvent("candidate-event-3", 3); + expect(await store.appendSkillCandidateRegistryEvents(scope, 0, [first, evaluation, synthesisFailed])).toBe( + "committed", + ); + expect(await peer.loadSkillCandidateRegistry(scope)).toEqual({ + revision: 3, + events: [first, evaluation, synthesisFailed], + }); expect( - await peer.appendSkillCandidateRegistryEvents(scope, 0, [candidateWorkEvent("candidate-event-3", 1)]), + await peer.appendSkillCandidateRegistryEvents(scope, 0, [candidateWorkEvent("candidate-event-4", 1)]), ).toBe("conflict"); await expect( - peer.appendSkillCandidateRegistryEvents(scope, 2, [candidateWorkEvent("candidate-event-3", 4)]), + peer.appendSkillCandidateRegistryEvents(scope, 3, [candidateWorkEvent("candidate-event-4", 5)]), ).rejects.toThrow("contiguous sequence"); }); diff --git a/packages/evolution/test/skill-candidate-registry.test.ts b/packages/evolution/test/skill-candidate-registry.test.ts index ce77007c6..126f2b7be 100644 --- a/packages/evolution/test/skill-candidate-registry.test.ts +++ b/packages/evolution/test/skill-candidate-registry.test.ts @@ -305,7 +305,7 @@ function registryFixture(store: MemoryRegistryStore = new MemoryRegistryStore()) } describe("persistent skill candidate registry", () => { - it("consolidates compatible cross-round proposals independently of their names", async () => { + it("requalifies an exhausted synthesis only after the proposal revision set changes", async () => { const fixture = registryFixture(); const firstInput = input([evidence("support-a", "task-a", 0)], "trace-a"); const firstCandidate = candidate({ @@ -360,6 +360,85 @@ describe("persistent skill candidate registry", () => { ]); expect(second.groups[0]?.synthesisIdempotencyKey).toMatch(/^skill-synthesis-/); expect(second.groups[0]?.evaluationIdempotencyKey).toMatch(/^skill-evaluation-/); + const group = second.groups[0]; + if (group === undefined) throw new Error("Expected a qualified candidate group"); + + const [failed] = await fixture.registry.recordSynthesisFailures({ + scope, + failures: [ + { + groupId: group.groupId, + proposalRevisionIds: group.proposalRevisions.map((revision) => revision.revisionId), + idempotencyKey: group.synthesisIdempotencyKey, + }, + ], + createdAt: "2026-08-26T12:01:00.000Z", + }); + expect(failed).toMatchObject({ + state: "synthesis-failed", + dispositionHistory: [ + expect.objectContaining({ to: "proposed" }), + expect.objectContaining({ to: "evidence-qualified" }), + expect.objectContaining({ + from: "evidence-qualified", + to: "synthesis-failed", + reasonCodes: ["skill-synthesis-attempts-exhausted"], + }), + ], + }); + expect( + await fixture.registry.recordSynthesisFailures({ + scope, + failures: [ + { + groupId: group.groupId, + proposalRevisionIds: group.proposalRevisions.map((revision) => revision.revisionId), + idempotencyKey: group.synthesisIdempotencyKey, + }, + ], + createdAt: "2026-08-26T12:01:00.000Z", + }), + ).toEqual([failed]); + + const unchanged = await fixture.registry.consolidate({ + sourceEvolutionJobId: "job-b", + input: secondInput, + candidates: [secondCandidate], + idempotencyKey: "job-b:registry", + createdAt: "2026-08-26T12:00:00.000Z", + }); + expect(unchanged.changedGroupIds).toEqual([]); + expect(unchanged.groups[0]?.state).toBe("synthesis-failed"); + + const thirdInput = input([evidence("support-c", "task-c", 0)], "trace-c"); + const thirdCandidate = candidate({ + candidateId: "candidate-c", + input: thirdInput, + proposedName: "focused-check-revised", + supportingEvidenceIds: ["support-c"], + taskLineageIds: ["task-c"], + }); + const revised = await fixture.registry.consolidate({ + sourceEvolutionJobId: "job-c", + input: thirdInput, + candidates: [thirdCandidate], + idempotencyKey: "job-c:registry", + createdAt: "2026-08-27T12:00:00.000Z", + }); + expect(revised.groups[0]).toMatchObject({ + state: "evidence-qualified", + dispositionHistory: [ + expect.objectContaining({ to: "proposed" }), + expect.objectContaining({ to: "evidence-qualified" }), + expect.objectContaining({ to: "synthesis-failed" }), + expect.objectContaining({ + from: "synthesis-failed", + to: "evidence-qualified", + reasonCodes: ["proposal-revision-set-changed"], + }), + ], + }); + expect(revised.groups[0]?.synthesisIdempotencyKey).not.toBe(group.synthesisIdempotencyKey); }); it("records an immutable draft and advances only an evidence-qualified group", async () => { diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index 6a06d422d..264a7f4d8 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -25,6 +25,7 @@ import { type SkillCandidateRegistrySnapshot, type SkillCandidateRegistryStore, SkillCandidateRegistryWorker, + type SkillCandidateSynthesisFailureRecordingInput, type SkillCandidateValidationRecordingInput, type SkillDraft, type SkillDraftValidationReport, @@ -335,6 +336,33 @@ class FixtureRegistry implements SkillCandidateRegistry { return { groups: this.groups, changedGroupIds: ["group-1"] }; } + async recordSynthesisFailures( + input: SkillCandidateSynthesisFailureRecordingInput, + ): Promise { + return input.failures.map((failure) => { + const index = this.groups.findIndex((candidateGroup) => candidateGroup.groupId === failure.groupId); + const group = this.groups[index]; + if (group === undefined) throw new Error("Unknown group"); + if (group.state === "synthesis-failed") return group; + const failed = { + ...group, + state: "synthesis-failed" as const, + dispositionHistory: [ + ...group.dispositionHistory, + { + eventId: `synthesis-failed-${failure.groupId}`, + from: group.state, + to: "synthesis-failed" as const, + reasonCodes: ["skill-synthesis-attempts-exhausted"], + createdAt: input.createdAt, + }, + ], + }; + this.groups[index] = failed; + return failed; + }); + } + async recordDraft(input: SkillCandidateDraftRecordingInput): Promise { this.draftRequests.push(input); const index = this.groups.findIndex((candidateGroup) => candidateGroup.groupId === input.groupId); @@ -772,6 +800,59 @@ describe("skill candidate registry worker", () => { ).toBe(true); }); + it("makes exhausted synthesis failures terminal until the proposal revision set changes", async () => { + const data = fixture(true); + const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; + if (candidateReference === undefined) throw new Error("Expected candidate reference"); + data.registry.groups.push(qualifiedGroup(candidateReference)); + data.synthesisFailures.add("group-qualified"); + + for (let attempt = 1; attempt <= 3; attempt += 1) { + await expect(data.worker.process(data.job.jobId)).rejects.toThrow( + "Skill synthesis failed for candidate groups: group-qualified", + ); + expect( + data.store.events.find( + (event) => event.type === "skill-candidate-work-failed" && event.attempt === attempt, + ), + ).toMatchObject({ retryNotBefore: attempt === 3 ? null : expect.any(String) }); + } + + expect(data.synthesisRequests).toHaveLength(3); + expect(data.registry.groups[0]).toMatchObject({ + state: "synthesis-failed", + dispositionHistory: [ + expect.objectContaining({ + from: "evidence-qualified", + to: "synthesis-failed", + reasonCodes: ["skill-synthesis-attempts-exhausted"], + }), + ], + }); + const source = new PollingSkillCandidateWorkSource({ + clock: data.clock, + jobs: data.jobs, + store: data.store, + registry: data.registry, + evaluatorAvailable: true, + }); + expect(await source.nextRunnableJob(now)).toBeNull(); + + const nextJob = { ...data.job, jobId: "job-2" }; + data.jobs.jobs.push(nextJob); + await data.worker.process(nextJob.jobId); + + expect(data.synthesisRequests).toHaveLength(3); + expect( + data.store.events.some( + (event) => + event.type === "skill-candidate-work-checkpointed" && + event.sourceEvolutionJobId === nextJob.jobId && + event.stage === "completed", + ), + ).toBe(true); + }); + it("accepts structurally valid drafts when retrieval and behavior evaluation is unavailable", async () => { const data = fixture(); const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; diff --git a/packages/evolution/test/skill-writer.test.ts b/packages/evolution/test/skill-writer.test.ts index cc74e5c72..35fa0c752 100644 --- a/packages/evolution/test/skill-writer.test.ts +++ b/packages/evolution/test/skill-writer.test.ts @@ -80,6 +80,7 @@ function registry(groups: readonly SkillCandidateGroup[]): SkillCandidateRegistr return { list: async () => groups, consolidate: unsupported, + recordSynthesisFailures: unsupported, recordDraft: unsupported, recordValidation: unsupported, skipEvaluation: unsupported, From 68c8b388a8304817b947d36bcc53b606bdcad93a Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 30 Aug 2026 00:02:37 -0400 Subject: [PATCH 05/10] fix(agent): persist skill synthesis failure diagnostics --- packages/coding-agent/CHANGELOG.md | 1 + .../src/core/evolution-runtime.ts | 12 ++- .../test/evolution-runtime.test.ts | 40 ++++++++ packages/evolution/src/contracts.ts | 23 +++++ packages/evolution/src/filesystem-store.ts | 59 ++++++++++-- packages/evolution/src/index.ts | 3 + .../evolution/src/skill-candidate-worker.ts | 95 ++++++++++++++++--- .../evolution/test/filesystem-store.test.ts | 55 +++++++++-- .../test/skill-candidate-worker.test.ts | 34 ++++++- 9 files changed, 290 insertions(+), 32 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d50d2319e..efc57e03a 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -23,6 +23,7 @@ - Fixed an open, idle `/evolve` menu retaining enabled actions after another Diffex process disables evolution. - Fixed one failed skill candidate synthesis preventing later qualified candidates from being drafted, validated, and evaluated. - Fixed exhausted skill synthesis being retried by unrelated later evolution jobs instead of remaining terminal until new evidence revises the candidate. +- Fixed persisted skill candidate failures discarding the affected candidate group, work stage, provider or schema error, stop reason, and structured diagnostics. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 5afaa8694..b5ce8418e 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -272,7 +272,17 @@ export class CodingAgentEvolutionModelExecutor implements EvolutionModelExecutor } } if (assistant.stopReason === "error" || assistant.stopReason === "aborted") { - throw new Error(assistant.errorMessage ?? `Evolution model stopped with ${assistant.stopReason}`); + throw new EvolutionExecutionError( + assistant.errorMessage ?? `Evolution model stopped with ${assistant.stopReason}`, + { + reasonCode: "model-call-failed", + retryable: true, + diagnostics: { + stopReason: assistant.stopReason, + providerError: assistant.errorMessage ?? null, + }, + }, + ); } messages.push(assistant); const calls = assistant.content.filter((part) => part.type === "toolCall"); diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index 5b7f751c4..a29eeee8b 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -659,6 +659,46 @@ describe("coding-agent evolution runtime", () => { }); }); + it("preserves provider stop diagnostics for evolution failures", async () => { + const harness = await createHarness(); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage("", { + stopReason: "error", + errorMessage: "Provider rejected the structured response", + }), + ]); + const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, { + async waitUntilBackgroundAllowed() {}, + }); + const model = harness.getModel(); + + await expect( + executor.execute({ + stage: "skill-synthesizer", + model: { + provider: model.provider, + model: model.id, + configurationDigest: `sha256:${"1".repeat(64)}`, + }, + promptVersion: "skill-synthesizer-v1", + idempotencyKey: "provider-stop-diagnostics", + input: {}, + outputSchema: { type: "object" }, + tools: [], + }), + ).rejects.toMatchObject({ + name: "EvolutionExecutionError", + message: "Provider rejected the structured response", + reasonCode: "model-call-failed", + retryable: true, + diagnostics: { + stopReason: "error", + providerError: "Provider rejected the structured response", + }, + }); + }); + it("corrects an invalid response-tool submission within the active transcript", async () => { const harness = await createHarness(); harnesses.push(harness); diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 1e191206e..c2200527e 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -976,6 +976,25 @@ export interface SkillCandidateWorkCheckpointedEvent extends SkillCandidateRegis readonly groupIds: readonly string[]; } +export interface SkillCandidateWorkErrorCause { + readonly name: string; + readonly message: string; +} + +export interface SkillCandidateWorkErrorDiagnostic { + readonly name: string; + readonly message: string; + readonly reasonCode: EvolutionFailureReasonCode | null; + readonly retryable: boolean | null; + readonly diagnostics: JsonValue | null; + readonly cause: SkillCandidateWorkErrorCause | null; +} + +export interface SkillCandidateGroupFailureDiagnostic { + readonly groupId: string | null; + readonly error: SkillCandidateWorkErrorDiagnostic; +} + export interface SkillCandidateWorkFailedEvent extends SkillCandidateRegistryEventBase { readonly type: "skill-candidate-work-failed"; readonly workId: string; @@ -984,6 +1003,10 @@ export interface SkillCandidateWorkFailedEvent extends SkillCandidateRegistryEve readonly attempt: number; readonly message: string; readonly retryNotBefore: string | null; + /** Absent only on records written before structured candidate-work failure diagnostics were introduced. */ + readonly stage?: SkillCandidateWorkStage; + /** Absent only on records written before structured candidate-work failure diagnostics were introduced. */ + readonly diagnostics?: readonly SkillCandidateGroupFailureDiagnostic[]; } export type SkillCandidateRegistryEvent = diff --git a/packages/evolution/src/filesystem-store.ts b/packages/evolution/src/filesystem-store.ts index 0079022ac..278850140 100644 --- a/packages/evolution/src/filesystem-store.ts +++ b/packages/evolution/src/filesystem-store.ts @@ -300,6 +300,34 @@ function isStringArray(value: unknown): value is readonly string[] { return Array.isArray(value) && value.every((item) => typeof item === "string"); } +function isJsonValue(value: unknown): boolean { + if (value === null || typeof value === "string" || typeof value === "boolean") return true; + if (typeof value === "number") return Number.isFinite(value); + if (Array.isArray(value)) return value.every(isJsonValue); + return isRecord(value) && Object.values(value).every(isJsonValue); +} + +function isCandidateWorkErrorDiagnostic(value: unknown): boolean { + return ( + isRecord(value) && + typeof value.name === "string" && + typeof value.message === "string" && + (value.reasonCode === null || typeof value.reasonCode === "string") && + (value.retryable === null || typeof value.retryable === "boolean") && + isJsonValue(value.diagnostics) && + (value.cause === null || + (isRecord(value.cause) && typeof value.cause.name === "string" && typeof value.cause.message === "string")) + ); +} + +function isCandidateGroupFailureDiagnostic(value: unknown): boolean { + return ( + isRecord(value) && + (value.groupId === null || typeof value.groupId === "string") && + isCandidateWorkErrorDiagnostic(value.error) + ); +} + function isArtifactReference(value: unknown): boolean { return ( isRecord(value) && @@ -403,16 +431,29 @@ function isCandidateRegistryEventContent(value: Readonly isStringArray(value.groupIds) ); } + if ( + value.type !== "skill-candidate-work-failed" || + typeof value.workId !== "string" || + typeof value.sourceEvolutionJobId !== "string" || + typeof value.idempotencyKey !== "string" || + !Number.isSafeInteger(value.attempt) || + (value.attempt as number) < 1 || + typeof value.message !== "string" || + (value.retryNotBefore !== null && + (typeof value.retryNotBefore !== "string" || !Number.isFinite(Date.parse(value.retryNotBefore)))) + ) { + return false; + } + const hasStage = value.stage !== undefined; + const hasDiagnostics = value.diagnostics !== undefined; + if (hasStage !== hasDiagnostics) return false; + if (!hasStage) return true; return ( - value.type === "skill-candidate-work-failed" && - typeof value.workId === "string" && - typeof value.sourceEvolutionJobId === "string" && - typeof value.idempotencyKey === "string" && - Number.isSafeInteger(value.attempt) && - (value.attempt as number) > 0 && - typeof value.message === "string" && - (value.retryNotBefore === null || - (typeof value.retryNotBefore === "string" && Number.isFinite(Date.parse(value.retryNotBefore)))) + typeof value.stage === "string" && + SKILL_CANDIDATE_WORK_STAGES.has(value.stage) && + Array.isArray(value.diagnostics) && + value.diagnostics.length > 0 && + value.diagnostics.every(isCandidateGroupFailureDiagnostic) ); } diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 1959d836a..df49cc68f 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -135,6 +135,7 @@ export { type SkillCandidateEvidenceContribution, type SkillCandidateFailureHandling, type SkillCandidateGroup, + type SkillCandidateGroupFailureDiagnostic, type SkillCandidateGroupingProfile, type SkillCandidateMemoryRelationship, type SkillCandidateOperation, @@ -150,6 +151,8 @@ export { type SkillCandidateStateTransitionedEvent, type SkillCandidateValidationRecordedEvent, type SkillCandidateWorkCheckpointedEvent, + type SkillCandidateWorkErrorCause, + type SkillCandidateWorkErrorDiagnostic, type SkillCandidateWorkFailedEvent, type SkillCandidateWorkStage, type SkillCandidateWorkspaceScope, diff --git a/packages/evolution/src/skill-candidate-worker.ts b/packages/evolution/src/skill-candidate-worker.ts index 9b1284c3f..bc662c19d 100644 --- a/packages/evolution/src/skill-candidate-worker.ts +++ b/packages/evolution/src/skill-candidate-worker.ts @@ -6,14 +6,17 @@ import type { EvolutionLeaseToken, EvolutionScope, SkillCandidate, + SkillCandidateGroupFailureDiagnostic, SkillCandidateRegistryEvent, SkillCandidateWorkCheckpointedEvent, + SkillCandidateWorkErrorDiagnostic, SkillCandidateWorkFailedEvent, SkillCandidateWorkStage, SkillDraft, SkillDraftValidationReport, } from "./contracts.ts"; import { SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION } from "./contracts.ts"; +import { EvolutionExecutionError } from "./errors.ts"; import type { EvolutionArtifact, EvolutionArtifactStore, @@ -190,6 +193,43 @@ function retryDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): n return Math.min(maxDelayMs, baseDelayMs * 2 ** Math.max(0, attempt - 1)); } +function errorDiagnostic(error: unknown): SkillCandidateWorkErrorDiagnostic { + const executionError = error instanceof EvolutionExecutionError ? error : null; + const cause = error instanceof Error ? error.cause : undefined; + return { + name: error instanceof Error ? error.name : "Error", + message: error instanceof Error ? error.message : String(error), + reasonCode: executionError?.reasonCode ?? null, + retryable: executionError?.retryable ?? null, + diagnostics: executionError?.diagnostics ?? null, + cause: + cause instanceof Error + ? { name: cause.name, message: cause.message } + : cause === undefined + ? null + : { name: "Error", message: String(cause) }, + }; +} + +class SkillCandidateWorkStageError extends AggregateError { + readonly stage: SkillCandidateWorkStage; + readonly failures: readonly { readonly groupId: string; readonly error: unknown }[]; + + constructor( + stage: SkillCandidateWorkStage, + failures: readonly { readonly groupId: string; readonly error: unknown }[], + message: string, + ) { + super( + failures.map((failure) => failure.error), + message, + ); + this.name = "SkillCandidateWorkStageError"; + this.stage = stage; + this.failures = failures; + } +} + export class PollingSkillCandidateWorkSource { readonly #dependencies: PollingSkillCandidateWorkSourceDependencies; readonly #pollIntervalMs: number; @@ -357,7 +397,14 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor }); } - async #appendFailure(job: EvolutionJob, attempt: number, error: unknown, lease: EvolutionLeaseToken): Promise { + async #appendFailure( + job: EvolutionJob, + attempt: number, + stage: SkillCandidateWorkStage, + diagnostics: readonly SkillCandidateGroupFailureDiagnostic[], + error: unknown, + lease: EvolutionLeaseToken, + ): Promise { const idempotencyKey = stableId("skill-candidate-work-failure", { sourceEvolutionJobId: job.jobId, attempt, @@ -393,9 +440,12 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor : new Date( Date.parse(now) + retryDelay(attempt, this.#retryBaseDelayMs, this.#retryMaxDelayMs), ).toISOString(), + stage, + diagnostics, }; + const groupIds = diagnostics.flatMap((diagnostic) => diagnostic.groupId ?? []); return { - metadata: this.#metadata(content.eventId, [job.jobId], job.metadata.scope, now, content), + metadata: this.#metadata(content.eventId, [job.jobId, ...groupIds], job.metadata.scope, now, content), ...content, }; }); @@ -498,8 +548,10 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor const attempt = history.filter((event) => event.type === "skill-candidate-work-failed").length + 1; if (attempt > this.#maxAttempts) throw new Error(`Skill candidate work for "${job.jobId}" exhausted retries`); + let activeStage: SkillCandidateWorkStage = "consolidating"; + let activeGroupId: string | null = null; try { - await this.#appendCheckpoint(job, "consolidating", attempt, [], claim()); + await this.#appendCheckpoint(job, activeStage, attempt, [], claim()); const inputArtifact = await this.#dependencies.artifacts.get(job.inputSnapshot); if (!isInputSnapshot(inputArtifact)) throw new Error("Evolution job input snapshot artifact is invalid"); const artifacts = await Promise.all( @@ -542,11 +594,13 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor ), ]), ]; - await this.#appendCheckpoint(job, "scoring", attempt, processedGroupIds, claim()); + activeStage = "scoring"; + await this.#appendCheckpoint(job, activeStage, attempt, processedGroupIds, claim()); const synthesisTargets = result.groups.filter((group) => group.state === "evidence-qualified"); + activeStage = "synthesizing"; await this.#appendCheckpoint( job, - "synthesizing", + activeStage, attempt, synthesisTargets.map((group) => group.groupId), claim(), @@ -559,6 +613,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor readonly error: unknown; }[] = []; for (const group of synthesisTargets) { + activeGroupId = group.groupId; signal.throwIfAborted(); let draftReference = group.draft; if (draftReference === null) { @@ -595,6 +650,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor }), ); } + activeGroupId = null; if (synthesisFailures.length > 0 && attempt >= this.#maxAttempts) { await this.#dependencies.registry.recordSynthesisFailures({ scope: job.metadata.scope, @@ -603,9 +659,10 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor createdAt: this.#dependencies.clock.now(), }); } + activeStage = "validating"; await this.#appendCheckpoint( job, - "validating", + activeStage, attempt, draftedGroups.map((group) => group.groupId), claim(), @@ -616,6 +673,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor .map((group) => [group.groupId, group]), ); for (const group of draftedGroups) { + activeGroupId = group.groupId; signal.throwIfAborted(); if (group.draft === null) throw new Error(`Drafted candidate group "${group.groupId}" has no draft`); const artifact = await this.#dependencies.artifacts.get(group.draft); @@ -643,10 +701,13 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor }); if (validated.state === "structurally-valid") evaluationGroups.set(validated.groupId, validated); } - await this.#appendCheckpoint(job, "evaluating", attempt, [...evaluationGroups.keys()], claim()); + activeStage = "evaluating"; + activeGroupId = null; + await this.#appendCheckpoint(job, activeStage, attempt, [...evaluationGroups.keys()], claim()); const evaluatedGroupIds: string[] = []; if (this.#dependencies.evaluator === undefined) { for (const evaluationGroup of evaluationGroups.values()) { + activeGroupId = evaluationGroup.groupId; signal.throwIfAborted(); if (evaluationGroup.draft === null || evaluationGroup.validation === null) { throw new Error(`Candidate group "${evaluationGroup.groupId}" lacks evaluation prerequisites`); @@ -673,6 +734,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor } } else { for (const evaluationGroup of evaluationGroups.values()) { + activeGroupId = evaluationGroup.groupId; signal.throwIfAborted(); if (evaluationGroup.draft === null || evaluationGroup.validation === null) { throw new Error(`Candidate group "${evaluationGroup.groupId}" lacks evaluation prerequisites`); @@ -718,22 +780,33 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor evaluatedGroupIds.push(pending.groupId); } } + activeGroupId = null; if (synthesisFailures.length > 0) { - throw new AggregateError( - synthesisFailures.map((failure) => failure.error), + throw new SkillCandidateWorkStageError( + "synthesizing", + synthesisFailures, `Skill synthesis failed for candidate groups: ${synthesisFailures.map((failure) => failure.groupId).join(", ")}`, ); } + activeStage = "completed"; await this.#appendCheckpoint( job, - "completed", + activeStage, attempt, [...new Set([...processedGroupIds, ...evaluationGroups.keys(), ...evaluatedGroupIds])], claim(), ); } catch (error) { if (signal.aborted) throw error; - await this.#appendFailure(job, attempt, error, claim()); + const stage = error instanceof SkillCandidateWorkStageError ? error.stage : activeStage; + const diagnostics: readonly SkillCandidateGroupFailureDiagnostic[] = + error instanceof SkillCandidateWorkStageError + ? error.failures.map((failure) => ({ + groupId: failure.groupId, + error: errorDiagnostic(failure.error), + })) + : [{ groupId: activeGroupId, error: errorDiagnostic(error) }]; + await this.#appendFailure(job, attempt, stage, diagnostics, error, claim()); throw error; } } diff --git a/packages/evolution/test/filesystem-store.test.ts b/packages/evolution/test/filesystem-store.test.ts index 26b396963..625071b4e 100644 --- a/packages/evolution/test/filesystem-store.test.ts +++ b/packages/evolution/test/filesystem-store.test.ts @@ -139,6 +139,46 @@ function candidateWorkEvent(eventId: string, registrySequence: number): SkillCan }; } +function candidateWorkFailureEvent(eventId: string, registrySequence: number): SkillCandidateRegistryEvent { + const content = { + schemaVersion: SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, + eventId, + registrySequence, + type: "skill-candidate-work-failed" as const, + workId: "candidate-work-1", + sourceEvolutionJobId: "job-1", + idempotencyKey: "candidate-work-1:failure-1", + attempt: 1, + message: "Skill synthesis failed for candidate groups: group-1", + retryNotBefore: null, + stage: "synthesizing" as const, + diagnostics: [ + { + groupId: "group-1", + error: { + name: "EvolutionExecutionError", + message: "Skill synthesizer failed to produce a draft", + reasonCode: "skill-synthesis-invalid-response" as const, + retryable: false, + diagnostics: { lastCorrection: "files must be an array" }, + cause: { name: "ProviderError", message: "Provider rejected the structured response" }, + }, + }, + ], + }; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: eventId, + producer, + sourceIds: ["job-1", "group-1"], + scope, + createdAt: "2026-08-19T12:00:00.000Z", + content, + }), + ...content, + }; +} + function candidateStateEvent(eventId: string, registrySequence: number): SkillCandidateRegistryEvent { const content = { schemaVersion: SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, @@ -295,18 +335,19 @@ describe("filesystem evolution persistence", () => { const evaluation = candidateEvaluationEvent("candidate-event-2", 2); const synthesisFailed = candidateStateEvent("candidate-event-3", 3); - expect(await store.appendSkillCandidateRegistryEvents(scope, 0, [first, evaluation, synthesisFailed])).toBe( - "committed", - ); + const workFailure = candidateWorkFailureEvent("candidate-event-4", 4); + expect( + await store.appendSkillCandidateRegistryEvents(scope, 0, [first, evaluation, synthesisFailed, workFailure]), + ).toBe("committed"); expect(await peer.loadSkillCandidateRegistry(scope)).toEqual({ - revision: 3, - events: [first, evaluation, synthesisFailed], + revision: 4, + events: [first, evaluation, synthesisFailed, workFailure], }); expect( - await peer.appendSkillCandidateRegistryEvents(scope, 0, [candidateWorkEvent("candidate-event-4", 1)]), + await peer.appendSkillCandidateRegistryEvents(scope, 0, [candidateWorkEvent("candidate-event-5", 1)]), ).toBe("conflict"); await expect( - peer.appendSkillCandidateRegistryEvents(scope, 3, [candidateWorkEvent("candidate-event-4", 5)]), + peer.appendSkillCandidateRegistryEvents(scope, 4, [candidateWorkEvent("candidate-event-5", 6)]), ).rejects.toThrow("contiguous sequence"); }); diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index 264a7f4d8..c96733104 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -6,6 +6,7 @@ import { type EvolutionArtifactProducer, type EvolutionArtifactReference, type EvolutionArtifactStore, + EvolutionExecutionError, type EvolutionInputSnapshot, type EvolutionJob, type EvolutionJobStore, @@ -453,7 +454,7 @@ function fixture(withEvaluator = false, store: MemoryRegistryStore = new MemoryR const leases = new MemoryLeaseStore(); const registry = new FixtureRegistry(); const synthesisRequests: SkillSynthesizerInput[] = []; - const synthesisFailures = new Set(); + const synthesisFailures = new Map(); const validationRequests: SkillDraftValidatorInput[] = []; const evaluationRequests: SkillEvaluatorInput[] = []; let clockTime = now; @@ -483,7 +484,8 @@ function fixture(withEvaluator = false, store: MemoryRegistryStore = new MemoryR synthesizer: { async synthesize(request): Promise { synthesisRequests.push(request); - if (synthesisFailures.has(request.group.groupId)) throw new Error("synthesis failed"); + const synthesisFailure = synthesisFailures.get(request.group.groupId); + if (synthesisFailure !== undefined) throw synthesisFailure; const content = { artifactKind: "skill-draft" as const, draftId: "draft-1", @@ -757,7 +759,17 @@ describe("skill candidate registry worker", () => { evaluationIdempotencyKey: "skill-evaluation-succeeding", }, ); - data.synthesisFailures.add("group-failing"); + const providerError = new Error("Provider rejected the structured response"); + providerError.name = "ProviderError"; + data.synthesisFailures.set( + "group-failing", + new EvolutionExecutionError("Skill synthesizer failed to produce a draft", { + reasonCode: "skill-synthesis-invalid-response", + retryable: false, + cause: providerError, + diagnostics: { lastCorrection: "files must be an array" }, + }), + ); await expect(data.worker.process(data.job.jobId)).rejects.toThrow( "Skill synthesis failed for candidate groups: group-failing", @@ -777,6 +789,20 @@ describe("skill candidate registry worker", () => { ); expect(data.store.events.find((event) => event.type === "skill-candidate-work-failed")).toMatchObject({ message: "Skill synthesis failed for candidate groups: group-failing", + stage: "synthesizing", + diagnostics: [ + { + groupId: "group-failing", + error: { + name: "EvolutionExecutionError", + message: "Skill synthesizer failed to produce a draft", + reasonCode: "skill-synthesis-invalid-response", + retryable: false, + diagnostics: { lastCorrection: "files must be an array" }, + cause: { name: "ProviderError", message: "Provider rejected the structured response" }, + }, + }, + ], }); expect( data.store.events.some( @@ -805,7 +831,7 @@ describe("skill candidate registry worker", () => { const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; if (candidateReference === undefined) throw new Error("Expected candidate reference"); data.registry.groups.push(qualifiedGroup(candidateReference)); - data.synthesisFailures.add("group-qualified"); + data.synthesisFailures.set("group-qualified", new Error("synthesis failed")); for (let attempt = 1; attempt <= 3; attempt += 1) { await expect(data.worker.process(data.job.jobId)).rejects.toThrow( From e511ff82b67bd7dc7c8ccfc99f8ae700aa02c3ea Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 30 Aug 2026 09:13:08 -0400 Subject: [PATCH 06/10] fix(agent): expose candidate lifecycle outcomes --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/docs/evolution.md | 3 +- .../src/core/evolution-runtime.ts | 1 + .../interactive/components/evolution-menu.ts | 134 +++++++++++-- .../test/evolution-footer-status.test.ts | 9 + .../coding-agent/test/evolution-menu.test.ts | 159 ++++++++++++++-- .../coding-agent/test/footer-width.test.ts | 18 ++ packages/evolution/src/control-service.ts | 176 ++++++++++++++++-- packages/evolution/src/index.ts | 3 + .../evolution/test/control-service.test.ts | 144 +++++++++++++- 10 files changed, 604 insertions(+), 44 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index efc57e03a..9ba096639 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -24,6 +24,7 @@ - Fixed one failed skill candidate synthesis preventing later qualified candidates from being drafted, validated, and evaluated. - Fixed exhausted skill synthesis being retried by unrelated later evolution jobs instead of remaining terminal until new evidence revises the candidate. - Fixed persisted skill candidate failures discarding the affected candidate group, work stage, provider or schema error, stop reason, and structured diagnostics. +- Fixed completed evolution analysis hiding asynchronous candidate-generation progress or failure, candidate lifecycle states and reasons, and distinct stage outcomes. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). diff --git a/packages/coding-agent/docs/evolution.md b/packages/coding-agent/docs/evolution.md index d02a8c3ed..e3d71d312 100644 --- a/packages/coding-agent/docs/evolution.md +++ b/packages/coding-agent/docs/evolution.md @@ -9,7 +9,8 @@ Use `/evolve` to: - inspect evidence eligibility and running or completed jobs; - start, resume, cancel, or browse evolution work; - inspect memory proposals and their evidence; -- inspect skill-candidate provenance, textual draft diffs, deterministic validation, and retrieval/behavior evaluation when available; +- distinguish analysis completion from waiting, retrying, completed, partially failed, or failed candidate generation; +- inspect each skill candidate's registry state, latest transition time and reason, provenance, textual draft diff, deterministic validation, and retrieval/behavior evaluation when available; - activate a skill candidate after deterministic validation succeeds and any available persisted evaluation is accepted; - inspect target revisions and roll back an activated evolved skill to its pre-activation revision, even when later descendants preserved it; - enable or disable future tracing and evolution work. diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index b5ce8418e..84ceaf625 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -1010,6 +1010,7 @@ export async function createCodingAgentEvolutionRuntime( harnesses: options.harnessState, progress, skillCandidates: candidateRegistry, + skillCandidateRegistryStore: persistence, skillActivation: { async activate(activationScope, groupId) { const lease = await persistence.acquire( diff --git a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts index 141aaac34..70df39b0b 100644 --- a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts +++ b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts @@ -79,6 +79,7 @@ type MenuView = interface MenuEntry { readonly label: string; readonly enabled: boolean; + readonly status?: string; readonly disabledStatus?: string; readonly select: () => void; } @@ -106,9 +107,21 @@ function firstCharacters(value: string, maximum: number): string { return [...sanitizeEvolutionText(value).replace(/\s+/gu, " ").trim()].slice(0, maximum).join(""); } +export function formatEvolutionJobResult(job: EvolutionJobSummary): string { + if (job.analysisStatus === "waiting") return "analysis waiting"; + if (job.analysisStatus === "processing") return "analysis processing"; + if (job.analysisStatus === "failed") return "analysis failed"; + if (job.candidateGeneration.status === "not-applicable" || job.candidateGeneration.status === "completed") { + return "full success"; + } + if (job.candidateGeneration.status === "partial-failure") return "partial failure"; + if (job.candidateGeneration.status === "failed") return "candidate generation failed"; + return `analysis completed; candidate generation ${job.candidateGeneration.status}`; +} + export function formatEvolutionJobLabel(job: EvolutionJobSummary, now = new Date()): string { const trigger = job.trigger === "automatic" ? "auto" : "manual"; - return `${formatEvolutionTimestamp(job.createdAt, now)} - ${trigger} - ${job.stage}`; + return `${formatEvolutionTimestamp(job.createdAt, now)} - ${trigger} - ${formatEvolutionJobResult(job)}`; } export function formatEvolutionProposalLabel(proposal: EvolutionProposalDetail["proposal"]): string { @@ -150,8 +163,65 @@ export function formatEvolutionEvidencePreview( return firstCharacters(content, 100); } +export type EvolutionSkillCandidateStage = "draft" | "evaluation" | "validation"; +export type EvolutionSkillCandidateStageStatus = + | "waiting" + | "blocked" + | "failed" + | "skipped" + | "completed" + | "not applicable"; + +export function evolutionSkillCandidateStageStatus( + detail: EvolutionSkillCandidateDetail, + stage: EvolutionSkillCandidateStage, +): EvolutionSkillCandidateStageStatus { + const { state } = detail.summary; + if (stage === "draft") { + if (detail.draft !== null) return "completed"; + if (state === "synthesis-failed") return "failed"; + if (state === "proposed" || state === "quarantined") return "blocked"; + if (state === "rejected" || state === "retired") return "not applicable"; + return "waiting"; + } + if (stage === "validation") { + if (detail.validation !== null) return detail.validation.disposition === "accepted" ? "completed" : "failed"; + if (state === "synthesis-failed") return "skipped"; + if (state === "proposed" || state === "quarantined") return "blocked"; + if (state === "rejected" || state === "retired") return "not applicable"; + return "waiting"; + } + if (detail.evaluation !== null) return detail.evaluation.disposition === "accepted" ? "completed" : "failed"; + if (state === "accepted" && detail.summary.latestReasonCodes.includes("retrieval-and-behavior-evaluation-skipped")) { + return "skipped"; + } + if (state === "proposed") return "blocked"; + if (state === "quarantined" || state === "rejected" || state === "synthesis-failed" || state === "retired") { + return "not applicable"; + } + return "waiting"; +} + +export function explainEvolutionCandidateReason(reasonCode: string): string { + const explanations: Readonly> = { + "awaiting-independent-recurrence": "Waiting for support from another independent task.", + "candidate-recorded": "The candidate was recorded for lifecycle review.", + "counterevidence-meets-or-exceeds-support": "Counterevidence matches or exceeds the supporting evidence.", + "deterministic-draft-validation-passed": "Deterministic draft validation passed.", + "evaluation-gates-not-passed": "Retrieval or behavior evaluation did not pass.", + "independent-recurrence-qualified": "Independent recurring evidence qualified the candidate.", + "outcome-aware-draft-recorded": "A candidate draft was generated and recorded.", + "proposal-revision-set-changed": "New proposal evidence requires candidate regeneration.", + "retrieval-and-behavior-evaluation-skipped": "Retrieval and behavior evaluation was unavailable and skipped.", + "retrieval-and-behavior-evaluation-started": "Retrieval and behavior evaluation started.", + "retrieval-and-behavior-gates-passed": "Retrieval and behavior evaluation passed.", + "skill-synthesis-attempts-exhausted": "Skill generation failed after all retry attempts.", + }; + return explanations[reasonCode] ?? `${reasonCode.replaceAll("-", " ")}.`; +} + export function formatEvolutionSkillDraftDiff(detail: EvolutionSkillCandidateDetail): readonly string[] { - if (detail.draft === null) return ["Draft pending"]; + if (detail.draft === null) return ["Draft not available"]; const before = detail.existingContents .map((skill) => `# Existing skill: ${skill.name} (${skill.version})\n${skill.content}`) .join("\n\n"); @@ -559,10 +629,25 @@ export class EvolutionMenuComponent extends Container implements Focusable { } #renderJob(job: EvolutionJobDetail): void { + const candidateStage = + job.candidateGeneration.stage === null || job.candidateGeneration.stage === "completed" + ? "" + : `, ${job.candidateGeneration.stage}`; + const candidateAttempt = + job.candidateGeneration.attempt === 0 ? "" : `, attempt ${job.candidateGeneration.attempt}`; for (const line of [ `Trigger: ${job.trigger}`, - `Stage: ${job.stage} (${job.outcome})`, - `Finished: ${formatEvolutionTimestamp(job.terminalAt)}`, + `Result: ${formatEvolutionJobResult(job)}`, + `Analysis: ${job.analysisStatus}`, + `Candidate generation: ${job.candidateGeneration.status.replaceAll("-", " ")}${candidateStage}${candidateAttempt}`, + `Analysis finished: ${formatEvolutionTimestamp(job.terminalAt)}`, + `Candidate updated: ${formatEvolutionTimestamp(job.candidateGeneration.updatedAt)}`, + ...(job.candidateGeneration.retryNotBefore === null + ? [] + : [`Candidate retry: ${formatEvolutionTimestamp(job.candidateGeneration.retryNotBefore)}`]), + ...(job.candidateGeneration.failureMessage === null + ? [] + : [`Candidate error: ${job.candidateGeneration.failureMessage}`]), `Evolution token: ${job.observability.estimatedEvidenceTokens.toLocaleString()}`, `Last checkpoint: ${formatEvolutionTimestamp(job.lastCheckpointAt)}`, `Base harness: ${formatHarnessVersionReference( @@ -573,7 +658,7 @@ export class EvolutionMenuComponent extends Container implements Focusable { )}`, `Target harness: ${ job.activatedHarnessVersion === null - ? "pending" + ? "not activated" : formatHarnessVersionReference( job.activatedHarnessVersion, this.#harnessVersionInfo, @@ -589,7 +674,7 @@ export class EvolutionMenuComponent extends Container implements Focusable { #renderProposal(detail: EvolutionProposalDetail): void { this.#addLine(formatEvolutionProposalLabel(detail.proposal)); this.addChild(new Spacer(1)); - this.#addLine(`Validation: ${detail.validationDisposition ?? "pending"}`); + this.#addLine(`Validation: ${detail.validationDisposition ?? "not attempted"}`); this.addChild(new Spacer(1)); this.#addLine(`Outcome: ${formatEvolutionProposalOutcome(detail.proposal)}`); } @@ -602,25 +687,30 @@ export class EvolutionMenuComponent extends Container implements Focusable { { label: "Draft diff", enabled: detail.draft !== null, - disabledStatus: "pending", + status: evolutionSkillCandidateStageStatus(detail, "draft"), select: () => review("draft"), }, { label: "Validation", enabled: detail.validation !== null, - disabledStatus: "pending", + status: evolutionSkillCandidateStageStatus(detail, "validation"), select: () => review("validation"), }, { label: "Evaluation", enabled: detail.evaluation !== null, - disabledStatus: detail.summary.state === "accepted" ? "skipped" : "pending", + status: evolutionSkillCandidateStageStatus(detail, "evaluation"), select: () => review("evaluation"), }, { label: "Activate", enabled: detail.activationAvailable, - disabledStatus: detail.targetHarnessVersion === null ? "unavailable" : "active", + disabledStatus: + detail.targetHarnessVersion !== null + ? "active" + : detail.summary.state === "accepted" + ? "unavailable" + : "blocked", select: () => this.#confirm("activate-skill", jobId, detail.summary.candidateId), }, { @@ -634,7 +724,17 @@ export class EvolutionMenuComponent extends Container implements Focusable { #renderSkillCandidate(detail: EvolutionSkillCandidateDetail): void { this.#addLine(`Skill candidate: ${detail.summary.proposedName}`, "accent"); - this.#addLine(`Target harness: ${detail.targetHarnessVersion ?? "pending activation"}`); + this.#addLine(`State: ${detail.summary.state}`); + this.#addLine(`State changed: ${formatEvolutionTimestamp(detail.summary.latestTransitionAt)}`); + for (const reasonCode of detail.summary.latestReasonCodes) { + this.#addLine(`Reason: ${explainEvolutionCandidateReason(reasonCode)}`); + this.#addLine(`Reason code: ${reasonCode}`, "muted"); + } + this.#addLine( + `Target harness: ${ + detail.targetHarnessVersion ?? (detail.summary.state === "accepted" ? "not activated" : "not applicable") + }`, + ); } #renderSkillCandidateReview( @@ -666,13 +766,15 @@ export class EvolutionMenuComponent extends Container implements Focusable { } if (section === "validation") { this.#addLine("Deterministic validation", "accent"); - this.#addLine(`Result: ${detail.validation?.disposition ?? "pending"}`); + this.#addLine(`Result: ${detail.validation?.disposition ?? "not attempted"}`); for (const issue of detail.validation?.issues ?? []) this.#addLine(`${issue.code}: ${issue.message}`, "warning"); return; } this.#addLine("Retrieval and behavior evaluation", "accent"); - this.#addLine(`Result: ${detail.evaluation?.disposition ?? "pending"}`); + this.#addLine( + `Result: ${detail.evaluation?.disposition ?? evolutionSkillCandidateStageStatus(detail, "evaluation")}`, + ); for (const [name, gate] of Object.entries(detail.evaluation?.gates ?? {})) { this.#addLine(`${name}: ${gate.passed ? "passed" : `failed (${gate.reasonCodes.join(", ")})`}`); } @@ -731,8 +833,10 @@ export class EvolutionMenuComponent extends Container implements Focusable { entries.forEach((entry, index) => { const selected = index === this.#selectedIndex; const prefix = selected ? SELECTED_LIST_PREFIX : UNSELECTED_LIST_PREFIX; - const disabled = entry.enabled ? "" : ` [${entry.disabledStatus ?? "unavailable"}]`; - const line = `${prefix}${entry.label}${disabled}`; + const status = entry.status === undefined ? "" : ` [${entry.status}]`; + const disabled = + entry.enabled || entry.status !== undefined ? "" : ` [${entry.disabledStatus ?? "unavailable"}]`; + const line = `${prefix}${entry.label}${status}${disabled}`; this.addChild( new TruncatedText(theme.fg(entry.enabled ? (selected ? "selectionAccent" : "text") : "muted", line), 1, 0), ); diff --git a/packages/coding-agent/test/evolution-footer-status.test.ts b/packages/coding-agent/test/evolution-footer-status.test.ts index e543ae273..50cefa720 100644 --- a/packages/coding-agent/test/evolution-footer-status.test.ts +++ b/packages/coding-agent/test/evolution-footer-status.test.ts @@ -41,6 +41,15 @@ function activeJob(): NonNullable { stage: "reflecting", outcome: "active", reasonCode: null, + analysisStatus: "processing", + candidateGeneration: { + status: "not-applicable", + stage: null, + updatedAt: null, + attempt: 0, + retryNotBefore: null, + failureMessage: null, + }, createdAt: "2026-08-20T00:00:00.000Z", updatedAt: "2026-08-20T00:01:00.000Z", terminalAt: null, diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index b877000a0..c8a7b5548 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -14,6 +14,8 @@ import { KeybindingsManager } from "../src/core/keybindings.ts"; import { type EvolutionMenuClient, EvolutionMenuComponent, + evolutionSkillCandidateStageStatus, + explainEvolutionCandidateReason, formatEvolutionEvidencePreview, formatEvolutionJobLabel, formatEvolutionProposalLabel, @@ -34,6 +36,15 @@ function job(overrides: Partial = {}): EvolutionJobSummary stage: "reflecting", outcome: "active", reasonCode: null, + analysisStatus: "processing", + candidateGeneration: { + status: "not-applicable", + stage: null, + updatedAt: null, + attempt: 0, + retryNotBefore: null, + failureMessage: null, + }, createdAt: "2026-08-20T00:00:00.000Z", updatedAt: "2026-08-20T00:01:00.000Z", terminalAt: null, @@ -185,6 +196,8 @@ function skillCandidateDetail(overrides: Partial proposedName: "focused-verification", operation: "CREATE", state: "accepted", + latestTransitionAt: "2026-08-20T00:00:02.000Z", + latestReasonCodes: ["retrieval-and-behavior-gates-passed"], }, candidate: { sourceLessonIds: ["lesson-1"], @@ -502,6 +515,7 @@ describe("EvolutionMenuComponent", () => { const failedDetail = jobDetail({ outcome: "terminal-failed", reasonCode: "reflection-invalid-response", + analysisStatus: "failed", terminalAt: terminalAt.toISOString(), emittedProposals: 0, eligibleProposals: 0, @@ -528,9 +542,10 @@ describe("EvolutionMenuComponent", () => { await vi.waitFor(() => expect(fixture.jobDetailCall.mock.calls.length).toBeGreaterThanOrEqual(2)); const output = stripAnsi(component.render(180).join("\n")); - expect(output).toContain("Finished: Today at 1:25 pm"); + expect(output).toContain("Analysis finished: Today at 1:25 pm"); + expect(output).toContain("Result: analysis failed"); + expect(output).toContain("Analysis: failed"); for (const removedLabel of [ - "Result:", "Proposals:", "Reflector budget:", "Coverage:", @@ -566,12 +581,14 @@ describe("EvolutionMenuComponent", () => { const output = stripAnsi(component.render(180).join("\n")); expect(output).toContain("Trigger: manual"); - expect(output).toContain("Stage: reflecting (active)"); - expect(output).toContain("Finished: never"); + expect(output).toContain("Result: analysis processing"); + expect(output).toContain("Analysis: processing"); + expect(output).toContain("Candidate generation: not applicable"); + expect(output).toContain("Analysis finished: never"); expect(output).toContain("Evolution token: 12,345"); expect(output).toContain("Last checkpoint: Today at 9:05 am"); expect(output).toContain("Base harness: [current] Default Diffex Agent"); - expect(output).toContain("Target harness: pending"); + expect(output).toContain("Target harness: not activated"); expect(output).toContain("Rollback available: no"); for (const removedLabel of [ "Job:", @@ -586,7 +603,6 @@ describe("EvolutionMenuComponent", () => { "Proposal dispositions:", "Activation conflicts:", "Next prompt will refresh:", - "Result:", "Proposals:", "Reflector budget:", "Coverage:", @@ -597,6 +613,54 @@ describe("EvolutionMenuComponent", () => { } }); + it("shows terminal asynchronous candidate failure after analysis completed", async () => { + const candidateGeneration = { + status: "partial-failure" as const, + stage: "synthesizing" as const, + updatedAt: "2026-08-20T12:35:00.000Z", + attempt: 3, + retryNotBefore: null, + failureMessage: "Skill synthesis failed for candidate group group-1", + }; + const summary = job({ + stage: "completed", + outcome: "completed", + analysisStatus: "completed", + candidateGeneration, + terminalAt: "2026-08-20T12:34:00.000Z", + }); + const fixture = client(overview()); + fixture.listJobs.mockResolvedValue([summary]); + fixture.jobDetailCall.mockResolvedValue( + jobDetail({ + ...summary, + candidateGeneration, + }), + ); + const component = new EvolutionMenuComponent( + fixture.client, + () => {}, + () => {}, + () => {}, + ); + await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Browse")); + + for (let index = 0; index < 4; index += 1) component.handleInput("\u001b[B"); + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain(formatEvolutionJobLabel(summary)), + ); + component.handleInput("\n"); + await vi.waitFor(() => expect(fixture.jobDetailCall).toHaveBeenCalledWith("job-active")); + + const output = stripAnsi(component.render(180).join("\n")); + expect(output).toContain("Result: partial failure"); + expect(output).toContain("Analysis: completed"); + expect(output).toContain("Candidate generation: partial failure, synthesizing, attempt 3"); + expect(output).toContain("Candidate error: Skill synthesis failed for candidate group group-1"); + component.dispose(); + }); + it("renders a readable proposal outcome without internal mutation diagnostics", async () => { const detail = proposalDetail(); const fixture = client(overview()); @@ -700,14 +764,17 @@ describe("EvolutionMenuComponent", () => { let output = stripAnsi(component.render(180).join("\n")); expect(output).toContain("Skill candidate: focused-verification"); - expect(output).toContain("Target harness: pending activation"); + expect(output).toContain("State: accepted"); + expect(output).toContain("State changed:"); + expect(output).toContain("Reason: Retrieval and behavior evaluation passed."); + expect(output).toContain("Reason code: retrieval-and-behavior-gates-passed"); + expect(output).toContain("Target harness: not activated"); expect(output).not.toContain("Operation:"); - expect(output).not.toContain("State:"); expect(output).not.toContain("Rollback available:"); expect(output).toContain("Provenance"); - expect(output).toContain("Draft diff"); - expect(output).toContain("Validation"); - expect(output).toContain("Evaluation"); + expect(output).toContain("Draft diff [completed]"); + expect(output).toContain("Validation [completed]"); + expect(output).toContain("Evaluation [completed]"); expect(output).toContain("Activate"); component.handleInput("\n"); @@ -744,7 +811,14 @@ describe("EvolutionMenuComponent", () => { }); it("labels skipped skill evaluation without blocking explicit activation", async () => { - const detail = skillCandidateDetail({ evaluation: null }); + const baseDetail = skillCandidateDetail(); + const detail = skillCandidateDetail({ + evaluation: null, + summary: { + ...baseDetail.summary, + latestReasonCodes: ["retrieval-and-behavior-evaluation-skipped"], + }, + }); const fixture = client(overview()); fixture.listJobs.mockResolvedValue([job()]); fixture.jobDetailCall.mockResolvedValue(jobDetail({ skillCandidates: [detail.summary] })); @@ -776,6 +850,50 @@ describe("EvolutionMenuComponent", () => { component.dispose(); }); + it("distinguishes blocked, failed, skipped, completed, and inapplicable candidate stages", () => { + const base = skillCandidateDetail(); + const detailWithState = ( + state: EvolutionSkillCandidateDetail["summary"]["state"], + overrides: Partial = {}, + ) => + skillCandidateDetail({ + ...overrides, + summary: { ...base.summary, state, ...(overrides.summary ?? {}) }, + }); + const proposed = detailWithState("proposed", { draft: null, validation: null, evaluation: null }); + expect([ + evolutionSkillCandidateStageStatus(proposed, "draft"), + evolutionSkillCandidateStageStatus(proposed, "validation"), + evolutionSkillCandidateStageStatus(proposed, "evaluation"), + ]).toEqual(["blocked", "blocked", "blocked"]); + + const synthesisFailed = detailWithState("synthesis-failed", { + draft: null, + validation: null, + evaluation: null, + }); + expect([ + evolutionSkillCandidateStageStatus(synthesisFailed, "draft"), + evolutionSkillCandidateStageStatus(synthesisFailed, "validation"), + evolutionSkillCandidateStageStatus(synthesisFailed, "evaluation"), + ]).toEqual(["failed", "skipped", "not applicable"]); + + const rejected = detailWithState("rejected", { + validation: { ...base.validation, disposition: "rejected" } as EvolutionSkillCandidateDetail["validation"], + evaluation: null, + }); + expect([ + evolutionSkillCandidateStageStatus(rejected, "draft"), + evolutionSkillCandidateStageStatus(rejected, "validation"), + evolutionSkillCandidateStageStatus(rejected, "evaluation"), + ]).toEqual(["completed", "failed", "not applicable"]); + + expect(explainEvolutionCandidateReason("counterevidence-meets-or-exceeds-support")).toBe( + "Counterevidence matches or exceeds the supporting evidence.", + ); + expect(explainEvolutionCandidateReason("custom-raw-code")).toBe("custom raw code."); + }); + it("formats create, update, and retirement draft diffs", () => { const createDiff = formatEvolutionSkillDraftDiff(skillCandidateDetail()); expect(createDiff).toContain("+Run checks."); @@ -841,10 +959,23 @@ describe("EvolutionMenuComponent", () => { expect(formatEvolutionTimestamp(new Date(2026, 6, 28, 21, 30).toISOString(), now)).toBe("7/28 at 9:30 pm"); expect( formatEvolutionJobLabel( - job({ trigger: "automatic", stage: "completed", createdAt: new Date(2026, 7, 20, 9, 5).toISOString() }), + job({ + trigger: "automatic", + stage: "completed", + analysisStatus: "completed", + candidateGeneration: { + status: "completed", + stage: "completed", + updatedAt: new Date(2026, 7, 20, 9, 6).toISOString(), + attempt: 1, + retryNotBefore: null, + failureMessage: null, + }, + createdAt: new Date(2026, 7, 20, 9, 5).toISOString(), + }), now, ), - ).toBe("Today at 9:05 am - auto - completed"); + ).toBe("Today at 9:05 am - auto - full success"); }); it("limits semantic proposal and evidence labels by Unicode characters", () => { diff --git a/packages/coding-agent/test/footer-width.test.ts b/packages/coding-agent/test/footer-width.test.ts index e373968b4..ae21b971e 100644 --- a/packages/coding-agent/test/footer-width.test.ts +++ b/packages/coding-agent/test/footer-width.test.ts @@ -256,6 +256,15 @@ describe("FooterComponent width handling", () => { stage: "reflecting", outcome: "active", reasonCode: null, + analysisStatus: "processing", + candidateGeneration: { + status: "not-applicable", + stage: null, + updatedAt: null, + attempt: 0, + retryNotBefore: null, + failureMessage: null, + }, createdAt: evolvedAt, updatedAt: evolvedAt, terminalAt: null, @@ -285,6 +294,15 @@ describe("FooterComponent width handling", () => { stage: "reflecting", outcome: "active", reasonCode: null, + analysisStatus: "processing", + candidateGeneration: { + status: "not-applicable", + stage: null, + updatedAt: null, + attempt: 0, + retryNotBefore: null, + failureMessage: null, + }, createdAt: evolvedAt, updatedAt: evolvedAt, terminalAt: null, diff --git a/packages/evolution/src/control-service.ts b/packages/evolution/src/control-service.ts index cc007adbc..581ea191a 100644 --- a/packages/evolution/src/control-service.ts +++ b/packages/evolution/src/control-service.ts @@ -25,7 +25,11 @@ import type { SkillCandidate, SkillCandidateGroup, SkillCandidateOperation, + SkillCandidateRegistryEvent, SkillCandidateState, + SkillCandidateWorkCheckpointedEvent, + SkillCandidateWorkFailedEvent, + SkillCandidateWorkStage, SkillDraft, SkillDraftValidationReport, SkillEvaluationReport, @@ -43,6 +47,7 @@ import type { EvolutionLedger, EvolutionProgressEvent, SkillCandidateRegistry, + SkillCandidateRegistryStore, } from "./ports.ts"; import { computeEvolutionTextDigest } from "./provenance.ts"; import { @@ -59,12 +64,34 @@ export interface EvolutionHarnessControl { selectVersionIfCurrent(version: HarnessVersion, expectedCurrent: HarnessVersion): Promise; } +export type EvolutionAnalysisStatus = "waiting" | "processing" | "completed" | "failed"; + +export type EvolutionCandidateGenerationStatus = + | "waiting" + | "processing" + | "retrying" + | "completed" + | "partial-failure" + | "failed" + | "not-applicable"; + +export interface EvolutionCandidateGenerationSummary { + readonly status: EvolutionCandidateGenerationStatus; + readonly stage: SkillCandidateWorkStage | null; + readonly updatedAt: string | null; + readonly attempt: number; + readonly retryNotBefore: string | null; + readonly failureMessage: string | null; +} + export interface EvolutionJobSummary { readonly jobId: string; readonly trigger: EvolutionJob["trigger"]; readonly stage: EvolutionJobStage; readonly outcome: EvolutionJobOutcome; readonly reasonCode: EvolutionFailureReasonCode | null; + readonly analysisStatus: EvolutionAnalysisStatus; + readonly candidateGeneration: EvolutionCandidateGenerationSummary; readonly createdAt: string; readonly updatedAt: string; readonly terminalAt: string | null; @@ -113,6 +140,8 @@ export interface EvolutionSkillCandidateSummary { readonly proposedName: string; readonly operation: SkillCandidateOperation; readonly state: SkillCandidateState; + readonly latestTransitionAt: string; + readonly latestReasonCodes: readonly string[]; } export interface EvolutionSkillCandidateExistingContent { @@ -224,6 +253,7 @@ export interface EvolutionControlServiceDependencies { readonly harnesses: EvolutionHarnessControl; readonly progress: EvolutionProgressBus; readonly skillCandidates?: SkillCandidateRegistry; + readonly skillCandidateRegistryStore?: SkillCandidateRegistryStore; readonly skillActivation?: EvolutionSkillActivationControl; readonly observedHarnessVersion: () => HarnessVersion | undefined; readonly currentDirectory: { @@ -273,13 +303,100 @@ function stringArray(value: unknown): readonly string[] { return Array.isArray(value) && value.every((entry) => typeof entry === "string") ? value : []; } -function summary(job: EvolutionJob): EvolutionJobSummary { +function analysisStatus(job: EvolutionJob): EvolutionAnalysisStatus { + if (job.outcome === "active") return job.stage === "queued" ? "waiting" : "processing"; + return job.outcome === "completed" ? "completed" : "failed"; +} + +function candidateGenerationSummary( + job: EvolutionJob, + groups: readonly SkillCandidateGroup[], + events: readonly SkillCandidateRegistryEvent[], +): EvolutionCandidateGenerationSummary { + const candidateIds = stringArray( + job.checkpoints.find((checkpoint) => checkpoint.stage === "skill-curating")?.state.candidateIds, + ); + if (job.outcome !== "completed" || candidateIds.length === 0) { + return { + status: "not-applicable", + stage: null, + updatedAt: null, + attempt: 0, + retryNotBefore: null, + failureMessage: null, + }; + } + const workEvents = events + .filter( + (event): event is SkillCandidateWorkCheckpointedEvent | SkillCandidateWorkFailedEvent => + (event.type === "skill-candidate-work-checkpointed" || event.type === "skill-candidate-work-failed") && + event.sourceEvolutionJobId === job.jobId, + ) + .sort((left, right) => left.registrySequence - right.registrySequence); + const latest = workEvents.at(-1); + if (latest === undefined) { + return { + status: "waiting", + stage: null, + updatedAt: null, + attempt: 0, + retryNotBefore: null, + failureMessage: null, + }; + } + if (latest.type === "skill-candidate-work-checkpointed") { + return { + status: latest.stage === "completed" ? "completed" : "processing", + stage: latest.stage, + updatedAt: latest.metadata.createdAt, + attempt: latest.attempt, + retryNotBefore: null, + failureMessage: null, + }; + } + if (latest.retryNotBefore !== null) { + return { + status: "retrying", + stage: latest.stage ?? null, + updatedAt: latest.metadata.createdAt, + attempt: latest.attempt, + retryNotBefore: latest.retryNotBefore, + failureMessage: latest.message, + }; + } + const relatedGroups = groups.filter((group) => + group.proposalRevisions.some((revision) => revision.sourceEvolutionJobId === job.jobId), + ); + const failedGroupIds = new Set([ + ...relatedGroups.filter((group) => group.state === "synthesis-failed").map((group) => group.groupId), + ...(latest.diagnostics ?? []).flatMap((diagnostic) => diagnostic.groupId ?? []), + ]); + return { + status: + failedGroupIds.size > 0 && relatedGroups.some((group) => !failedGroupIds.has(group.groupId)) + ? "partial-failure" + : "failed", + stage: latest.stage ?? null, + updatedAt: latest.metadata.createdAt, + attempt: latest.attempt, + retryNotBefore: null, + failureMessage: latest.message, + }; +} + +function summary( + job: EvolutionJob, + groups: readonly SkillCandidateGroup[] = [], + events: readonly SkillCandidateRegistryEvent[] = [], +): EvolutionJobSummary { return { jobId: job.jobId, trigger: job.trigger, stage: job.stage, outcome: job.outcome, reasonCode: job.reasonCode, + analysisStatus: analysisStatus(job), + candidateGeneration: candidateGenerationSummary(job, groups, events), createdAt: job.createdAt, updatedAt: job.updatedAt, terminalAt: job.terminalAt, @@ -340,12 +457,15 @@ function candidateSummary( candidate: SkillCandidate, group: SkillCandidateGroup | null, ): EvolutionSkillCandidateSummary { + const latestTransition = group?.dispositionHistory.at(-1); return { candidateId: candidate.candidateId, groupId: group?.groupId ?? null, proposedName: candidate.proposedName, operation: candidate.operation, state: group?.state ?? "proposed", + latestTransitionAt: latestTransition?.createdAt ?? candidate.metadata.createdAt, + latestReasonCodes: latestTransition?.reasonCodes ?? [], }; } @@ -413,6 +533,30 @@ export class EvolutionControlService { this.#dependencies = dependencies; } + async #candidateState(): Promise<{ + readonly groups: readonly SkillCandidateGroup[]; + readonly events: readonly SkillCandidateRegistryEvent[]; + }> { + const [groups, snapshot] = await Promise.all([ + this.#dependencies.skillCandidates?.list(this.#dependencies.scope) ?? Promise.resolve([]), + this.#dependencies.skillCandidateRegistryStore?.loadSkillCandidateRegistry(this.#dependencies.scope) ?? + Promise.resolve({ revision: 0, events: [] }), + ]); + return { groups, events: snapshot.events }; + } + + async #summaries(jobs: readonly EvolutionJob[]): Promise { + const candidateState = await this.#candidateState(); + return jobs.map((job) => summary(job, candidateState.groups, candidateState.events)); + } + + async #summary(job: EvolutionJob): Promise { + const summaries = await this.#summaries([job]); + const result = summaries[0]; + if (result === undefined) throw new Error(`Evolution job "${job.jobId}" summary is unavailable`); + return result; + } + start(): void { if (this.#started) return; this.#started = true; @@ -494,6 +638,11 @@ export class EvolutionControlService { previousAutomaticTerminalAt: lastAutomaticRunAt, }); const observedHarnessVersion = this.#dependencies.observedHarnessVersion() ?? null; + const projectedJobs = await this.#summaries([ + ...(active === null ? [] : [active]), + ...(resumable === null ? [] : [resumable]), + ]); + const projectedById = new Map(projectedJobs.map((job) => [job.jobId, job])); return { scope: this.#dependencies.scope, processedEvidenceCursor: cursor.sequence, @@ -505,8 +654,8 @@ export class EvolutionControlService { lastEvolutionAt, cooldownRemainingMs, automaticEligibility: eligibility, - activeJob: active === null ? null : summary(active), - newestResumableJob: resumable === null ? null : summary(resumable), + activeJob: active === null ? null : (projectedById.get(active.jobId) ?? null), + newestResumableJob: resumable === null ? null : (projectedById.get(resumable.jobId) ?? null), currentHarnessVersion: versions.currentVersion, observedHarnessVersion, nextPromptWillRefresh: @@ -518,10 +667,10 @@ export class EvolutionControlService { } async listJobs(): Promise { - return (await this.#dependencies.jobs.list(this.#dependencies.scope)) + const jobs = (await this.#dependencies.jobs.list(this.#dependencies.scope)) .filter((job) => !isCancelledJob(job)) - .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || right.jobId.localeCompare(left.jobId)) - .map(summary); + .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || right.jobId.localeCompare(left.jobId)); + return this.#summaries(jobs); } async #repositoryMatchesCurrentDirectory(repository: EvolutionRepositorySnapshot): Promise { @@ -593,11 +742,11 @@ export class EvolutionControlService { async #start(trigger: "automatic" | "manual"): Promise { for (let contentionAttempt = 0; contentionAttempt < 20; contentionAttempt += 1) { const result = await this.#dependencies.controlPlane.createJob({ scope: this.#dependencies.scope, trigger }); - if (result.status === "created") return { status: "created", job: summary(result.job) }; + if (result.status === "created") return { status: "created", job: await this.#summary(result.job) }; if (result.reason === "blocking-job" && result.blockingJobId !== undefined) { const blocking = await this.#dependencies.jobs.load(result.blockingJobId); if (blocking !== null) { - const blockingSummary = summary(blocking); + const blockingSummary = await this.#summary(blocking); if (blockingSummary.cancellable || blockingSummary.resumable) { return { status: "attached", job: blockingSummary }; } @@ -606,7 +755,7 @@ export class EvolutionControlService { } if (result.reason !== "scope-busy") return { status: "not-created", reason: result.reason }; const active = activeJob(await this.#dependencies.jobs.list(this.#dependencies.scope)); - if (active !== null) return { status: "attached", job: summary(active) }; + if (active !== null) return { status: "attached", job: await this.#summary(active) }; await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); } return { status: "not-created", reason: "scope-busy" }; @@ -623,7 +772,7 @@ export class EvolutionControlService { async resumeNewest(): Promise { const job = resumableJob(await this.#dependencies.jobs.list(this.#dependencies.scope)); if (job === null) throw new Error("No interrupted or retryable evolution job is available"); - return summary(await this.#dependencies.controlPlane.requestResume(job.jobId)); + return this.#summary(await this.#dependencies.controlPlane.requestResume(job.jobId)); } async cancelJob(jobId: string): Promise { @@ -664,16 +813,17 @@ export class EvolutionControlService { ) { throw new Error(`Unknown evolution job "${jobId}"`); } - const [artifacts, events, versions] = await Promise.all([ + const [artifacts, events, versions, candidateState] = await Promise.all([ loadArtifacts(job, this.#dependencies.artifacts), this.#dependencies.ledger.read(jobId), this.#dependencies.harnesses.listVersions(), + this.#candidateState(), ]); const input = artifacts.find(isInputSnapshot); if (input === undefined) throw new Error(`Evolution job "${jobId}" has no input snapshot`); const scored = artifacts.filter(isScoredProposal); const skillCandidates = artifacts.filter(isSkillCandidate); - const candidateGroups = (await this.#dependencies.skillCandidates?.list(this.#dependencies.scope)) ?? []; + const candidateGroups = candidateState.groups; const discoveredSlices = new Set( artifacts.flatMap((artifact) => ("sliceIds" in artifact ? stringArray(artifact.sliceIds) : [])), ).size; @@ -757,7 +907,7 @@ export class EvolutionControlService { versions.currentVersion === activation.resultingHarnessVersion && validVersions.has(activation.rollbackParentHarnessVersion); return { - ...summary(job), + ...summary(job, candidateGroups, candidateState.events), evidenceHighWaterCursor: input.ingestionHighWaterCursor, completedSlices, discoveredSlices, diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index df49cc68f..bd310baf7 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -204,6 +204,9 @@ export { type EvolutionJobCreationResult, } from "./control-plane.ts"; export { + type EvolutionAnalysisStatus, + type EvolutionCandidateGenerationStatus, + type EvolutionCandidateGenerationSummary, EvolutionControlService, type EvolutionControlServiceDependencies, type EvolutionHarnessControl, diff --git a/packages/evolution/test/control-service.test.ts b/packages/evolution/test/control-service.test.ts index 4d21b2738..9ad316106 100644 --- a/packages/evolution/test/control-service.test.ts +++ b/packages/evolution/test/control-service.test.ts @@ -25,6 +25,7 @@ import { PollingEvolutionWorkSource, type SkillCandidate, type SkillCandidateGroup, + type SkillCandidateRegistryEvent, type SkillDraft, type SkillDraftValidationReport, type SkillEvaluationReport, @@ -543,6 +544,130 @@ describe("evolution control service", () => { } }); + it("separates analysis completion from asynchronous candidate generation outcomes", async () => { + const timestamp = "2026-08-20T00:00:00.000Z"; + const candidateJob = (jobId: string): EvolutionJob => + ({ + metadata: { scope }, + jobId, + trigger: "manual", + stage: "completed", + outcome: "completed", + reasonCode: null, + failureMessage: null, + completion: { kind: "no-op", resultingHarnessVersion: null, processedEvidenceCursor: 1 }, + resolution: null, + checkpoints: [ + { + stage: "skill-curating", + state: { candidateIds: [`candidate-${jobId}`] }, + outputArtifacts: [], + completedAt: timestamp, + }, + ], + attempt: 1, + revision: 1, + createdAt: timestamp, + updatedAt: timestamp, + terminalAt: timestamp, + retryNotBefore: null, + }) as unknown as EvolutionJob; + const jobs = [ + candidateJob("waiting"), + candidateJob("retrying"), + candidateJob("completed"), + candidateJob("partial"), + candidateJob("failed"), + ]; + const event = (registrySequence: number, value: Readonly>): SkillCandidateRegistryEvent => + ({ + metadata: { createdAt: new Date(Date.parse(timestamp) + registrySequence).toISOString() }, + registrySequence, + ...value, + }) as unknown as SkillCandidateRegistryEvent; + const events = [ + event(1, { + type: "skill-candidate-work-failed", + sourceEvolutionJobId: "retrying", + attempt: 1, + stage: "synthesizing", + message: "temporary synthesis failure", + retryNotBefore: "2026-08-20T00:01:00.000Z", + diagnostics: [{ groupId: "group-retrying" }], + }), + event(2, { + type: "skill-candidate-work-checkpointed", + sourceEvolutionJobId: "completed", + attempt: 1, + stage: "completed", + }), + event(3, { + type: "skill-candidate-work-failed", + sourceEvolutionJobId: "partial", + attempt: 3, + stage: "synthesizing", + message: "one candidate exhausted synthesis", + retryNotBefore: null, + diagnostics: [{ groupId: "group-partial-failed" }], + }), + event(4, { + type: "skill-candidate-work-failed", + sourceEvolutionJobId: "failed", + attempt: 3, + stage: "validating", + message: "candidate validation exhausted retries", + retryNotBefore: null, + diagnostics: [{ groupId: "group-failed" }], + }), + ]; + const group = (groupId: string, sourceEvolutionJobId: string, state: SkillCandidateGroup["state"]) => + ({ groupId, state, proposalRevisions: [{ sourceEvolutionJobId }] }) as unknown as SkillCandidateGroup; + const groups = [ + group("group-partial-failed", "partial", "synthesis-failed"), + group("group-partial-accepted", "partial", "accepted"), + group("group-failed", "failed", "synthesis-failed"), + ]; + const service = new EvolutionControlService({ + scope, + jobs: { list: async () => jobs }, + skillCandidates: { list: async () => groups }, + skillCandidateRegistryStore: { + loadSkillCandidateRegistry: async () => ({ revision: events.length, events }), + }, + } as unknown as EvolutionControlServiceDependencies); + + const summaries = new Map((await service.listJobs()).map((summary) => [summary.jobId, summary])); + expect(summaries.get("waiting")).toMatchObject({ + analysisStatus: "completed", + candidateGeneration: { status: "waiting", attempt: 0 }, + }); + expect(summaries.get("retrying")).toMatchObject({ + analysisStatus: "completed", + candidateGeneration: { + status: "retrying", + stage: "synthesizing", + attempt: 1, + retryNotBefore: "2026-08-20T00:01:00.000Z", + failureMessage: "temporary synthesis failure", + }, + }); + expect(summaries.get("completed")).toMatchObject({ + analysisStatus: "completed", + candidateGeneration: { status: "completed", stage: "completed", attempt: 1 }, + }); + expect(summaries.get("partial")).toMatchObject({ + candidateGeneration: { status: "partial-failure", stage: "synthesizing", attempt: 3 }, + }); + expect(summaries.get("failed")).toMatchObject({ + candidateGeneration: { + status: "failed", + stage: "validating", + attempt: 3, + failureMessage: "candidate validation exhausted retries", + }, + }); + }); + it("exposes candidate provenance, gates activation, and rolls back through preserving descendants", async () => { const inputReference = { artifactId: "input-1", contentDigest: `sha256:${"1".repeat(64)}` }; const candidateReference = { artifactId: "candidate-1", contentDigest: `sha256:${"2".repeat(64)}` }; @@ -551,6 +676,7 @@ describe("evolution control service", () => { const evaluationReference = { artifactId: "evaluation-1", contentDigest: `sha256:${"5".repeat(64)}` }; const candidate = { artifactKind: "skill-candidate", + metadata: { createdAt: "2026-08-20T00:00:00.000Z" }, candidateId: "candidate-1", operation: "CREATE", proposedName: "focused-verification", @@ -591,6 +717,15 @@ describe("evolution control service", () => { const group = { groupId: "group-1", state: "accepted", + dispositionHistory: [ + { + eventId: "transition-1", + from: "evaluation-pending", + to: "accepted", + reasonCodes: ["retrieval-and-behavior-gates-passed"], + createdAt: "2026-08-20T00:00:02.000Z", + }, + ], proposalRevisions: [ { revisionId: "revision-1", @@ -726,12 +861,19 @@ describe("evolution control service", () => { proposedName: candidate.proposedName, operation: "CREATE", state: "accepted", + latestTransitionAt: "2026-08-20T00:00:02.000Z", + latestReasonCodes: ["retrieval-and-behavior-gates-passed"], }, ], }); const detail = await service.getSkillCandidate(job.jobId, candidate.candidateId); expect(detail).toMatchObject({ - summary: { groupId: group.groupId, state: "accepted" }, + summary: { + groupId: group.groupId, + state: "accepted", + latestTransitionAt: "2026-08-20T00:00:02.000Z", + latestReasonCodes: ["retrieval-and-behavior-gates-passed"], + }, sourceEvolutionJobIds: [job.jobId], proposalRevisionIds: ["revision-1"], draft: { draftId: "draft-1" }, From 4d9c5cbfa01cbe89371ca0c88e238ea0259a2b9e Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 30 Aug 2026 09:13:52 -0400 Subject: [PATCH 07/10] fix(coding-agent): keep candidate status views current --- .../interactive/components/evolution-menu.ts | 22 +++++++- .../coding-agent/test/evolution-menu.test.ts | 51 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts index 70df39b0b..fe30ad9dc 100644 --- a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts +++ b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts @@ -384,7 +384,10 @@ export class EvolutionMenuComponent extends Container implements Focusable { this.#proposal = await this.#client.proposalDetail(view.jobId, view.proposalId); } if (view.kind === "skill-candidate" || view.kind === "skill-candidate-review") { - this.#skillCandidate = await this.#client.skillCandidateDetail(view.jobId, view.candidateId); + [this.#job, this.#skillCandidate] = await Promise.all([ + this.#client.jobDetail(view.jobId), + this.#client.skillCandidateDetail(view.jobId, view.candidateId), + ]); } } catch (error) { this.#onStatus(error instanceof Error ? error.message : String(error), true); @@ -399,7 +402,22 @@ export class EvolutionMenuComponent extends Container implements Focusable { #schedulePoll(): void { if (this.#pollTimeout !== null) clearTimeout(this.#pollTimeout); this.#pollTimeout = null; - if (this.#disposed || this.#overview === null || this.#overview.activeJob === null) return; + const nonterminalCandidateStatuses = new Set(["waiting", "processing", "retrying"]); + const view = this.#view(); + const viewingJob = + view.kind === "job" || view.kind === "skill-candidate" || view.kind === "skill-candidate-review"; + const viewedJobHasCandidateWork = + viewingJob && this.#job !== null && nonterminalCandidateStatuses.has(this.#job.candidateGeneration.status); + const listedJobHasCandidateWork = + view.kind === "jobs" && + this.#jobs.some((job) => nonterminalCandidateStatuses.has(job.candidateGeneration.status)); + if ( + this.#disposed || + this.#overview === null || + (this.#overview.activeJob === null && !viewedJobHasCandidateWork && !listedJobHasCandidateWork) + ) { + return; + } this.#pollTimeout = setTimeout(() => { this.#pollTimeout = null; void this.refresh(); diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index c8a7b5548..299ce6b3f 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -661,6 +661,57 @@ describe("EvolutionMenuComponent", () => { component.dispose(); }); + it("continues polling candidate work after the analysis job completes", async () => { + const waitingGeneration = { + status: "waiting" as const, + stage: null, + updatedAt: null, + attempt: 0, + retryNotBefore: null, + failureMessage: null, + }; + const completedGeneration = { + status: "completed" as const, + stage: "completed" as const, + updatedAt: "2026-08-20T12:35:00.000Z", + attempt: 1, + retryNotBefore: null, + failureMessage: null, + }; + const summary = job({ + stage: "completed", + outcome: "completed", + analysisStatus: "completed", + candidateGeneration: waitingGeneration, + terminalAt: "2026-08-20T12:34:00.000Z", + }); + const fixture = client(overview()); + fixture.listJobs.mockResolvedValue([summary]); + fixture.jobDetailCall + .mockResolvedValueOnce(jobDetail({ ...summary, candidateGeneration: waitingGeneration })) + .mockResolvedValue(jobDetail({ ...summary, candidateGeneration: completedGeneration })); + const component = new EvolutionMenuComponent( + fixture.client, + () => {}, + () => {}, + () => {}, + 5, + ); + await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Browse")); + + for (let index = 0; index < 4; index += 1) component.handleInput("\u001b[B"); + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain(formatEvolutionJobLabel(summary)), + ); + component.handleInput("\n"); + await vi.waitFor(() => expect(fixture.jobDetailCall.mock.calls.length).toBeGreaterThanOrEqual(2)); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain("Candidate generation: completed"), + ); + component.dispose(); + }); + it("renders a readable proposal outcome without internal mutation diagnostics", async () => { const detail = proposalDetail(); const fixture = client(overview()); From c15c98bf82c77af32c8a89393810d79e0884dbad Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 30 Aug 2026 09:29:07 -0400 Subject: [PATCH 08/10] fix(agent): cover mixed candidate lifecycle outcomes --- .../coding-agent/test/evolution-menu.test.ts | 19 +++ .../test/evolution-runtime.test.ts | 19 +++ .../test/skill-candidate-worker.test.ts | 108 ++++++++++++++---- 3 files changed, 123 insertions(+), 23 deletions(-) diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index 299ce6b3f..da203a151 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -939,6 +939,25 @@ describe("EvolutionMenuComponent", () => { evolutionSkillCandidateStageStatus(rejected, "evaluation"), ]).toEqual(["completed", "failed", "not applicable"]); + const quarantined = detailWithState("quarantined", { + evaluation: { ...base.evaluation, disposition: "quarantined" } as EvolutionSkillCandidateDetail["evaluation"], + }); + expect([ + evolutionSkillCandidateStageStatus(quarantined, "draft"), + evolutionSkillCandidateStageStatus(quarantined, "validation"), + evolutionSkillCandidateStageStatus(quarantined, "evaluation"), + ]).toEqual(["completed", "completed", "failed"]); + + const skipped = detailWithState("accepted", { + evaluation: null, + summary: { ...base.summary, latestReasonCodes: ["retrieval-and-behavior-evaluation-skipped"] }, + }); + expect([ + evolutionSkillCandidateStageStatus(skipped, "draft"), + evolutionSkillCandidateStageStatus(skipped, "validation"), + evolutionSkillCandidateStageStatus(skipped, "evaluation"), + ]).toEqual(["completed", "completed", "skipped"]); + expect(explainEvolutionCandidateReason("counterevidence-meets-or-exceeds-support")).toBe( "Counterevidence matches or exceeds the supporting evidence.", ); diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index a29eeee8b..56781bca1 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -1347,6 +1347,25 @@ describe("coding-agent evolution runtime", () => { harnessRevision: activation.targetHarnessVersion, }), ]); + + await expect(runtime.control.getSkillCandidate(started.job.jobId, candidateId)).resolves.toMatchObject({ + targetHarnessVersion: activation.targetHarnessVersion, + rollbackTargetHarnessVersion: root.manifest.version, + rollbackAvailable: true, + activationAvailable: false, + }); + await expect(runtime.control.rollbackSkillCandidate(started.job.jobId, candidateId)).resolves.toMatchObject({ + fromHarnessVersion: activation.targetHarnessVersion, + toHarnessVersion: root.manifest.version, + }); + expect((await store.loadCurrent()).manifest.version).toBe(root.manifest.version); + await expect(runtime.control.getSkillCandidate(started.job.jobId, candidateId)).resolves.toMatchObject({ + summary: { state: "accepted" }, + targetHarnessVersion: null, + rollbackTargetHarnessVersion: null, + rollbackAvailable: false, + activationAvailable: true, + }); } finally { await Promise.all([peerRuntime.stop(), runtime.stop()]); } diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index c96733104..89044e8e4 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -1,3 +1,6 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { computeEvolutionContentDigest, @@ -6,6 +9,8 @@ import { type EvolutionArtifactProducer, type EvolutionArtifactReference, type EvolutionArtifactStore, + EvolutionControlService, + type EvolutionControlServiceDependencies, EvolutionExecutionError, type EvolutionInputSnapshot, type EvolutionJob, @@ -13,6 +18,7 @@ import { type EvolutionLeaseStore, type EvolutionLeaseToken, type EvolutionScope, + FileEvolutionPersistence, PollingSkillCandidateWorkSource, type SkillCandidate, type SkillCandidateDraftRecordingInput, @@ -235,7 +241,11 @@ class MemoryArtifactStore implements EvolutionArtifactStore { } class MemoryRegistryStore implements SkillCandidateRegistryStore { - readonly events: SkillCandidateRegistryEvent[] = []; + readonly events: SkillCandidateRegistryEvent[]; + + constructor(events: readonly SkillCandidateRegistryEvent[] = []) { + this.events = [...events]; + } async loadSkillCandidateRegistry(): Promise { return { revision: this.events.length, events: [...this.events] }; @@ -826,16 +836,32 @@ describe("skill candidate registry worker", () => { ).toBe(true); }); - it("makes exhausted synthesis failures terminal until the proposal revision set changes", async () => { + it("durably exposes exhausted synthesis without blocking a later candidate in the same batch", async () => { const data = fixture(true); const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; if (candidateReference === undefined) throw new Error("Expected candidate reference"); - data.registry.groups.push(qualifiedGroup(candidateReference)); - data.synthesisFailures.set("group-qualified", new Error("synthesis failed")); + const group = qualifiedGroup(candidateReference); + data.registry.groups.push( + { + ...group, + groupId: "group-failing", + synthesisIdempotencyKey: "skill-synthesis-failing", + validationIdempotencyKey: "skill-validation-failing", + evaluationIdempotencyKey: "skill-evaluation-failing", + }, + { + ...group, + groupId: "group-succeeding", + synthesisIdempotencyKey: "skill-synthesis-succeeding", + validationIdempotencyKey: "skill-validation-succeeding", + evaluationIdempotencyKey: "skill-evaluation-succeeding", + }, + ); + data.synthesisFailures.set("group-failing", new Error("synthesis failed")); for (let attempt = 1; attempt <= 3; attempt += 1) { await expect(data.worker.process(data.job.jobId)).rejects.toThrow( - "Skill synthesis failed for candidate groups: group-qualified", + "Skill synthesis failed for candidate groups: group-failing", ); expect( data.store.events.find( @@ -844,31 +870,67 @@ describe("skill candidate registry worker", () => { ).toMatchObject({ retryNotBefore: attempt === 3 ? null : expect.any(String) }); } - expect(data.synthesisRequests).toHaveLength(3); - expect(data.registry.groups[0]).toMatchObject({ - state: "synthesis-failed", - dispositionHistory: [ + expect(data.synthesisRequests.map((request) => request.group.groupId)).toEqual([ + "group-failing", + "group-succeeding", + "group-failing", + "group-failing", + ]); + expect(data.registry.groups).toEqual([ + expect.objectContaining({ + groupId: "group-failing", + state: "synthesis-failed", + dispositionHistory: [ + expect.objectContaining({ + from: "evidence-qualified", + to: "synthesis-failed", + reasonCodes: ["skill-synthesis-attempts-exhausted"], + }), + ], + }), + expect.objectContaining({ groupId: "group-succeeding", state: "accepted" }), + ]); + + const persistenceRoot = await mkdtemp(join(tmpdir(), "diffex-mixed-candidate-outcomes-")); + try { + const durableStore = new FileEvolutionPersistence(persistenceRoot); + expect(await durableStore.appendSkillCandidateRegistryEvents(scope, 0, data.store.events)).toBe("committed"); + const reloadedStore = new FileEvolutionPersistence(persistenceRoot); + const source = new PollingSkillCandidateWorkSource({ + clock: data.clock, + jobs: data.jobs, + store: reloadedStore, + registry: data.registry, + evaluatorAvailable: true, + }); + expect(await source.nextRunnableJob(now)).toBeNull(); + const control = new EvolutionControlService({ + scope, + jobs: data.jobs, + skillCandidates: data.registry, + skillCandidateRegistryStore: reloadedStore, + } as unknown as EvolutionControlServiceDependencies); + expect(await control.listJobs()).toEqual([ expect.objectContaining({ - from: "evidence-qualified", - to: "synthesis-failed", - reasonCodes: ["skill-synthesis-attempts-exhausted"], + candidateGeneration: { + status: "partial-failure", + stage: "synthesizing", + updatedAt: expect.any(String), + attempt: 3, + retryNotBefore: null, + failureMessage: "Skill synthesis failed for candidate groups: group-failing", + }, }), - ], - }); - const source = new PollingSkillCandidateWorkSource({ - clock: data.clock, - jobs: data.jobs, - store: data.store, - registry: data.registry, - evaluatorAvailable: true, - }); - expect(await source.nextRunnableJob(now)).toBeNull(); + ]); + } finally { + await rm(persistenceRoot, { recursive: true, force: true }); + } const nextJob = { ...data.job, jobId: "job-2" }; data.jobs.jobs.push(nextJob); await data.worker.process(nextJob.jobId); - expect(data.synthesisRequests).toHaveLength(3); + expect(data.synthesisRequests).toHaveLength(4); expect( data.store.events.some( (event) => From 2baf78949b788383736dfabfcdd4b68926f9c085 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 30 Aug 2026 10:12:26 -0400 Subject: [PATCH 09/10] fix(agent): show readable skill candidate evidence --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/README.md | 2 +- packages/coding-agent/docs/evolution.md | 4 +- .../interactive/components/evolution-menu.ts | 105 +++++++++++++----- .../coding-agent/test/evolution-menu.test.ts | 55 +++++++-- packages/evolution/src/control-service.ts | 32 +++--- .../evolution/test/control-service.test.ts | 25 ++++- 7 files changed, 165 insertions(+), 59 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 9ba096639..525e0d669 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -25,6 +25,7 @@ - Fixed exhausted skill synthesis being retried by unrelated later evolution jobs instead of remaining terminal until new evidence revises the candidate. - Fixed persisted skill candidate failures discarding the affected candidate group, work stage, provider or schema error, stop reason, and structured diagnostics. - Fixed completed evolution analysis hiding asynchronous candidate-generation progress or failure, candidate lifecycle states and reasons, and distinct stage outcomes. +- Fixed skill candidate review leading with unreadable provenance IDs instead of an `EVOLVE.md`-style summary and materialized supporting and contradicting evidence. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). diff --git a/packages/coding-agent/README.md b/packages/coding-agent/README.md index 2240f3c8f..4eb2e32a3 100644 --- a/packages/coding-agent/README.md +++ b/packages/coding-agent/README.md @@ -234,7 +234,7 @@ Configure delivery in [settings](docs/settings.md): `steeringMode` and `followUp ### Harness Evolution -`/evolve` shows evidence eligibility, jobs, memory proposals, generated skill candidates, provenance, draft diffs, validation, evaluation, target revisions, activation, and rollback. `/version` selects a complete immutable harness revision. Memory and evolved skills change together at the next prompt boundary; in-flight and admitted queued requests remain pinned. The footer shows the active harness and background activity. See [Harness Evolution](docs/evolution.md). +`/evolve` shows evidence eligibility, jobs, memory proposals, generated skill candidates, readable supporting and contradicting evidence, draft diffs, validation, evaluation, target revisions, activation, and rollback. `/version` selects a complete immutable harness revision. Memory and evolved skills change together at the next prompt boundary; in-flight and admitted queued requests remain pinned. The footer shows the active harness and background activity. See [Harness Evolution](docs/evolution.md). --- diff --git a/packages/coding-agent/docs/evolution.md b/packages/coding-agent/docs/evolution.md index e3d71d312..2a70e45f8 100644 --- a/packages/coding-agent/docs/evolution.md +++ b/packages/coding-agent/docs/evolution.md @@ -10,7 +10,7 @@ Use `/evolve` to: - start, resume, cancel, or browse evolution work; - inspect memory proposals and their evidence; - distinguish analysis completion from waiting, retrying, completed, partially failed, or failed candidate generation; -- inspect each skill candidate's registry state, latest transition time and reason, provenance, textual draft diff, deterministic validation, and retrieval/behavior evaluation when available; +- inspect each skill candidate's registry state, latest transition time and reason, readable supporting and contradicting evidence, textual draft diff, deterministic validation, and retrieval/behavior evaluation when available; - activate a skill candidate after deterministic validation succeeds and any available persisted evaluation is accepted; - inspect target revisions and roll back an activated evolved skill to its pre-activation revision, even when later descendants preserved it; - enable or disable future tracing and evolution work. @@ -40,7 +40,7 @@ Evolution model calls use the configured provider and may consume provider quota Source traces, tool output, files, and generated drafts are treated as untrusted data. Deterministic validation checks structure, provenance, workspace scope, immutable versions, the closed package inventory, portable paths, local-reference resolution, aggregate size limits, and bounded sensitive-data rules. Retrieval and behavioral evaluation remain separate capabilities but are temporarily skipped when the runtime has no evaluator. -These checks do not prove semantic safety and are not a sandbox. When evaluation is skipped, Diffex has not verified retrieval quality, behavior improvement, misapplication, or regressions. Review candidate provenance, draft diffs, validation issues, and any available evaluation results before activation. See [Security](security.md). +These checks do not prove semantic safety and are not a sandbox. When evaluation is skipped, Diffex has not verified retrieval quality, behavior improvement, misapplication, or regressions. Review candidate supporting and contradicting evidence, draft diffs, validation issues, and any available evaluation results before activation. See [Security](security.md). ## Storage and recovery diff --git a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts index fe30ad9dc..41f06a843 100644 --- a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts +++ b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts @@ -61,7 +61,13 @@ type MenuView = readonly kind: "skill-candidate-review"; readonly jobId: string; readonly candidateId: string; - readonly section: "draft" | "evaluation" | "provenance" | "validation"; + readonly section: "draft" | "evaluation" | "validation"; + } + | { + readonly kind: "skill-candidate-evidence"; + readonly jobId: string; + readonly candidateId: string; + readonly evidenceKind: "supporting" | "contradicting"; } | { readonly kind: "proposal-evidence"; @@ -220,6 +226,12 @@ export function explainEvolutionCandidateReason(reasonCode: string): string { return explanations[reasonCode] ?? `${reasonCode.replaceAll("-", " ")}.`; } +export function formatEvolutionSkillCandidateOutcome(detail: EvolutionSkillCandidateDetail): string { + return detail.candidate.procedure.steps + .map((step) => (step.condition === null ? step.action : `${step.action} (${step.condition})`)) + .join(" "); +} + export function formatEvolutionSkillDraftDiff(detail: EvolutionSkillCandidateDetail): readonly string[] { if (detail.draft === null) return ["Draft not available"]; const before = detail.existingContents @@ -318,7 +330,11 @@ export class EvolutionMenuComponent extends Container implements Focusable { if (view.kind === "overview" || view.kind === "confirm") return this.#overview !== null; if (view.kind === "jobs") return this.#jobsLoaded; if (view.kind === "job") return this.#job?.jobId === view.jobId; - if (view.kind === "skill-candidate" || view.kind === "skill-candidate-review") { + if ( + view.kind === "skill-candidate" || + view.kind === "skill-candidate-review" || + view.kind === "skill-candidate-evidence" + ) { return this.#skillCandidate?.summary.candidateId === view.candidateId; } return this.#proposal?.proposal.proposalId === view.proposalId; @@ -383,7 +399,11 @@ export class EvolutionMenuComponent extends Container implements Focusable { if (view.kind === "proposal" || view.kind === "proposal-evidence") { this.#proposal = await this.#client.proposalDetail(view.jobId, view.proposalId); } - if (view.kind === "skill-candidate" || view.kind === "skill-candidate-review") { + if ( + view.kind === "skill-candidate" || + view.kind === "skill-candidate-review" || + view.kind === "skill-candidate-evidence" + ) { [this.#job, this.#skillCandidate] = await Promise.all([ this.#client.jobDetail(view.jobId), this.#client.skillCandidateDetail(view.jobId, view.candidateId), @@ -405,7 +425,10 @@ export class EvolutionMenuComponent extends Container implements Focusable { const nonterminalCandidateStatuses = new Set(["waiting", "processing", "retrying"]); const view = this.#view(); const viewingJob = - view.kind === "job" || view.kind === "skill-candidate" || view.kind === "skill-candidate-review"; + view.kind === "job" || + view.kind === "skill-candidate" || + view.kind === "skill-candidate-review" || + view.kind === "skill-candidate-evidence"; const viewedJobHasCandidateWork = viewingJob && this.#job !== null && nonterminalCandidateStatuses.has(this.#job.candidateGeneration.status); const listedJobHasCandidateWork = @@ -700,8 +723,24 @@ export class EvolutionMenuComponent extends Container implements Focusable { #skillCandidateEntries(detail: EvolutionSkillCandidateDetail, jobId: string): readonly MenuEntry[] { const review = (section: Extract["section"]) => this.#push({ kind: "skill-candidate-review", jobId, candidateId: detail.summary.candidateId, section }); + const evidence = (evidenceKind: Extract["evidenceKind"]) => + this.#push({ + kind: "skill-candidate-evidence", + jobId, + candidateId: detail.summary.candidateId, + evidenceKind, + }); return [ - { label: "Provenance", enabled: true, select: () => review("provenance") }, + { + label: `Supporting evidence (${detail.supportingEvidence.length})`, + enabled: true, + select: () => evidence("supporting"), + }, + { + label: `Contradicting evidence (${detail.contradictingEvidence.length})`, + enabled: true, + select: () => evidence("contradicting"), + }, { label: "Draft diff", enabled: detail.draft !== null, @@ -741,18 +780,19 @@ export class EvolutionMenuComponent extends Container implements Focusable { } #renderSkillCandidate(detail: EvolutionSkillCandidateDetail): void { - this.#addLine(`Skill candidate: ${detail.summary.proposedName}`, "accent"); + this.#addLine(`Skill candidate ${detail.summary.proposedName}`, "accent"); + this.addChild(new Spacer(1)); this.#addLine(`State: ${detail.summary.state}`); this.#addLine(`State changed: ${formatEvolutionTimestamp(detail.summary.latestTransitionAt)}`); for (const reasonCode of detail.summary.latestReasonCodes) { this.#addLine(`Reason: ${explainEvolutionCandidateReason(reasonCode)}`); - this.#addLine(`Reason code: ${reasonCode}`, "muted"); } - this.#addLine( - `Target harness: ${ - detail.targetHarnessVersion ?? (detail.summary.state === "accepted" ? "not activated" : "not applicable") - }`, - ); + this.addChild(new Spacer(1)); + this.#addLine(`Use when: ${detail.candidate.trigger.description}`); + this.#addLine(`Outcome: ${formatEvolutionSkillCandidateOutcome(detail)}`); + if (detail.targetHarnessVersion !== null) { + this.#addLine(`Active harness: ${detail.targetHarnessVersion}`); + } } #renderSkillCandidateReview( @@ -768,20 +808,6 @@ export class EvolutionMenuComponent extends Container implements Focusable { if (lines.length > 200) this.#addLine(`... ${lines.length - 200} more lines`, "muted"); return; } - if (section === "provenance") { - this.#addLine("Provenance", "accent"); - for (const [label, values] of [ - ["Evolution jobs", detail.sourceEvolutionJobIds], - ["Candidate revisions", detail.proposalRevisionIds], - ["Lessons", detail.candidate.sourceLessonIds], - ["Evidence", [...detail.candidate.supportingEvidenceIds, ...detail.candidate.contradictingEvidenceIds]], - ["Traces", [...detail.candidate.supportingTraceIds, ...detail.candidate.contradictingTraceIds]], - ["Task lineages", detail.candidate.taskLineageIds], - ] as const) { - this.#addLine(`${label}: ${values.length === 0 ? "none" : values.join(", ")}`); - } - return; - } if (section === "validation") { this.#addLine("Deterministic validation", "accent"); this.#addLine(`Result: ${detail.validation?.disposition ?? "not attempted"}`); @@ -831,10 +857,7 @@ export class EvolutionMenuComponent extends Container implements Focusable { ]; } - #renderProposalEvidence(detail: EvolutionProposalDetail, evidenceKind: "supporting" | "contradicting"): void { - const supporting = evidenceKind === "supporting"; - const label = supporting ? "Supporting evidence" : "Contradicting evidence"; - const evidence = supporting ? detail.supportingEvidence : detail.contradictingEvidence; + #renderEvidence(label: string, evidence: EvolutionProposalDetail["supportingEvidence"]): void { this.#addLine(`${label} (${evidence.length})`, "accent"); if (evidence.length === 0) { this.#addLine(`No ${label.toLowerCase()}`, "muted"); @@ -845,6 +868,25 @@ export class EvolutionMenuComponent extends Container implements Focusable { } } + #renderProposalEvidence(detail: EvolutionProposalDetail, evidenceKind: "supporting" | "contradicting"): void { + const supporting = evidenceKind === "supporting"; + this.#renderEvidence( + supporting ? "Supporting evidence" : "Contradicting evidence", + supporting ? detail.supportingEvidence : detail.contradictingEvidence, + ); + } + + #renderSkillCandidateEvidence( + detail: EvolutionSkillCandidateDetail, + evidenceKind: "supporting" | "contradicting", + ): void { + const supporting = evidenceKind === "supporting"; + this.#renderEvidence( + supporting ? "Supporting evidence" : "Contradicting evidence", + supporting ? detail.supportingEvidence : detail.contradictingEvidence, + ); + } + #renderEntries(entries: readonly MenuEntry[]): void { this.#entries = entries; this.#selectedIndex = Math.max(0, Math.min(this.#selectedIndex, Math.max(0, entries.length - 1))); @@ -923,6 +965,9 @@ export class EvolutionMenuComponent extends Container implements Focusable { } else if (view.kind === "skill-candidate-review" && this.#skillCandidate) { this.#renderSkillCandidateReview(this.#skillCandidate, view.section); this.#entries = []; + } else if (view.kind === "skill-candidate-evidence" && this.#skillCandidate) { + this.#renderSkillCandidateEvidence(this.#skillCandidate, view.evidenceKind); + this.#entries = []; } else if (view.kind === "confirm") { for (const line of this.#confirmationLines(view)) this.#addLine(line, "warning"); this.addChild(new Spacer(1)); diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index da203a151..89c0ddfa7 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -20,6 +20,7 @@ import { formatEvolutionJobLabel, formatEvolutionProposalLabel, formatEvolutionProposalOutcome, + formatEvolutionSkillCandidateOutcome, formatEvolutionSkillDraftDiff, formatEvolutionTimestamp, sanitizeEvolutionText, @@ -202,11 +203,26 @@ function skillCandidateDetail(overrides: Partial candidate: { sourceLessonIds: ["lesson-1"], supportingEvidenceIds: ["evidence-1"], - contradictingEvidenceIds: [], + contradictingEvidenceIds: ["evidence-2"], supportingTraceIds: ["trace-1"], - contradictingTraceIds: [], + contradictingTraceIds: ["trace-2"], taskLineageIds: ["task-1"], + trigger: { + description: "A focused change needs outcome-based verification.", + positiveExamples: [], + negativeExamples: [], + }, + procedure: { + steps: [ + { sequence: 1, action: "Run the focused check.", condition: null }, + { sequence: 2, action: "Inspect the observed result.", condition: "the check fails" }, + ], + requiredTools: [], + environment: [], + }, }, + supportingEvidence: [evidence("evidence-1", "The focused browser verification passed after the change.")], + contradictingEvidence: [evidence("evidence-2", "A static check passed without exercising the changed path.")], sourceEvolutionJobIds: ["job-active"], proposalRevisionIds: ["revision-1"], existingContents: [], @@ -814,15 +830,21 @@ describe("EvolutionMenuComponent", () => { ); let output = stripAnsi(component.render(180).join("\n")); - expect(output).toContain("Skill candidate: focused-verification"); + expect(output).toContain("Skill candidate focused-verification"); expect(output).toContain("State: accepted"); expect(output).toContain("State changed:"); expect(output).toContain("Reason: Retrieval and behavior evaluation passed."); - expect(output).toContain("Reason code: retrieval-and-behavior-gates-passed"); - expect(output).toContain("Target harness: not activated"); + expect(output).toContain("Use when: A focused change needs outcome-based verification."); + expect(output).toContain("Outcome: Run the focused check. Inspect the observed result. (the check fails)"); + expect(output).not.toContain("Reason code:"); + expect(output).not.toContain("Target harness:"); expect(output).not.toContain("Operation:"); expect(output).not.toContain("Rollback available:"); - expect(output).toContain("Provenance"); + expect(output).not.toContain("Provenance"); + expect(output).not.toContain("job-active"); + expect(output).not.toContain("evidence-1"); + expect(output).toContain("Supporting evidence (1)"); + expect(output).toContain("Contradicting evidence (1)"); expect(output).toContain("Draft diff [completed]"); expect(output).toContain("Validation [completed]"); expect(output).toContain("Evaluation [completed]"); @@ -830,14 +852,24 @@ describe("EvolutionMenuComponent", () => { component.handleInput("\n"); await vi.waitFor(() => - expect(stripAnsi(component.render(180).join("\n"))).toContain("Evolution jobs: job-active"), + expect(stripAnsi(component.render(180).join("\n"))).toContain( + "1. The focused browser verification passed after the change.", + ), ); component.handleInput("\u001b"); component.handleInput("\u001b[B"); component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain( + "1. A static check passed without exercising the changed path.", + ), + ); + component.handleInput("\u001b"); + for (let index = 0; index < 2; index += 1) component.handleInput("\u001b[B"); + component.handleInput("\n"); await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("+Run checks.")); component.handleInput("\u001b"); - for (let index = 0; index < 3; index += 1) component.handleInput("\u001b[B"); + for (let index = 0; index < 4; index += 1) component.handleInput("\u001b[B"); component.handleInput("\n"); await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Retrieval and behavior evaluation"), @@ -847,7 +879,7 @@ describe("EvolutionMenuComponent", () => { expect(output).toContain("Negative regressions: 0"); component.handleInput("\u001b"); - for (let index = 0; index < 4; index += 1) component.handleInput("\u001b[B"); + for (let index = 0; index < 5; index += 1) component.handleInput("\u001b[B"); component.handleInput("\n"); await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Activate focused-verification?"), @@ -964,7 +996,10 @@ describe("EvolutionMenuComponent", () => { expect(explainEvolutionCandidateReason("custom-raw-code")).toBe("custom raw code."); }); - it("formats create, update, and retirement draft diffs", () => { + it("formats a readable candidate outcome and create, update, and retirement draft diffs", () => { + expect(formatEvolutionSkillCandidateOutcome(skillCandidateDetail())).toBe( + "Run the focused check. Inspect the observed result. (the check fails)", + ); const createDiff = formatEvolutionSkillDraftDiff(skillCandidateDetail()); expect(createDiff).toContain("+Run checks."); expect(createDiff).toContain("+# Supporting file: scripts/check.sh"); diff --git a/packages/evolution/src/control-service.ts b/packages/evolution/src/control-service.ts index 581ea191a..5a480753b 100644 --- a/packages/evolution/src/control-service.ts +++ b/packages/evolution/src/control-service.ts @@ -154,6 +154,8 @@ export interface EvolutionSkillCandidateExistingContent { export interface EvolutionSkillCandidateDetail { readonly summary: EvolutionSkillCandidateSummary; readonly candidate: SkillCandidate; + readonly supportingEvidence: readonly MaterializedEvidence[]; + readonly contradictingEvidence: readonly MaterializedEvidence[]; readonly sourceEvolutionJobIds: readonly string[]; readonly proposalRevisionIds: readonly string[]; readonly existingContents: readonly EvolutionSkillCandidateExistingContent[]; @@ -299,6 +301,19 @@ function isSkillEvaluation(artifact: EvolutionArtifact): artifact is SkillEvalua return "artifactKind" in artifact && artifact.artifactKind === "skill-evaluation"; } +function resolveEvidence(input: EvolutionInputSnapshot, evidenceIds: readonly string[]): MaterializedEvidence[] { + const evidence = new Map( + input.evidenceSources.flatMap((source) => + source.invocation.evidence.map((item) => [item.evidenceId, item] as const), + ), + ); + return evidenceIds.map((evidenceId) => { + const item = evidence.get(evidenceId); + if (item === undefined) throw new Error(`Evolution evidence "${evidenceId}" is unavailable`); + return item; + }); +} + function stringArray(value: unknown): readonly string[] { return Array.isArray(value) && value.every((entry) => typeof entry === "string") ? value : []; } @@ -1007,6 +1022,8 @@ export class EvolutionControlService { return { summary: candidateSummary(candidate, group), candidate, + supportingEvidence: resolveEvidence(input, candidate.supportingEvidenceIds), + contradictingEvidence: resolveEvidence(input, candidate.contradictingEvidenceIds), sourceEvolutionJobIds: group?.proposalRevisions.map((revision) => revision.sourceEvolutionJobId) ?? [jobId], proposalRevisionIds: group?.proposalRevisions.map((revision) => revision.revisionId) ?? [], existingContents, @@ -1076,23 +1093,12 @@ export class EvolutionControlService { if (proposal === undefined) throw new Error(`Unknown evolution proposal "${proposalId}"`); const input = artifacts.find(isInputSnapshot); if (input === undefined) throw new Error(`Evolution job "${jobId}" has no input snapshot`); - const evidence = new Map( - input.evidenceSources.flatMap((source) => - source.invocation.evidence.map((item) => [item.evidenceId, item] as const), - ), - ); - const resolveEvidence = (evidenceIds: readonly string[]): MaterializedEvidence[] => - evidenceIds.map((evidenceId) => { - const item = evidence.get(evidenceId); - if (item === undefined) throw new Error(`Evolution evidence "${evidenceId}" is unavailable`); - return item; - }); const validation = artifacts.find(isValidationReport); const activation = harnessActivation(events); return { proposal, - supportingEvidence: resolveEvidence(proposal.supportingEvidenceIds), - contradictingEvidence: resolveEvidence(proposal.contradictingEvidenceIds), + supportingEvidence: resolveEvidence(input, proposal.supportingEvidenceIds), + contradictingEvidence: resolveEvidence(input, proposal.contradictingEvidenceIds), score, validationDisposition: validation?.proposalResults.find((result) => result.proposalId === proposalId)?.disposition ?? null, diff --git a/packages/evolution/test/control-service.test.ts b/packages/evolution/test/control-service.test.ts index 9ad316106..50f9ded17 100644 --- a/packages/evolution/test/control-service.test.ts +++ b/packages/evolution/test/control-service.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { HarnessStateStore, type HarnessVersion, type HarnessVersionList } from "@earendil-works/diffex-harness-state"; -import type { MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import type { MaterializedEvidence, MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; import { afterEach, describe, expect, it, vi } from "vitest"; import { computeEvolutionContentDigest, @@ -668,7 +668,7 @@ describe("evolution control service", () => { }); }); - it("exposes candidate provenance, gates activation, and rolls back through preserving descendants", async () => { + it("exposes candidate evidence, gates activation, and rolls back through preserving descendants", async () => { const inputReference = { artifactId: "input-1", contentDigest: `sha256:${"1".repeat(64)}` }; const candidateReference = { artifactId: "candidate-1", contentDigest: `sha256:${"2".repeat(64)}` }; const draftReference = { artifactId: "draft-1", contentDigest: `sha256:${"3".repeat(64)}` }; @@ -737,9 +737,26 @@ describe("evolution control service", () => { validation: validationReference, evaluation: evaluationReference, } as unknown as SkillCandidateGroup; + const supportingEvidence = { + evidenceId: "evidence-1", + kind: "session_item", + itemType: "message", + order: { traceSequence: 1, timestamp: null, causalIdentifiers: [], ambiguity: [] }, + traceSources: [], + payload: { message: { content: [{ type: "text", text: "The focused verification passed." }] } }, + } as MaterializedEvidence; const input = { ingestionHighWaterCursor: 1, - evidenceSources: [], + evidenceSources: [ + { + sourceId: "source-1", + materializationId: "materialization-1", + ingestionCursor: 1, + origin: "product-runtime", + contentDigest: computeEvolutionContentDigest("source-1"), + invocation: { ...invocation(1), evidence: [supportingEvidence] }, + }, + ], harness: { baseVersion: null }, expectedHarnessBaseVersion: `sha256:${"a".repeat(64)}`, skillCatalog: [], @@ -876,6 +893,8 @@ describe("evolution control service", () => { }, sourceEvolutionJobIds: [job.jobId], proposalRevisionIds: ["revision-1"], + supportingEvidence: [supportingEvidence], + contradictingEvidence: [], draft: { draftId: "draft-1" }, validation: { disposition: "accepted" }, evaluation: { disposition: "accepted" }, From 045b668742400a5238afcff8e629d1b055c12723 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 30 Aug 2026 14:34:26 -0400 Subject: [PATCH 10/10] fix(coding-agent): simplify evolution job details --- .../interactive/components/evolution-menu.ts | 19 ------------------- .../coding-agent/test/evolution-menu.test.ts | 18 +++++++++--------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts index 41f06a843..a108cc916 100644 --- a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts +++ b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts @@ -670,19 +670,10 @@ export class EvolutionMenuComponent extends Container implements Focusable { } #renderJob(job: EvolutionJobDetail): void { - const candidateStage = - job.candidateGeneration.stage === null || job.candidateGeneration.stage === "completed" - ? "" - : `, ${job.candidateGeneration.stage}`; - const candidateAttempt = - job.candidateGeneration.attempt === 0 ? "" : `, attempt ${job.candidateGeneration.attempt}`; for (const line of [ `Trigger: ${job.trigger}`, `Result: ${formatEvolutionJobResult(job)}`, `Analysis: ${job.analysisStatus}`, - `Candidate generation: ${job.candidateGeneration.status.replaceAll("-", " ")}${candidateStage}${candidateAttempt}`, - `Analysis finished: ${formatEvolutionTimestamp(job.terminalAt)}`, - `Candidate updated: ${formatEvolutionTimestamp(job.candidateGeneration.updatedAt)}`, ...(job.candidateGeneration.retryNotBefore === null ? [] : [`Candidate retry: ${formatEvolutionTimestamp(job.candidateGeneration.retryNotBefore)}`]), @@ -697,16 +688,6 @@ export class EvolutionMenuComponent extends Container implements Focusable { this.#overview?.currentHarnessVersion, job.baseHarnessInfo, )}`, - `Target harness: ${ - job.activatedHarnessVersion === null - ? "not activated" - : formatHarnessVersionReference( - job.activatedHarnessVersion, - this.#harnessVersionInfo, - this.#overview?.currentHarnessVersion, - ) - }`, - `Rollback available: ${job.observability.rollbackAvailable ? "yes" : "no"}`, ]) { this.#addLine(line); } diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index 89c0ddfa7..5e0991ab8 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -558,10 +558,10 @@ describe("EvolutionMenuComponent", () => { await vi.waitFor(() => expect(fixture.jobDetailCall.mock.calls.length).toBeGreaterThanOrEqual(2)); const output = stripAnsi(component.render(180).join("\n")); - expect(output).toContain("Analysis finished: Today at 1:25 pm"); expect(output).toContain("Result: analysis failed"); expect(output).toContain("Analysis: failed"); for (const removedLabel of [ + "Analysis finished:", "Proposals:", "Reflector budget:", "Coverage:", @@ -599,14 +599,15 @@ describe("EvolutionMenuComponent", () => { expect(output).toContain("Trigger: manual"); expect(output).toContain("Result: analysis processing"); expect(output).toContain("Analysis: processing"); - expect(output).toContain("Candidate generation: not applicable"); - expect(output).toContain("Analysis finished: never"); expect(output).toContain("Evolution token: 12,345"); expect(output).toContain("Last checkpoint: Today at 9:05 am"); expect(output).toContain("Base harness: [current] Default Diffex Agent"); - expect(output).toContain("Target harness: not activated"); - expect(output).toContain("Rollback available: no"); for (const removedLabel of [ + "Candidate generation:", + "Candidate updated:", + "Analysis finished:", + "Target harness:", + "Rollback available:", "Job:", "Evidence high-water cursor:", "Slices:", @@ -672,7 +673,7 @@ describe("EvolutionMenuComponent", () => { const output = stripAnsi(component.render(180).join("\n")); expect(output).toContain("Result: partial failure"); expect(output).toContain("Analysis: completed"); - expect(output).toContain("Candidate generation: partial failure, synthesizing, attempt 3"); + expect(output).not.toContain("Candidate generation:"); expect(output).toContain("Candidate error: Skill synthesis failed for candidate group group-1"); component.dispose(); }); @@ -722,9 +723,8 @@ describe("EvolutionMenuComponent", () => { ); component.handleInput("\n"); await vi.waitFor(() => expect(fixture.jobDetailCall.mock.calls.length).toBeGreaterThanOrEqual(2)); - await vi.waitFor(() => - expect(stripAnsi(component.render(180).join("\n"))).toContain("Candidate generation: completed"), - ); + await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Result: full success")); + expect(stripAnsi(component.render(180).join("\n"))).not.toContain("Candidate generation:"); component.dispose(); });