From 681dc086b91ebe1b9b12bde44bf7f998b0f3006b Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Fri, 28 Aug 2026 11:19:23 -0400 Subject: [PATCH 1/3] feat(agent): record curator procedural information loss --- packages/evolution/src/contracts.ts | 23 +++ packages/evolution/src/index.ts | 3 + packages/evolution/src/pipeline.ts | 13 +- packages/evolution/src/ports.ts | 3 + packages/evolution/src/skill-curator-tools.ts | 6 + packages/evolution/src/skill-curator.ts | 134 +++++++++++++++++- packages/evolution/test/control-plane.test.ts | 7 +- .../evolution/test/proposal-ranking.test.ts | 2 +- packages/evolution/test/skill-curator.test.ts | 46 ++++++ 9 files changed, 230 insertions(+), 7 deletions(-) diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index defbe56d4..b18b11989 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -423,6 +423,29 @@ export interface Lesson { readonly actionableTakeaway: string; } +export type ProceduralInformationLossKind = + | "action-order" + | "decision-branch" + | "failure-signal" + | "mitigation-or-recovery" + | "tool-or-environment-dependency" + | "verification-action"; + +export interface ProceduralInformationLoss { + readonly missing: readonly ProceduralInformationLossKind[]; + readonly supportingEvidenceIds: readonly string[]; + readonly explanation: string; +} + +export interface ProceduralInformationLossFinding { + readonly artifactKind: "procedural-information-loss"; + readonly metadata: EvolutionArtifactMetadata; + readonly findingId: string; + readonly detectedBy: "skill-curator"; + readonly sourceLessonIds: readonly string[]; + readonly loss: ProceduralInformationLoss; +} + export type SkillCandidateOperation = "CREATE" | "MERGE" | "RETIRE" | "UPDATE"; export type SkillabilityClassification = diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index fb0448589..2f0adf1ea 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -104,6 +104,9 @@ export { type MutationWriteResult, type MutationWrittenLedgerEvent, type OrchestrationSourceTarget, + type ProceduralInformationLoss, + type ProceduralInformationLossFinding, + type ProceduralInformationLossKind, type ProposalDispositionLedgerEvent, type ProposalRolledBackLedgerEvent, type ProposalValidationResult, diff --git a/packages/evolution/src/pipeline.ts b/packages/evolution/src/pipeline.ts index a21f49fa0..22b111148 100644 --- a/packages/evolution/src/pipeline.ts +++ b/packages/evolution/src/pipeline.ts @@ -704,11 +704,22 @@ export class EvolutionPipeline implements EvolutionStageExecutor { curation.deferredFindings.map((finding) => finding.findingId), "deferred skill finding ID", ); + assertUniqueIds( + curation.proceduralInformationLossFindings.map((finding) => finding.findingId), + "procedural information loss finding ID", + ); return { - artifacts: [...curation.candidates, ...curation.deferredFindings], + artifacts: [ + ...curation.candidates, + ...curation.deferredFindings, + ...curation.proceduralInformationLossFindings, + ], state: { candidateIds: curation.candidates.map((candidate) => candidate.candidateId), deferredFindingIds: curation.deferredFindings.map((finding) => finding.findingId), + proceduralInformationLossFindingIds: curation.proceduralInformationLossFindings.map( + (finding) => finding.findingId, + ), selectedMemoryProposalIds: selectedMemoryProposals.map((scored) => scored.proposal.proposalId), }, directive: selectedMemoryProposals.length === 0 ? "complete-no-op" : "continue", diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index b16918777..d4fc846d1 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -31,6 +31,7 @@ import type { Lesson, MutationBatch, MutationWriteReceipt, + ProceduralInformationLossFinding, ReflectionNode, ScoredProposal, SkillCandidate, @@ -54,6 +55,7 @@ export type EvolutionArtifact = | InstructionSnapshot | Lesson | MutationBatch + | ProceduralInformationLossFinding | MutationWriteReceipt | ReflectionNode | ScoredProposal @@ -248,6 +250,7 @@ export interface SkillCuratorInput { export interface SkillCurationResult { readonly candidates: readonly SkillCandidate[]; readonly deferredFindings: readonly DeferredSkillCandidateFinding[]; + readonly proceduralInformationLossFindings: readonly ProceduralInformationLossFinding[]; } export interface SkillCurator { diff --git a/packages/evolution/src/skill-curator-tools.ts b/packages/evolution/src/skill-curator-tools.ts index 3d7a2c75d..24dccca51 100644 --- a/packages/evolution/src/skill-curator-tools.ts +++ b/packages/evolution/src/skill-curator-tools.ts @@ -118,6 +118,7 @@ export class SkillCuratorToolRegistry { readonly #maxToolCalls: number; readonly #maxReferencesPerCall: number; readonly #recoverableDenials: SkillCuratorRecoverableToolDenial[] = []; + readonly #inspectedEvidenceIds = new Set(); readonly #inspectedSkillIds = new Set(); readonly evidence: EvolutionModelReferenceCatalog; readonly skills: EvolutionModelReferenceCatalog; @@ -246,6 +247,10 @@ export class SkillCuratorToolRegistry { return entry.ref; } + wasEvidenceInspected(evidenceId: string): boolean { + return this.#inspectedEvidenceIds.has(evidenceId); + } + wasSkillInspected(skillId: string): boolean { return this.#inspectedSkillIds.has(skillId); } @@ -275,6 +280,7 @@ export class SkillCuratorToolRegistry { const context = [...contextBySource.values()].sort( (left, right) => left.evidence.order.traceSequence - right.evidence.order.traceSequence, ); + for (const binding of selected) this.#inspectedEvidenceIds.add(binding.evidence.evidenceId); return { output: { evidence: context.map((binding) => ({ diff --git a/packages/evolution/src/skill-curator.ts b/packages/evolution/src/skill-curator.ts index 99e8086fa..7f95d7c13 100644 --- a/packages/evolution/src/skill-curator.ts +++ b/packages/evolution/src/skill-curator.ts @@ -5,6 +5,9 @@ import type { EvolutionSkillCatalogEntry, JsonObject, Lesson, + ProceduralInformationLoss, + ProceduralInformationLossFinding, + ProceduralInformationLossKind, ScoredProposal, SkillabilityClassification, SkillCandidate, @@ -48,8 +51,17 @@ const MEMORY_RELATIONSHIPS = new Set([ "supersedes", ]); const OPERATIONS = new Set(["CREATE", "MERGE", "RETIRE", "UPDATE"]); +const PROCEDURAL_INFORMATION_LOSS_KINDS = new Set([ + "action-order", + "decision-branch", + "failure-signal", + "mitigation-or-recovery", + "tool-or-environment-dependency", + "verification-action", +]); interface SkillCuratorSchemaReferences { + readonly evidenceRefs: readonly string[]; readonly lessonRefs: readonly string[]; readonly memoryProposalRefs: readonly string[]; readonly skillRefs: readonly string[]; @@ -80,9 +92,16 @@ interface DeferredFindingDraft { readonly reason: string; } +interface ProceduralInformationLossDraft { + readonly lossKey: string; + readonly lessonIds: readonly string[]; + readonly loss: ProceduralInformationLoss; +} + interface ParsedSkillCuratorOutput { readonly candidates: readonly CandidateDraft[]; readonly deferredFindings: readonly DeferredFindingDraft[]; + readonly proceduralInformationLosses: readonly ProceduralInformationLossDraft[]; } interface EvidenceReferences { @@ -203,6 +222,20 @@ export function createSkillCuratorOutputSchema(references: SkillCuratorSchemaRef skillability, memoryRelationship: relationship, }; + const proceduralInformationLoss = schema( + { + lossKey: { type: "string", minLength: 1 }, + lessonRefs: referenceArraySchema(references.lessonRefs, 1), + missing: { + type: "array", + minItems: 1, + items: { type: "string", enum: [...PROCEDURAL_INFORMATION_LOSS_KINDS] }, + }, + supportingEvidenceRefs: referenceArraySchema(references.evidenceRefs, 1), + explanation: { type: "string", minLength: 1 }, + }, + ["lossKey", "lessonRefs", "missing", "supportingEvidenceRefs", "explanation"], + ); return schema( { decisions: { @@ -248,8 +281,9 @@ export function createSkillCuratorOutputSchema(references: SkillCuratorSchemaRef ], }, }, + proceduralInformationLosses: { type: "array", items: proceduralInformationLoss }, }, - ["decisions"], + ["decisions", "proceduralInformationLosses"], ); } @@ -447,6 +481,53 @@ function existingSkills(value: unknown, registry: SkillCuratorToolRegistry): rea return selected; } +function proceduralInformationLossDraft( + value: unknown, + lessons: EvolutionModelReferenceCatalog, + registry: SkillCuratorToolRegistry, +): ProceduralInformationLossDraft { + if (!isRecord(value)) throw new Error("proceduralInformationLosses entries must be objects"); + assertOnlyFields( + value, + ["lossKey", "lessonRefs", "missing", "supportingEvidenceRefs", "explanation"], + "proceduralInformationLoss", + ); + const lossKey = requiredString(value, "lossKey"); + if (!DECISION_KEY_PATTERN.test(lossKey)) throw new Error(`Invalid procedural information loss key "${lossKey}"`); + const selectedLessons = referenceArray(value.lessonRefs, "lessonRefs", lessons, { allowEmpty: false }); + const missing = stringArray(value.missing, "missing", false); + for (const item of missing) { + if (!PROCEDURAL_INFORMATION_LOSS_KINDS.has(item as ProceduralInformationLossKind)) { + throw new Error(`Unsupported missing procedural information kind "${item}"`); + } + } + const evidence = referenceArray(value.supportingEvidenceRefs, "supportingEvidenceRefs", registry.evidence, { + allowEmpty: false, + }); + const lessonEvidenceIds = new Set( + selectedLessons.flatMap((lesson) => [...lesson.evidenceIds, ...lesson.counterevidenceIds]), + ); + for (const binding of evidence) { + if (!lessonEvidenceIds.has(binding.evidence.evidenceId)) { + throw new Error( + `Procedural information loss evidence "${binding.evidence.evidenceId}" is not cited by its lessons`, + ); + } + if (!registry.wasEvidenceInspected(binding.evidence.evidenceId)) { + throw new Error(`Procedural information loss evidence "${binding.evidence.evidenceId}" must be inspected`); + } + } + return { + lossKey, + lessonIds: selectedLessons.map((lesson) => lesson.lessonId), + loss: { + missing: missing as readonly ProceduralInformationLossKind[], + supportingEvidenceIds: evidence.map((binding) => binding.evidence.evidenceId), + explanation: requiredString(value, "explanation"), + }, + }; +} + function candidateDraft( decisionKey: string, lessonIds: readonly string[], @@ -522,8 +603,11 @@ function parseOutput( proposals: EvolutionModelReferenceCatalog, registry: SkillCuratorToolRegistry, ): ParsedSkillCuratorOutput { - assertOnlyFields(output, ["decisions"], "Skill Curator response"); + assertOnlyFields(output, ["decisions", "proceduralInformationLosses"], "Skill Curator response"); if (!Array.isArray(output.decisions)) throw new Error("Skill Curator decisions must be an array"); + if (!Array.isArray(output.proceduralInformationLosses)) { + throw new Error("Skill Curator proceduralInformationLosses must be an array"); + } const seenKeys = new Set(); const seenCandidateNames = new Set(); const candidates: CandidateDraft[] = []; @@ -586,7 +670,16 @@ function parseOutput( seenCandidateNames.add(normalizedName); candidates.push(draft); } - return { candidates, deferredFindings }; + const seenLossKeys = new Set(); + const proceduralInformationLosses = output.proceduralInformationLosses.map((value) => { + const loss = proceduralInformationLossDraft(value, lessons, registry); + if (seenLossKeys.has(loss.lossKey)) { + throw new Error(`Duplicate procedural information loss key "${loss.lossKey}"`); + } + seenLossKeys.add(loss.lossKey); + return loss; + }); + return { candidates, deferredFindings, proceduralInformationLosses }; } function taskLineageIds(evidence: readonly MaterializedEvidence[]): readonly string[] { @@ -749,7 +842,33 @@ function resultsFromDrafts( ...content, }; }); - return { candidates, deferredFindings }; + const proceduralInformationLossFindings = parsed.proceduralInformationLosses.map( + (draft): ProceduralInformationLossFinding => { + const content = { + artifactKind: "procedural-information-loss" as const, + findingId: stableId("procedural-information-loss", { + idempotencyKey: input.idempotencyKey, + lossKey: draft.lossKey, + draft, + }), + detectedBy: "skill-curator" as const, + sourceLessonIds: draft.lessonIds, + loss: draft.loss, + }; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: content.findingId, + producer, + sourceIds: unique([...content.sourceLessonIds, ...content.loss.supportingEvidenceIds]), + scope: input.input.metadata.scope, + createdAt: input.input.metadata.createdAt, + content, + }), + ...content, + }; + }, + ); + return { candidates, deferredFindings, proceduralInformationLossFindings }; } function modelInput( @@ -783,6 +902,12 @@ function modelInput( workspaceTarget: jsonObject(workspace(input)), globalEvolvedSkillTargetPermitted: false, proceduralObservations: [], + reportProceduralInformationLoss: { + when: "A lesson preserves a conclusion but omits material procedural information that cited source evidence contains, preventing or weakening a candidate.", + requireSourceEvidenceInspection: true, + missingKinds: [...PROCEDURAL_INFORMATION_LOSS_KINDS], + doNotReportAbsentSourceInformation: true, + }, ...(responseCorrection === null ? {} : { @@ -885,6 +1010,7 @@ export class StructuredSkillCurator implements SkillCurator { : { maxReferencesPerCall: this.#maxReferencesPerToolCall }), }); const outputSchema = createSkillCuratorOutputSchema({ + evidenceRefs: registry.evidence.references, lessonRefs: lessonCatalog.references, memoryProposalRefs: proposalCatalog.references, skillRefs: registry.skills.references, diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index b049d3dce..78089791e 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -888,7 +888,11 @@ describe("checkpointed evolution pipeline", () => { metadata: metadata("skill-candidate-1", ["lesson-1", "proposal-1"], candidateContent), ...candidateContent, }; - return { candidates: [candidate], deferredFindings: [] }; + return { + candidates: [candidate], + deferredFindings: [], + proceduralInformationLossFindings: [], + }; }, }, validator: { @@ -958,6 +962,7 @@ describe("checkpointed evolution pipeline", () => { state: { candidateIds: ["skill-candidate-1"], deferredFindingIds: [], + proceduralInformationLossFindingIds: [], selectedMemoryProposalIds: ["proposal-1"], }, }); diff --git a/packages/evolution/test/proposal-ranking.test.ts b/packages/evolution/test/proposal-ranking.test.ts index 0df3f8a35..72c35d032 100644 --- a/packages/evolution/test/proposal-ranking.test.ts +++ b/packages/evolution/test/proposal-ranking.test.ts @@ -722,7 +722,7 @@ describe("deterministic proposal evidence ranking", () => { selectedMemoryProposalIds.push( input.selectedMemoryProposals.map((scored) => scored.proposal.proposalId), ); - return { candidates: [], deferredFindings: [] }; + return { candidates: [], deferredFindings: [], proceduralInformationLossFindings: [] }; }, }, validator: ranker, diff --git a/packages/evolution/test/skill-curator.test.ts b/packages/evolution/test/skill-curator.test.ts index 83a74b72c..ad0d8a924 100644 --- a/packages/evolution/test/skill-curator.test.ts +++ b/packages/evolution/test/skill-curator.test.ts @@ -399,6 +399,7 @@ function candidateOutput(): JsonObject { deferredReason: null, }, ], + proceduralInformationLosses: [], }; } @@ -466,6 +467,7 @@ describe("StructuredSkillCurator", () => { skillCatalog: [{ skillRef: "K1", skillId: "installed:workspace:existing-skill" }], }); expect(result.deferredFindings).toEqual([]); + expect(result.proceduralInformationLossFindings).toEqual([]); expect(result.candidates).toHaveLength(1); expect(result.candidates[0]).toMatchObject({ artifactKind: "skill-candidate", @@ -518,6 +520,7 @@ describe("StructuredSkillCurator", () => { deferredReason: "No distinct procedural content remains for a skill.", }, ], + proceduralInformationLosses: [], }; request.validateOutput?.(output); return { output, model: { provider: "faux", model: "fixture" } }; @@ -536,6 +539,48 @@ describe("StructuredSkillCurator", () => { ]); }); + it("persists evidence-grounded procedural information loss without emitting a candidate", async () => { + const storedSkill = skillArtifact(); + const artifacts = new FixtureArtifactStore([storedSkill]); + const curator = new StructuredSkillCurator({ version: "1", artifacts }); + const input = fixtureInput(storedSkill.reference); + const result = await curator.curate( + curatorInput(async (request) => { + const evidenceTool = request.tools.find((tool) => tool.name === "inspect_source_evidence"); + await evidenceTool?.execute({ evidenceRefs: ["E1"], contextAfter: 1 }); + const output: JsonObject = { + decisions: [], + proceduralInformationLosses: [ + { + lossKey: "missing-recovery-order", + lessonRefs: ["L1"], + missing: ["action-order", "mitigation-or-recovery"], + supportingEvidenceRefs: ["E1"], + explanation: + "The evidence contains the recovery order, but the final lesson retains only the conclusion.", + }, + ], + }; + request.validateOutput?.(output); + return { output, model: { provider: "faux", model: "fixture" } }; + }, input), + ); + + expect(result.candidates).toEqual([]); + expect(result.proceduralInformationLossFindings).toMatchObject([ + { + artifactKind: "procedural-information-loss", + detectedBy: "skill-curator", + sourceLessonIds: ["lesson-final"], + loss: { + missing: ["action-order", "mitigation-or-recovery"], + supportingEvidenceIds: ["support-1"], + }, + }, + ]); + expect(result.proceduralInformationLossFindings[0]?.metadata.sourceIds).toEqual(["lesson-final", "support-1"]); + }); + it("rejects references to skill content that was not inspected", async () => { const storedSkill = skillArtifact(); const artifacts = new FixtureArtifactStore([storedSkill]); @@ -558,6 +603,7 @@ describe("StructuredSkillCurator", () => { it("uses a strict Codex-compatible output schema", () => { const schema = createSkillCuratorOutputSchema({ + evidenceRefs: ["E1"], lessonRefs: ["L1"], memoryProposalRefs: ["P1"], skillRefs: ["K1"], From 059654b19b3446b203354ea88e5ee9d68fd4a84f Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Fri, 28 Aug 2026 11:21:34 -0400 Subject: [PATCH 2/3] feat(agent): detect procedural loss in skill drafts --- .../evolution/src/skill-draft-validator.ts | 46 +++++++++ packages/evolution/src/skill-synthesizer.ts | 8 ++ .../test/skill-draft-validator.test.ts | 95 ++++++++++++++++++- .../evolution/test/skill-synthesizer.test.ts | 8 ++ 4 files changed, 155 insertions(+), 2 deletions(-) diff --git a/packages/evolution/src/skill-draft-validator.ts b/packages/evolution/src/skill-draft-validator.ts index 8f6015a7c..2af33abc2 100644 --- a/packages/evolution/src/skill-draft-validator.ts +++ b/packages/evolution/src/skill-draft-validator.ts @@ -380,6 +380,52 @@ export class DeterministicSkillDraftValidator implements SkillDraftValidator { } } const normalizedContent = normalizedText(input.draft.content); + for (const trigger of [latest.candidate.trigger.description, ...latest.candidate.trigger.positiveExamples]) { + if (!normalizedContent.includes(normalizedText(trigger))) { + add("trigger-condition-missing", `The draft omits trigger condition "${trigger}"`); + } + } + for (const precondition of latest.candidate.preconditions) { + if (!normalizedContent.includes(normalizedText(precondition))) { + add("precondition-missing", `The draft omits precondition "${precondition}"`); + } + } + let previousActionIndex = -1; + for (const step of latest.candidate.procedure.steps) { + const actionIndex = normalizedContent.indexOf(normalizedText(step.action)); + if (actionIndex === -1) { + add("procedure-action-missing", `The draft omits procedure action "${step.action}"`); + } else { + if (actionIndex <= previousActionIndex) { + add("procedure-action-order-invalid", "The draft changes the candidate procedure action order"); + } + previousActionIndex = Math.max(previousActionIndex, actionIndex); + } + if (step.condition !== null && !normalizedContent.includes(normalizedText(step.condition))) { + add("decision-branch-missing", `The draft omits procedure condition "${step.condition}"`); + } + } + for (const tool of latest.candidate.procedure.requiredTools) { + if (!normalizedContent.includes(normalizedText(tool))) { + add("tool-dependency-missing", `The draft omits required tool "${tool}"`); + } + } + for (const requirement of latest.candidate.procedure.environment) { + if (!normalizedContent.includes(normalizedText(requirement))) { + add("environment-dependency-missing", `The draft omits environment requirement "${requirement}"`); + } + } + for (const handling of latest.candidate.failureHandling) { + if (!normalizedContent.includes(normalizedText(handling.signal))) { + add("failure-signal-missing", `The draft omits failure signal "${handling.signal}"`); + } + if (!normalizedContent.includes(normalizedText(handling.mitigation))) { + add("failure-mitigation-missing", `The draft omits failure mitigation "${handling.mitigation}"`); + } + if (!normalizedContent.includes(normalizedText(handling.recovery))) { + add("recovery-action-missing", `The draft omits recovery action "${handling.recovery}"`); + } + } for (const boundary of latest.candidate.applicabilityBoundaries) { if (!normalizedContent.includes(normalizedText(boundary))) { add("applicability-boundary-missing", `The draft omits applicability boundary "${boundary}"`); diff --git a/packages/evolution/src/skill-synthesizer.ts b/packages/evolution/src/skill-synthesizer.ts index f195bd171..6bb3f8166 100644 --- a/packages/evolution/src/skill-synthesizer.ts +++ b/packages/evolution/src/skill-synthesizer.ts @@ -272,7 +272,15 @@ function modelInput( removeTaskSpecificPathsSecretsAndTransientValues: true, removeUnsupportedGeneralizations: true, retainExplicitCompatibilityRequirements: true, + retainCandidateProceduralContractVerbatim: true, retainCandidateApplicabilityBoundariesNegativeTriggersAndVerificationContractVerbatim: true, + proceduralInformationLossValidation: [ + "trigger conditions and preconditions", + "ordered actions and decision branches", + "tool and environment dependencies", + "failure signals, mitigations, and recovery actions", + "verification actions and observable completion conditions", + ], requiredContent: [ "Agent Skills frontmatter with a specific name and description", "recognizable use conditions", diff --git a/packages/evolution/test/skill-draft-validator.test.ts b/packages/evolution/test/skill-draft-validator.test.ts index 992ee77a6..35a36df6c 100644 --- a/packages/evolution/test/skill-draft-validator.test.ts +++ b/packages/evolution/test/skill-draft-validator.test.ts @@ -180,7 +180,14 @@ function candidate(input: EvolutionInputSnapshot): SkillCandidate { }, preconditions: ["A focused test command exists."], procedure: { - steps: [{ sequence: 1, action: "Run the focused test.", condition: null }], + steps: [ + { + sequence: 1, + action: "Run the focused test.", + condition: "A focused repository change is ready for verification.", + }, + { sequence: 2, action: "Record the focused test result.", condition: null }, + ], requiredTools: ["shell"], environment: ["Repository checkout"], }, @@ -278,9 +285,23 @@ name: focused-verification description: Verify focused repository changes with a deterministic test and recovery routine. --- +Use when A focused repository change needs verification. +For example: A source file changed. Use only for Repositories with a focused test command. Do not use when No repository files changed. -Run the focused test and recover from deterministic failures. + +Precondition: A focused test command exists. +Tool: shell. +Environment: Repository checkout. + +Condition: A focused repository change is ready for verification. +1. Run the focused test. +2. Record the focused test result. + +Failure signal: The test fails. +Mitigation: Inspect the diagnostic. +Recovery: Correct the issue and rerun the test. + Verify that The focused test exits successfully. Finish only when The focused test passes. `; @@ -460,6 +481,76 @@ describe("DeterministicSkillDraftValidator", () => { }), code: "applicability-boundary-missing", }, + { + name: "omitted trigger conditions", + fixture: () => + fixture({ + content: validMarkdown.replace("A focused repository change needs verification.", "Use this workflow."), + }), + code: "trigger-condition-missing", + }, + { + name: "omitted preconditions", + fixture: () => + fixture({ content: validMarkdown.replace("A focused test command exists.", "Prepare the project.") }), + code: "precondition-missing", + }, + { + name: "omitted procedure actions", + fixture: () => fixture({ content: validMarkdown.replace("Run the focused test.", "Run a check.") }), + code: "procedure-action-missing", + }, + { + name: "changed procedure action order", + fixture: () => + fixture({ + content: validMarkdown.replace( + "1. Run the focused test.\n2. Record the focused test result.", + "1. Record the focused test result.\n2. Run the focused test.", + ), + }), + code: "procedure-action-order-invalid", + }, + { + name: "omitted decision branches", + fixture: () => + fixture({ + content: validMarkdown.replace( + "A focused repository change is ready for verification.", + "The project is available.", + ), + }), + code: "decision-branch-missing", + }, + { + name: "omitted tool dependencies", + fixture: () => fixture({ content: validMarkdown.replace("Tool: shell.", "Use the available runner.") }), + code: "tool-dependency-missing", + }, + { + name: "omitted environment dependencies", + fixture: () => + fixture({ + content: validMarkdown.replace("Environment: Repository checkout.", "Prepare the environment."), + }), + code: "environment-dependency-missing", + }, + { + name: "omitted failure signals", + fixture: () => fixture({ content: validMarkdown.replace("The test fails.", "A problem occurs.") }), + code: "failure-signal-missing", + }, + { + name: "omitted failure mitigations", + fixture: () => fixture({ content: validMarkdown.replace("Inspect the diagnostic.", "Investigate.") }), + code: "failure-mitigation-missing", + }, + { + name: "omitted recovery actions", + fixture: () => + fixture({ content: validMarkdown.replace("Correct the issue and rerun the test.", "Recover safely.") }), + code: "recovery-action-missing", + }, ] as const)("rejects $name", async ({ fixture: createFixture, code }) => { const data = createFixture(); const report = await data.validator.validate({ diff --git a/packages/evolution/test/skill-synthesizer.test.ts b/packages/evolution/test/skill-synthesizer.test.ts index a0f26a8cb..5211df783 100644 --- a/packages/evolution/test/skill-synthesizer.test.ts +++ b/packages/evolution/test/skill-synthesizer.test.ts @@ -449,7 +449,15 @@ describe("StructuredSkillSynthesizer", () => { generatedScriptsBinariesDependenciesAndExecutableAssetsPermitted: false, useSuccessfulOutcomesForPrimaryProcedure: true, useFailuresOnlyForPitfallsMitigationsRecoveryAndVerification: true, + retainCandidateProceduralContractVerbatim: true, retainCandidateApplicabilityBoundariesNegativeTriggersAndVerificationContractVerbatim: true, + proceduralInformationLossValidation: [ + "trigger conditions and preconditions", + "ordered actions and decision branches", + "tool and environment dependencies", + "failure signals, mitigations, and recovery actions", + "verification actions and observable completion conditions", + ], }, sourceEvidence: [ { evidenceId: "support-1", relationship: "supporting", terminationStatus: "completed" }, From 8d9635e4abb59b49a7ab068aa19d28e3d039ad24 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Fri, 28 Aug 2026 11:23:58 -0400 Subject: [PATCH 3/3] feat(agent): version procedural loss prompts --- packages/coding-agent/src/core/evolution-runtime.ts | 4 ++-- packages/coding-agent/test/evolution-runtime.test.ts | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 11eaaa2a0..3303b3370 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -770,8 +770,8 @@ class CodingAgentSnapshotSource implements EvolutionSnapshotSource { promptVersions: { reflector: "reflector-v2", curator: "curator-v1", - skillCurator: "skill-curator-v1", - skillSynthesizer: "skill-synthesizer-v1", + skillCurator: "skill-curator-v2", + skillSynthesizer: "skill-synthesizer-v2", validator: "validator-v1", writer: "writer-v1", }, diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index f26631f21..f860659fb 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -1061,9 +1061,10 @@ describe("coding-agent evolution runtime", () => { '"objective":"classify-operational-patterns-and-produce-non-executable-skill-candidates"', ); expect(prompt).not.toContain(traceId); - return fauxAssistantMessage(fauxToolCall("submit_evolution_response", { decisions: [] }), { - stopReason: "toolUse", - }); + return fauxAssistantMessage( + fauxToolCall("submit_evolution_response", { decisions: [], proceduralInformationLosses: [] }), + { stopReason: "toolUse" }, + ); }, ]); const runtime = await createCodingAgentEvolutionRuntime({