From c06b688696329a928cb8a4eed4cb101ddb148728 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Wed, 19 Aug 2026 18:12:33 -0700 Subject: [PATCH 1/2] feat(agent): define structured deltas - Preserves lesson provenance and execution scope across proposal aggregation - Limits executable memory changes to atomic add, remove, and replace deltas --- packages/evolution/src/contracts.ts | 99 ++++++++++++++++++- packages/evolution/src/index.ts | 13 +++ packages/evolution/src/pipeline.ts | 2 +- packages/evolution/test/control-plane.test.ts | 41 +++++++- 4 files changed, 148 insertions(+), 7 deletions(-) diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 985d2722c..c506c51f7 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -364,23 +364,115 @@ export interface OrchestrationSourceTarget { export type EvolutionTarget = EvolutionMemoryTarget | SkillTarget | OrchestrationSourceTarget; +export type EvolutionProposalAction = "ADD" | "DEFER" | "REMOVE" | "REPLACE"; + +export interface EvolutionProposalScope { + readonly repository: EvolutionScope; + readonly materializationIds: readonly string[]; + readonly traceIds: readonly string[]; + readonly sessionIds: readonly string[]; + readonly taskLineageIds: readonly string[]; + readonly runtimeKinds: readonly string[]; + readonly sourceRevisions: readonly string[]; + readonly harnessVersions: readonly HarnessVersion[]; + readonly applicabilityBoundaries: readonly string[]; +} + +export interface ApplicableInstructionReference { + readonly kind: "applicable-instruction"; + readonly path: string; + readonly precedence: number; + readonly contentDigest: string; + readonly locator: string; +} + +export interface EvolutionMemoryReference { + readonly kind: "evolution-memory"; + readonly memoryItemId: string | null; + readonly contentDigest: string; + readonly locator: string; +} + +export type ExistingInstructionReference = ApplicableInstructionReference | EvolutionMemoryReference; + +export type InstructionRelationshipClassification = "conflicts" | "duplicate" | "novel" | "strengthens"; + +export type InstructionStrengtheningKind = "applicability-boundary" | "condition" | "enforcement" | "precision"; + +export interface InstructionStrengthening { + readonly kind: InstructionStrengtheningKind; + readonly description: string; +} + +export interface EvolutionProposalInstructionRelationship { + readonly classification: Extract; + readonly relevantEntries: readonly ExistingInstructionReference[]; + readonly strengthening: InstructionStrengthening | null; +} + +export interface EvolutionMemoryAddDelta { + readonly operation: "ADD"; + readonly destination: { + readonly section: string | null; + readonly anchor: string | null; + }; + readonly content: string; + readonly intendedScope: readonly string[]; +} + +export interface EvolutionMemoryRemoveDelta { + readonly operation: "REMOVE"; + readonly memoryItemId: string; + readonly expectedContentDigest: string; + readonly reason: string; +} + +export interface EvolutionMemoryReplaceDelta { + readonly operation: "REPLACE"; + readonly removal: Omit; + readonly addition: Omit; +} + +export type EvolutionMemoryDelta = + | EvolutionMemoryAddDelta + | EvolutionMemoryRemoveDelta + | EvolutionMemoryReplaceDelta; + interface EvolutionProposalBase { readonly metadata: EvolutionArtifactMetadata; readonly proposalId: string; readonly summary: string; + readonly scope: EvolutionProposalScope; readonly lessonIds: readonly string[]; - readonly evidenceIds: readonly string[]; + readonly supportingEvidenceIds: readonly string[]; + readonly contradictingEvidenceIds: readonly string[]; + readonly supportingTraceIds: readonly string[]; + readonly contradictingTraceIds: readonly string[]; + readonly expectedBenefit: string; + readonly acceptanceCriterion: string; + readonly risk: string; + readonly rollbackDescription: string; + readonly curatorConfidence: number; + readonly uncertainty: string | null; + readonly counterexamples: readonly string[]; + readonly conflictingProposalIds: readonly string[]; + readonly instructionRelationship: EvolutionProposalInstructionRelationship; } export interface EvolutionMemoryProposal extends EvolutionProposalBase { readonly disposition: "actionable"; readonly target: EvolutionMemoryTarget; - readonly executablePayload: JsonObject; + readonly action: Exclude; + readonly exactDelta: EvolutionMemoryDelta; + readonly expectedBaseDigest: string; } export interface DeferredEvolutionProposal extends EvolutionProposalBase { readonly disposition: "deferred"; readonly target: SkillTarget | OrchestrationSourceTarget; + readonly action: "DEFER"; + readonly exactDelta: null; + readonly expectedBaseDigest: null; readonly finding: string; } @@ -420,7 +512,7 @@ export interface EvolutionMutation { readonly mutationId: string; readonly proposalId: string; readonly target: EvolutionMemoryTarget; - readonly payload: JsonObject; + readonly payload: EvolutionMemoryDelta; } export interface MutationBatch { @@ -485,6 +577,7 @@ export type EvolutionFailureReasonCode = | "lease-lost" | "model-call-failed" | "pipeline-invariant" + | "curation-invalid-response" | "reflection-budget-exhausted" | "reflection-invalid-response" | "shutdown" diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 47944306b..643fa9bc9 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -27,6 +27,8 @@ export { type EvidenceSliceEdge, type EvidenceSliceOpenReason, type EvidenceSourceCursor, + type EvolutionMemoryAddDelta, + type EvolutionMemoryDelta, type EvolutionArtifactMetadata, type EvolutionArtifactProducer, type EvolutionArtifactReference, @@ -45,13 +47,19 @@ export { type EvolutionMemoryItem, type EvolutionMemoryItemLedgerMetadata, type EvolutionMemoryProposal, + type EvolutionMemoryRemoveDelta, + type EvolutionMemoryReplaceDelta, + type EvolutionMemoryReference, type EvolutionMemoryTarget, type EvolutionModelIdentity, type EvolutionMutation, type EvolutionProcessedEvidenceCursor, type EvolutionPromptVersions, type EvolutionProposal, + type EvolutionProposalAction, + type EvolutionProposalInstructionRelationship, type EvolutionProposalLedgerDisposition, + type EvolutionProposalScope, type EvolutionRepositorySnapshot, type EvolutionScope, type EvolutionStageCheckpoint, @@ -59,11 +67,15 @@ export { type EvolutionTokenMeasurement, type EvolutionTriggerKind, type ExcludedEvidenceSource, + type ExistingInstructionReference, type HarnessActivationReceipt, type HarnessActivationResult, type ImmutableHarnessSnapshot, type InstructionDocumentSnapshot, + type InstructionRelationshipClassification, type InstructionSnapshot, + type InstructionStrengthening, + type InstructionStrengtheningKind, type JobCompletedLedgerEvent, type JobCreatedLedgerEvent, type JobFailedLedgerEvent, @@ -93,6 +105,7 @@ export { type ValidationIssue, type ValidationReport, type ValidationSeverity, + type ApplicableInstructionReference, } from "./contracts.ts"; export { EvolutionControlPlane, diff --git a/packages/evolution/src/pipeline.ts b/packages/evolution/src/pipeline.ts index 0c4cf0df4..0ba36a831 100644 --- a/packages/evolution/src/pipeline.ts +++ b/packages/evolution/src/pipeline.ts @@ -395,7 +395,7 @@ export class EvolutionPipeline implements EvolutionStageExecutor { mutationId: stableId("mutation", { idempotencyKey: context.idempotencyKey, proposalId: proposal.proposalId }), proposalId: proposal.proposalId, target: proposal.target, - payload: proposal.executablePayload, + payload: proposal.exactDelta, })); const batchId = stableId("mutation-batch", { idempotencyKey: context.idempotencyKey, mutations }); return { diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index 9a3724deb..4e95f51fb 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -448,15 +448,50 @@ describe("checkpointed evolution pipeline", () => { input: {}, signal: input.signal, }); + const exactDelta = { + operation: "ADD" as const, + destination: { section: "Focused guidance", anchor: null }, + content: "Append one focused rule", + intendedScope: ["repository"], + }; const proposal: EvolutionMemoryProposal = { - metadata: metadata("proposal-1", ["lesson-1"], { content: "Append one focused rule" }), + metadata: metadata("proposal-1", ["lesson-1"], exactDelta), proposalId: "proposal-1", summary: "Append one focused rule", + scope: { + repository: scope, + materializationIds: [], + traceIds: [], + sessionIds: [], + taskLineageIds: [], + runtimeKinds: [], + sourceRevisions: [], + harnessVersions: [FIRST_HARNESS_VERSION], + applicabilityBoundaries: [], + }, lessonIds: ["lesson-1"], - evidenceIds: [], + supportingEvidenceIds: [], + contradictingEvidenceIds: [], + supportingTraceIds: [], + contradictingTraceIds: [], + expectedBenefit: "Keep future changes focused", + acceptanceCriterion: "The memory contains the focused rule", + risk: "The rule may be too broad", + rollbackDescription: "Remove the added memory item", + curatorConfidence: 0.9, + uncertainty: null, + counterexamples: [], + conflictingProposalIds: [], + instructionRelationship: { + classification: "novel", + relevantEntries: [], + strengthening: null, + }, disposition: "actionable", target: { kind: "evolution-memory", expectedHarnessVersion: FIRST_HARNESS_VERSION }, - executablePayload: { content: "Append one focused rule" }, + action: "ADD", + exactDelta, + expectedBaseDigest: computeEvolutionContentDigest("# Evolution memory\n"), }; return [proposal]; }, From 7dc3ee16d4e7535d47680375e2d61a28dce02f96 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Wed, 19 Aug 2026 18:36:53 -0700 Subject: [PATCH 2/2] feat(agent): add structured curator - Converts verified lessons into evidence-closed atomic deltas - Suppresses known overlaps and keeps unsupported targets non-executable --- packages/evolution/src/contracts.ts | 7 +- packages/evolution/src/curator.ts | 959 ++++++++++++++++++ packages/evolution/src/index.ts | 14 +- packages/evolution/src/ports.ts | 1 + packages/evolution/test/control-plane.test.ts | 3 + packages/evolution/test/curator.test.ts | 693 +++++++++++++ 6 files changed, 1669 insertions(+), 8 deletions(-) create mode 100644 packages/evolution/src/curator.ts create mode 100644 packages/evolution/test/curator.test.ts diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index c506c51f7..8247c3577 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -368,11 +368,13 @@ export type EvolutionProposalAction = "ADD" | "DEFER" | "REMOVE" | "REPLACE"; export interface EvolutionProposalScope { readonly repository: EvolutionScope; + readonly repositoryRevision: string; readonly materializationIds: readonly string[]; readonly traceIds: readonly string[]; readonly sessionIds: readonly string[]; readonly taskLineageIds: readonly string[]; readonly runtimeKinds: readonly string[]; + readonly runtimeVersions: readonly string[]; readonly sourceRevisions: readonly string[]; readonly harnessVersions: readonly HarnessVersion[]; readonly applicabilityBoundaries: readonly string[]; @@ -433,10 +435,7 @@ export interface EvolutionMemoryReplaceDelta { readonly addition: Omit; } -export type EvolutionMemoryDelta = - | EvolutionMemoryAddDelta - | EvolutionMemoryRemoveDelta - | EvolutionMemoryReplaceDelta; +export type EvolutionMemoryDelta = EvolutionMemoryAddDelta | EvolutionMemoryRemoveDelta | EvolutionMemoryReplaceDelta; interface EvolutionProposalBase { readonly metadata: EvolutionArtifactMetadata; diff --git a/packages/evolution/src/curator.ts b/packages/evolution/src/curator.ts new file mode 100644 index 000000000..56bcf0c20 --- /dev/null +++ b/packages/evolution/src/curator.ts @@ -0,0 +1,959 @@ +import type { HarnessVersion } from "@earendil-works/diffex-harness-state"; +import type { MaterializedEvidence, MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import type { + ApplicableInstructionReference, + DeferredEvolutionProposal, + EvolutionArtifactProducer, + EvolutionMemoryAddDelta, + EvolutionMemoryDelta, + EvolutionMemoryReference, + EvolutionMemoryRemoveDelta, + EvolutionMemoryReplaceDelta, + EvolutionProposal, + EvolutionProposalInstructionRelationship, + EvolutionProposalScope, + EvolutionTarget, + ExistingInstructionReference, + InstructionRelationshipClassification, + InstructionStrengthening, + InstructionStrengtheningKind, + JsonObject, + JsonValue, + Lesson, +} from "./contracts.ts"; +import { EvolutionExecutionError } from "./errors.ts"; +import type { Curator, CuratorInput, EvolutionModelResponse } from "./ports.ts"; +import { + computeEvolutionContentDigest, + createEvolutionArtifactMetadata, + serializeEvolutionValue, +} from "./provenance.ts"; + +const DEFAULT_MAX_RESPONSE_ATTEMPTS = 2; +const PROPOSAL_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const STRENGTHENING_KINDS = new Set([ + "applicability-boundary", + "condition", + "enforcement", + "precision", +]); +const RELATIONSHIPS = new Set([ + "conflicts", + "duplicate", + "novel", + "strengthens", +]); + +export const CURATOR_OUTPUT_SCHEMA = { + $defs: { + stringArray: { + type: "array", + items: { type: "string", minLength: 1 }, + }, + relationshipReference: { + oneOf: [ + { + type: "object", + additionalProperties: false, + required: ["kind", "path", "locator"], + properties: { + kind: { const: "applicable-instruction" }, + path: { type: "string", minLength: 1 }, + locator: { type: "string", minLength: 1 }, + }, + }, + { + type: "object", + additionalProperties: false, + required: ["kind", "memoryItemId", "locator"], + properties: { + kind: { const: "evolution-memory" }, + memoryItemId: { type: ["string", "null"] }, + locator: { type: "string", minLength: 1 }, + }, + }, + ], + }, + relationship: { + type: "object", + additionalProperties: false, + required: ["classification", "relevantEntries", "strengthening"], + properties: { + classification: { enum: ["conflicts", "duplicate", "novel", "strengthens"] }, + relevantEntries: { + type: "array", + items: { $ref: "#/$defs/relationshipReference" }, + }, + strengthening: { + oneOf: [ + { type: "null" }, + { + type: "object", + additionalProperties: false, + required: ["kind", "description"], + properties: { + kind: { enum: ["applicability-boundary", "condition", "enforcement", "precision"] }, + description: { type: "string", minLength: 1 }, + }, + }, + ], + }, + }, + }, + target: { + oneOf: [ + { + type: "object", + additionalProperties: false, + required: ["kind"], + properties: { kind: { const: "evolution-memory" } }, + }, + { + type: "object", + additionalProperties: false, + required: ["kind", "skillName"], + properties: { + kind: { const: "skill" }, + skillName: { type: "string", minLength: 1 }, + }, + }, + { + type: "object", + additionalProperties: false, + required: ["kind", "sourcePath"], + properties: { + kind: { const: "orchestration-source" }, + sourcePath: { type: "string", minLength: 1 }, + }, + }, + ], + }, + destination: { + type: "object", + additionalProperties: false, + required: ["section", "anchor"], + properties: { + section: { type: ["string", "null"] }, + anchor: { type: ["string", "null"] }, + }, + }, + addition: { + type: "object", + additionalProperties: false, + required: ["destination", "content", "intendedScope"], + properties: { + destination: { $ref: "#/$defs/destination" }, + content: { type: "string", minLength: 1 }, + intendedScope: { $ref: "#/$defs/stringArray" }, + }, + }, + removal: { + type: "object", + additionalProperties: false, + required: ["memoryItemId", "reason"], + properties: { + memoryItemId: { type: "string", minLength: 1 }, + reason: { type: "string", minLength: 1 }, + }, + }, + exactDelta: { + oneOf: [ + { + type: "object", + additionalProperties: false, + required: ["operation", "destination", "content", "intendedScope"], + properties: { + operation: { const: "ADD" }, + destination: { $ref: "#/$defs/destination" }, + content: { type: "string", minLength: 1 }, + intendedScope: { $ref: "#/$defs/stringArray" }, + }, + }, + { + type: "object", + additionalProperties: false, + required: ["operation", "memoryItemId", "reason"], + properties: { + operation: { const: "REMOVE" }, + memoryItemId: { type: "string", minLength: 1 }, + reason: { type: "string", minLength: 1 }, + }, + }, + { + type: "object", + additionalProperties: false, + required: ["operation", "removal", "addition"], + properties: { + operation: { const: "REPLACE" }, + removal: { $ref: "#/$defs/removal" }, + addition: { $ref: "#/$defs/addition" }, + }, + }, + { type: "null" }, + ], + }, + proposal: { + type: "object", + additionalProperties: false, + required: [ + "target", + "action", + "summary", + "exactDelta", + "expectedBenefit", + "acceptanceCriterion", + "risk", + "rollbackDescription", + "curatorConfidence", + "uncertainty", + "counterexamples", + "conflictingDecisionKeys", + "finding", + ], + properties: { + target: { $ref: "#/$defs/target" }, + action: { enum: ["ADD", "DEFER", "REMOVE", "REPLACE"] }, + summary: { type: "string", minLength: 1 }, + exactDelta: { $ref: "#/$defs/exactDelta" }, + expectedBenefit: { type: "string", minLength: 1 }, + acceptanceCriterion: { type: "string", minLength: 1 }, + risk: { type: "string", minLength: 1 }, + rollbackDescription: { type: "string", minLength: 1 }, + curatorConfidence: { type: "number", minimum: 0, maximum: 1 }, + uncertainty: { type: ["string", "null"] }, + counterexamples: { $ref: "#/$defs/stringArray" }, + conflictingDecisionKeys: { $ref: "#/$defs/stringArray" }, + finding: { type: ["string", "null"] }, + }, + }, + }, + type: "object", + additionalProperties: false, + required: ["decisions"], + properties: { + decisions: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["decisionKey", "lessonIds", "relationship", "proposal"], + properties: { + decisionKey: { type: "string", minLength: 1 }, + lessonIds: { type: "array", minItems: 1, items: { type: "string", minLength: 1 } }, + relationship: { $ref: "#/$defs/relationship" }, + proposal: { oneOf: [{ $ref: "#/$defs/proposal" }, { type: "null" }] }, + }, + }, + }, + }, +} as const satisfies JsonObject; + +export interface StructuredDeltaCuratorDependencies { + readonly version: string; +} + +export interface StructuredDeltaCuratorOptions { + readonly maxResponseAttempts?: number; +} + +interface ProposalDraft { + readonly decisionKey: string; + readonly lessonIds: readonly string[]; + readonly relationship: EvolutionProposalInstructionRelationship; + readonly target: EvolutionTarget; + readonly action: "ADD" | "DEFER" | "REMOVE" | "REPLACE"; + readonly summary: string; + readonly exactDelta: EvolutionMemoryDelta | null; + readonly expectedBenefit: string; + readonly acceptanceCriterion: string; + readonly risk: string; + readonly rollbackDescription: string; + readonly curatorConfidence: number; + readonly uncertainty: string | null; + readonly counterexamples: readonly string[]; + readonly conflictingDecisionKeys: readonly string[]; + readonly finding: string | null; +} + +interface EvidenceContext { + readonly materializationId: string; + readonly traceId: string | null; + readonly invocation: MaterializedInvocation; + readonly evidence: MaterializedEvidence; +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function unique(values: readonly string[]): string[] { + return [...new Set(values)]; +} + +function stableId(kind: string, value: unknown): string { + return `${kind}-${computeEvolutionContentDigest(value).slice("sha256:".length, "sha256:".length + 32)}`; +} + +function jsonClone(value: unknown): JsonValue { + return JSON.parse(serializeEvolutionValue(value)) as JsonValue; +} + +function assertOnlyFields(record: Readonly>, allowed: readonly string[], name: string): void { + const allowedFields = new Set(allowed); + const unexpected = Object.keys(record).find((field) => !allowedFields.has(field)); + if (unexpected !== undefined) throw new Error(`${name} does not permit field "${unexpected}"`); +} + +function requiredString(record: Readonly>, name: string): string { + const value = record[name]; + if (typeof value !== "string" || value.length === 0) throw new Error(`${name} must be a non-empty string`); + return value; +} + +function nullableString(record: Readonly>, name: string): string | null { + const value = record[name]; + if (value !== null && (typeof value !== "string" || value.length === 0)) { + throw new Error(`${name} must be a non-empty string or null`); + } + return value; +} + +function stringArray(value: unknown, name: string, allowEmpty = true): readonly string[] { + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) { + throw new Error(`${name} must be an array of non-empty strings`); + } + const result = unique(value); + if (!allowEmpty && result.length === 0) throw new Error(`${name} must not be empty`); + return result; +} + +function confidence(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) { + throw new Error("curatorConfidence must be a finite number from 0 through 1"); + } + return value; +} + +function positiveInteger(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback; + if (!Number.isSafeInteger(resolved) || resolved < 1) throw new Error(`${name} must be a positive integer`); + return resolved; +} + +function instructionReference(value: unknown, input: CuratorInput): ExistingInstructionReference { + if (!isRecord(value)) throw new Error("relationship reference must be an object"); + const kind = value.kind; + if (kind === "applicable-instruction") { + assertOnlyFields(value, ["kind", "path", "locator"], "applicable instruction reference"); + const path = requiredString(value, "path"); + const document = input.instructions.documents.find((candidate) => candidate.path === path); + if (document === undefined) throw new Error(`Instruction reference uses unknown path "${path}"`); + const reference: ApplicableInstructionReference = { + kind, + path, + precedence: document.precedence, + contentDigest: document.contentDigest, + locator: requiredString(value, "locator"), + }; + return reference; + } + if (kind === "evolution-memory") { + assertOnlyFields(value, ["kind", "memoryItemId", "locator"], "evolution memory reference"); + const memoryItemId = value.memoryItemId; + if (memoryItemId !== null && (typeof memoryItemId !== "string" || memoryItemId.length === 0)) { + throw new Error("memoryItemId must be a non-empty string or null"); + } + const item = + memoryItemId === null + ? null + : input.input.harness.activeMemoryItems.find((candidate) => candidate.memoryItemId === memoryItemId); + if (memoryItemId !== null && item === undefined) { + throw new Error(`Evolution memory reference uses unknown item "${memoryItemId}"`); + } + const reference: EvolutionMemoryReference = { + kind, + memoryItemId, + contentDigest: item?.contentDigest ?? input.input.harness.memoryContentDigest, + locator: requiredString(value, "locator"), + }; + return reference; + } + throw new Error(`Unsupported relationship reference kind "${String(kind)}"`); +} + +function strengthening(value: unknown): InstructionStrengthening | null { + if (value === null) return null; + if (!isRecord(value)) throw new Error("strengthening must be an object or null"); + assertOnlyFields(value, ["kind", "description"], "strengthening"); + const kind = value.kind; + if (typeof kind !== "string" || !STRENGTHENING_KINDS.has(kind as InstructionStrengtheningKind)) { + throw new Error(`Unsupported strengthening kind "${String(kind)}"`); + } + return { kind: kind as InstructionStrengtheningKind, description: requiredString(value, "description") }; +} + +function relationship( + value: unknown, + input: CuratorInput, +): { + readonly classification: InstructionRelationshipClassification; + readonly relevantEntries: readonly ExistingInstructionReference[]; + readonly strengthening: InstructionStrengthening | null; +} { + if (!isRecord(value)) throw new Error("relationship must be an object"); + assertOnlyFields(value, ["classification", "relevantEntries", "strengthening"], "relationship"); + const classification = value.classification; + if ( + typeof classification !== "string" || + !RELATIONSHIPS.has(classification as InstructionRelationshipClassification) + ) { + throw new Error(`Unsupported instruction relationship "${String(classification)}"`); + } + if (!Array.isArray(value.relevantEntries)) throw new Error("relevantEntries must be an array"); + const relevantEntries = value.relevantEntries.map((entry) => instructionReference(entry, input)); + const parsedStrengthening = strengthening(value.strengthening); + if (classification === "strengthens") { + if (relevantEntries.length === 0) throw new Error("A strengthening must cite an existing entry"); + if (parsedStrengthening === null) throw new Error("A strengthening must describe its concrete delta"); + } else if (parsedStrengthening !== null) { + throw new Error(`A ${classification} relationship cannot include strengthening details`); + } + if ((classification === "duplicate" || classification === "conflicts") && relevantEntries.length === 0) { + throw new Error(`A ${classification} relationship must cite an existing entry`); + } + return { + classification: classification as InstructionRelationshipClassification, + relevantEntries, + strengthening: parsedStrengthening, + }; +} + +function target(value: unknown, input: CuratorInput): EvolutionTarget { + if (!isRecord(value)) throw new Error("target must be an object"); + if (value.kind === "evolution-memory") { + assertOnlyFields(value, ["kind"], "evolution memory target"); + return { kind: value.kind, expectedHarnessVersion: input.input.expectedHarnessBaseVersion }; + } + if (value.kind === "skill") { + assertOnlyFields(value, ["kind", "skillName"], "skill target"); + return { kind: value.kind, skillName: requiredString(value, "skillName") }; + } + if (value.kind === "orchestration-source") { + assertOnlyFields(value, ["kind", "sourcePath"], "orchestration source target"); + return { kind: value.kind, sourcePath: requiredString(value, "sourcePath") }; + } + throw new Error(`Unsupported target kind "${String(value.kind)}"`); +} + +function addition(value: unknown, nested = false): EvolutionMemoryAddDelta { + if (!isRecord(value)) throw new Error("ADD exactDelta must be an object"); + assertOnlyFields( + value, + nested ? ["destination", "content", "intendedScope"] : ["operation", "destination", "content", "intendedScope"], + "ADD exactDelta", + ); + if (!nested && value.operation !== "ADD") throw new Error("ADD exactDelta operation must be ADD"); + if (!isRecord(value.destination)) throw new Error("ADD destination must be an object"); + assertOnlyFields(value.destination, ["section", "anchor"], "ADD destination"); + const section = nullableString(value.destination, "section"); + const anchor = nullableString(value.destination, "anchor"); + if (section === null && anchor === null) throw new Error("ADD must identify a destination section or anchor"); + return { + operation: "ADD", + destination: { section, anchor }, + content: requiredString(value, "content"), + intendedScope: stringArray(value.intendedScope, "intendedScope", false), + }; +} + +function removal(value: unknown, input: CuratorInput, nested: boolean): EvolutionMemoryRemoveDelta { + if (!isRecord(value)) throw new Error("REMOVE exactDelta must be an object"); + assertOnlyFields( + value, + nested ? ["memoryItemId", "reason"] : ["operation", "memoryItemId", "reason"], + "REMOVE exactDelta", + ); + if (!nested && value.operation !== "REMOVE") throw new Error("REMOVE exactDelta operation must be REMOVE"); + const memoryItemId = requiredString(value, "memoryItemId"); + const item = input.input.harness.activeMemoryItems.find((candidate) => candidate.memoryItemId === memoryItemId); + if (item === undefined) throw new Error(`REMOVE uses unknown memory item "${memoryItemId}"`); + return { + operation: "REMOVE", + memoryItemId, + expectedContentDigest: item.contentDigest, + reason: requiredString(value, "reason"), + }; +} + +function replacement(value: unknown, input: CuratorInput): EvolutionMemoryReplaceDelta { + if (!isRecord(value)) throw new Error("REPLACE exactDelta must be an object"); + assertOnlyFields(value, ["operation", "removal", "addition"], "REPLACE exactDelta"); + if (value.operation !== "REPLACE") throw new Error("REPLACE exactDelta operation must be REPLACE"); + const parsedRemoval = removal(value.removal, input, true); + if (!isRecord(value.addition)) throw new Error("REPLACE addition must be an object"); + const parsedAddition = addition(value.addition, true); + const { operation: _removeOperation, ...removalDelta } = parsedRemoval; + const { operation: _addOperation, ...additionDelta } = parsedAddition; + return { operation: "REPLACE", removal: removalDelta, addition: additionDelta }; +} + +function delta(value: unknown, action: ProposalDraft["action"], input: CuratorInput): EvolutionMemoryDelta | null { + if (action === "DEFER") { + if (value !== null) throw new Error("Deferred proposals cannot include exactDelta"); + return null; + } + if (action === "ADD") return addition(value); + if (action === "REMOVE") return removal(value, input, false); + return replacement(value, input); +} + +function proposalDraft( + value: unknown, + decisionKey: string, + lessonIds: readonly string[], + parsedRelationship: EvolutionProposalInstructionRelationship, + input: CuratorInput, +): ProposalDraft { + if (!isRecord(value)) throw new Error("proposal must be an object"); + assertOnlyFields( + value, + [ + "target", + "action", + "summary", + "exactDelta", + "expectedBenefit", + "acceptanceCriterion", + "risk", + "rollbackDescription", + "curatorConfidence", + "uncertainty", + "counterexamples", + "conflictingDecisionKeys", + "finding", + ], + "proposal", + ); + const parsedTarget = target(value.target, input); + const action = value.action; + if (action !== "ADD" && action !== "DEFER" && action !== "REMOVE" && action !== "REPLACE") { + throw new Error(`Unsupported proposal action "${String(action)}"`); + } + if (parsedTarget.kind === "evolution-memory" && action === "DEFER") { + throw new Error("Evolution memory proposals must contain an executable delta"); + } + if (parsedTarget.kind !== "evolution-memory" && action !== "DEFER") { + throw new Error(`Target "${parsedTarget.kind}" must be deferred`); + } + const finding = value.finding; + if (action === "DEFER") { + if (typeof finding !== "string" || finding.length === 0) { + throw new Error("Deferred proposals must include a finding"); + } + } else if (finding !== null) { + throw new Error("Actionable proposal finding must be null"); + } + return { + decisionKey, + lessonIds, + relationship: parsedRelationship, + target: parsedTarget, + action, + summary: requiredString(value, "summary"), + exactDelta: delta(value.exactDelta, action, input), + expectedBenefit: requiredString(value, "expectedBenefit"), + acceptanceCriterion: requiredString(value, "acceptanceCriterion"), + risk: requiredString(value, "risk"), + rollbackDescription: requiredString(value, "rollbackDescription"), + curatorConfidence: confidence(value.curatorConfidence), + uncertainty: nullableString(value, "uncertainty"), + counterexamples: stringArray(value.counterexamples, "counterexamples"), + conflictingDecisionKeys: stringArray(value.conflictingDecisionKeys, "conflictingDecisionKeys"), + finding: typeof finding === "string" ? finding : null, + }; +} + +function finalLessons(input: CuratorInput): readonly Lesson[] { + const lessonsById = new Map(input.reflection.lessons.map((lesson) => [lesson.lessonId, lesson])); + const finalIds = unique(input.reflection.finalLessonIds); + if (finalIds.length !== input.reflection.finalLessonIds.length) { + throw new EvolutionExecutionError("Final reflection lessons contain duplicate IDs", { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } + return finalIds.map((lessonId) => { + const lesson = lessonsById.get(lessonId); + if (lesson === undefined) { + throw new EvolutionExecutionError(`Final reflection lesson "${lessonId}" is missing`, { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } + return lesson; + }); +} + +function evidenceIndex(input: CuratorInput): ReadonlyMap { + const result = new Map(); + for (const source of input.input.evidenceSources) { + const document = input.corpus.documents.find( + (candidate) => + candidate.sourceId === source.sourceId && candidate.materializationId === source.materializationId, + ); + if (document === undefined) { + throw new EvolutionExecutionError(`Corpus document for evidence source "${source.sourceId}" is missing`, { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } + const permittedEvidenceIds = new Set(document.evidenceIds); + for (const evidence of source.invocation.evidence) { + if (!permittedEvidenceIds.has(evidence.evidenceId)) continue; + if (result.has(evidence.evidenceId)) { + throw new EvolutionExecutionError(`Evidence ID "${evidence.evidenceId}" is not unique`, { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } + result.set(evidence.evidenceId, { + materializationId: source.materializationId, + traceId: source.invocation.traceId, + invocation: source.invocation, + evidence, + }); + } + } + return result; +} + +function parseDecisions( + response: EvolutionModelResponse, + input: CuratorInput, + lessons: readonly Lesson[], +): readonly ProposalDraft[] { + assertOnlyFields(response.output, ["decisions"], "Curator response"); + if (!Array.isArray(response.output.decisions)) throw new Error("Curator response decisions must be an array"); + const permittedLessons = new Map(lessons.map((lesson) => [lesson.lessonId, lesson])); + const seenDecisionKeys = new Set(); + const drafts: ProposalDraft[] = []; + for (const value of response.output.decisions) { + if (!isRecord(value)) throw new Error("Curator decision must be an object"); + assertOnlyFields(value, ["decisionKey", "lessonIds", "relationship", "proposal"], "Curator decision"); + const decisionKey = requiredString(value, "decisionKey"); + if (!PROPOSAL_ID_PATTERN.test(decisionKey)) throw new Error(`Invalid decision key "${decisionKey}"`); + if (seenDecisionKeys.has(decisionKey)) throw new Error(`Duplicate decision key "${decisionKey}"`); + seenDecisionKeys.add(decisionKey); + const lessonIds = stringArray(value.lessonIds, "lessonIds", false); + for (const lessonId of lessonIds) { + if (!permittedLessons.has(lessonId)) throw new Error(`Decision cites non-final lesson "${lessonId}"`); + } + const parsedRelationship = relationship(value.relationship, input); + if (parsedRelationship.classification === "duplicate" || parsedRelationship.classification === "conflicts") { + if (value.proposal !== null) { + throw new Error(`${parsedRelationship.classification} decisions must not contain a proposal`); + } + continue; + } + if (value.proposal === null) throw new Error(`${parsedRelationship.classification} decisions require a proposal`); + const actionableRelationship: EvolutionProposalInstructionRelationship = { + classification: parsedRelationship.classification, + relevantEntries: parsedRelationship.relevantEntries, + strengthening: parsedRelationship.strengthening, + }; + drafts.push(proposalDraft(value.proposal, decisionKey, lessonIds, actionableRelationship, input)); + } + const emittedDecisionKeys = new Set(drafts.map((draft) => draft.decisionKey)); + for (const draft of drafts) { + for (const conflictingDecisionKey of draft.conflictingDecisionKeys) { + if (conflictingDecisionKey === draft.decisionKey) { + throw new Error(`Decision "${draft.decisionKey}" cannot conflict with itself`); + } + if (!emittedDecisionKeys.has(conflictingDecisionKey)) { + throw new Error(`Decision "${draft.decisionKey}" cites unknown conflict "${conflictingDecisionKey}"`); + } + } + } + return drafts; +} + +function evidenceReferences( + lessons: readonly Lesson[], + index: ReadonlyMap, +): { + readonly supportingEvidenceIds: readonly string[]; + readonly contradictingEvidenceIds: readonly string[]; + readonly supportingTraceIds: readonly string[]; + readonly contradictingTraceIds: readonly string[]; +} { + const supportingEvidenceIds = unique(lessons.flatMap((lesson) => lesson.evidenceIds)); + const contradictingEvidenceIds = unique(lessons.flatMap((lesson) => lesson.counterevidenceIds)); + if (supportingEvidenceIds.length === 0) throw new Error("A proposal must retain supporting evidence"); + for (const evidenceId of [...supportingEvidenceIds, ...contradictingEvidenceIds]) { + if (!index.has(evidenceId)) throw new Error(`Lesson cites unknown evidence "${evidenceId}"`); + } + const traceIds = (evidenceIds: readonly string[]) => + unique( + evidenceIds + .map((evidenceId) => index.get(evidenceId)?.traceId ?? null) + .filter((traceId): traceId is string => traceId !== null), + ); + return { + supportingEvidenceIds, + contradictingEvidenceIds, + supportingTraceIds: traceIds(supportingEvidenceIds), + contradictingTraceIds: traceIds(contradictingEvidenceIds), + }; +} + +function harnessVersions(invocation: MaterializedInvocation): readonly HarnessVersion[] { + return unique( + invocation.evidence.flatMap((evidence) => + evidence.kind === "trace_record" && evidence.payload.type === "harness_activate" + ? [evidence.payload.harnessVersion] + : [], + ), + ); +} + +function taskLineageIds(evidence: readonly MaterializedEvidence[]): readonly string[] { + return unique( + evidence.flatMap((item) => + item.order.causalIdentifiers + .filter((identifier) => { + const normalized = identifier.name.toLowerCase(); + return normalized.includes("task") && normalized.includes("lineage"); + }) + .map((identifier) => identifier.value), + ), + ); +} + +function proposalScope( + input: CuratorInput, + lessons: readonly Lesson[], + references: ReturnType, + index: ReadonlyMap, +): EvolutionProposalScope { + const evidence = [...references.supportingEvidenceIds, ...references.contradictingEvidenceIds].map((evidenceId) => { + const context = index.get(evidenceId); + if (context === undefined) throw new Error(`Evidence "${evidenceId}" is unavailable`); + return context; + }); + const invocations = [...new Set(evidence.map((context) => context.invocation))]; + return { + repository: input.input.metadata.scope, + repositoryRevision: input.input.repository.revision, + materializationIds: unique(evidence.map((context) => context.materializationId)), + traceIds: unique(evidence.flatMap((context) => (context.traceId === null ? [] : [context.traceId]))), + sessionIds: unique( + evidence.flatMap((context) => + context.evidence.sessionSource === undefined ? [] : [context.evidence.sessionSource.sessionId], + ), + ), + taskLineageIds: taskLineageIds(evidence.map((context) => context.evidence)), + runtimeKinds: unique(invocations.flatMap((invocation) => invocation.producer?.runtime.kind ?? [])), + runtimeVersions: unique(invocations.flatMap((invocation) => invocation.producer?.runtime.version ?? [])), + sourceRevisions: unique(invocations.flatMap((invocation) => invocation.producer?.sourceRevision ?? [])), + harnessVersions: unique(invocations.flatMap(harnessVersions)), + applicabilityBoundaries: unique(lessons.flatMap((lesson) => lesson.applicabilityBoundaries)), + }; +} + +function proposalsFromDrafts( + drafts: readonly ProposalDraft[], + input: CuratorInput, + lessons: readonly Lesson[], + response: EvolutionModelResponse, + version: string, +): readonly EvolutionProposal[] { + const lessonsById = new Map(lessons.map((lesson) => [lesson.lessonId, lesson])); + const evidence = evidenceIndex(input); + const proposalIds = new Map( + drafts.map((draft) => [ + draft.decisionKey, + stableId("proposal", { idempotencyKey: input.idempotencyKey, decisionKey: draft.decisionKey, draft }), + ]), + ); + const producer: EvolutionArtifactProducer = { + name: "diffex-evolution-curator", + version, + promptVersion: input.input.promptVersions.curator, + model: { + provider: response.model.provider, + model: response.model.model, + configurationDigest: input.input.modelConfiguration.configurationDigest, + }, + }; + return drafts.map((draft): EvolutionProposal => { + const selectedLessons = draft.lessonIds.map((lessonId) => { + const lesson = lessonsById.get(lessonId); + if (lesson === undefined) throw new Error(`Lesson "${lessonId}" is unavailable`); + return lesson; + }); + const references = evidenceReferences(selectedLessons, evidence); + const proposalId = proposalIds.get(draft.decisionKey); + if (proposalId === undefined) throw new Error(`Proposal ID for "${draft.decisionKey}" is unavailable`); + const common = { + proposalId, + summary: draft.summary, + scope: proposalScope(input, selectedLessons, references, evidence), + lessonIds: draft.lessonIds, + ...references, + expectedBenefit: draft.expectedBenefit, + acceptanceCriterion: draft.acceptanceCriterion, + risk: draft.risk, + rollbackDescription: draft.rollbackDescription, + curatorConfidence: draft.curatorConfidence, + uncertainty: draft.uncertainty, + counterexamples: draft.counterexamples, + conflictingProposalIds: draft.conflictingDecisionKeys.map((key) => { + const conflictingId = proposalIds.get(key); + if (conflictingId === undefined) throw new Error(`Proposal ID for conflict "${key}" is unavailable`); + return conflictingId; + }), + instructionRelationship: draft.relationship, + }; + const content = { + ...common, + disposition: draft.target.kind === "evolution-memory" ? "actionable" : "deferred", + target: draft.target, + action: draft.action, + exactDelta: draft.exactDelta, + expectedBaseDigest: draft.target.kind === "evolution-memory" ? input.input.harness.memoryContentDigest : null, + ...(draft.finding === null ? {} : { finding: draft.finding }), + }; + const metadata = createEvolutionArtifactMetadata({ + artifactId: proposalId, + producer, + sourceIds: unique([ + ...draft.lessonIds, + ...references.supportingEvidenceIds, + ...references.contradictingEvidenceIds, + ]), + scope: input.input.metadata.scope, + createdAt: input.input.metadata.createdAt, + content, + }); + if (draft.target.kind === "evolution-memory") { + if (draft.action === "DEFER" || draft.exactDelta === null) { + throw new Error("Evolution memory proposal lost its executable delta"); + } + return { + metadata, + ...common, + disposition: "actionable", + target: draft.target, + action: draft.action, + exactDelta: draft.exactDelta, + expectedBaseDigest: input.input.harness.memoryContentDigest, + }; + } + if (draft.action !== "DEFER" || draft.exactDelta !== null || draft.finding === null) { + throw new Error("Deferred proposal contains an executable delta"); + } + const proposal: DeferredEvolutionProposal = { + metadata, + ...common, + disposition: "deferred", + target: draft.target, + action: "DEFER", + exactDelta: null, + expectedBaseDigest: null, + finding: draft.finding, + }; + return proposal; + }); +} + +function modelInput(input: CuratorInput, lessons: readonly Lesson[]): JsonObject { + return { + control: { + objective: "curate-target-neutral-structured-deltas", + returnOnlySchemaConformingOutput: true, + omitDuplicates: true, + omitHigherPriorityConflicts: true, + strengthenOnlyWithConcretePrecision: true, + memoryOperations: ["ADD", "REMOVE", "REPLACE"], + deferredTargets: ["skill", "orchestration-source"], + }, + lessons: jsonClone(lessons), + repository: jsonClone(input.input.repository), + memoryCatalog: { + renderedMemory: input.input.harness.memory, + baseDigest: input.input.harness.memoryContentDigest, + harnessVersion: input.input.harness.harnessVersion, + activeItems: jsonClone(input.input.harness.activeMemoryItems), + }, + instructions: jsonClone(input.instructions.documents), + evidenceScope: input.input.evidenceSources.map((source) => ({ + sourceId: source.sourceId, + materializationId: source.materializationId, + traceId: source.invocation.traceId, + integrity: source.invocation.integrity, + runtimeKind: source.invocation.producer?.runtime.kind ?? null, + sourceRevision: source.invocation.producer?.sourceRevision ?? null, + harnessVersions: harnessVersions(source.invocation), + sessionIds: unique( + source.invocation.evidence.flatMap((evidence) => + evidence.sessionSource === undefined ? [] : [evidence.sessionSource.sessionId], + ), + ), + taskLineageIds: taskLineageIds(source.invocation.evidence), + })), + }; +} + +function abortable(promise: Promise, signal?: AbortSignal): Promise { + if (signal === undefined) return promise; + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + void promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort)); + }); +} + +export class StructuredDeltaCurator implements Curator { + readonly #dependencies: StructuredDeltaCuratorDependencies; + readonly #maxResponseAttempts: number; + + constructor(dependencies: StructuredDeltaCuratorDependencies, options: StructuredDeltaCuratorOptions = {}) { + this.#dependencies = dependencies; + this.#maxResponseAttempts = positiveInteger( + options.maxResponseAttempts, + DEFAULT_MAX_RESPONSE_ATTEMPTS, + "maxResponseAttempts", + ); + } + + async curate(input: CuratorInput): Promise { + input.signal?.throwIfAborted(); + const lessons = finalLessons(input); + let lastError: unknown = null; + for (let attempt = 1; attempt <= this.#maxResponseAttempts; attempt += 1) { + const response = await abortable( + input.model.execute({ + stage: "curator", + promptVersion: input.input.promptVersions.curator, + idempotencyKey: input.idempotencyKey, + input: modelInput(input, lessons), + outputSchema: CURATOR_OUTPUT_SCHEMA, + signal: input.signal, + }), + input.signal, + ); + try { + const drafts = parseDecisions(response, input, lessons); + return proposalsFromDrafts(drafts, input, lessons, response, this.#dependencies.version); + } catch (error) { + lastError = error; + } + } + throw new EvolutionExecutionError("Curator returned an invalid structured response", { + reasonCode: "curation-invalid-response", + retryable: false, + cause: lastError, + }); + } +} diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 643fa9bc9..14d6511f2 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -1,5 +1,6 @@ export { type ActiveEvolutionStage, + type ApplicableInstructionReference, type CheckpointSavedLedgerEvent, COUNTER_OBSERVATION_SCHEMA_VERSION, COUNTER_REPORT_SCHEMA_VERSION, @@ -27,8 +28,6 @@ export { type EvidenceSliceEdge, type EvidenceSliceOpenReason, type EvidenceSourceCursor, - type EvolutionMemoryAddDelta, - type EvolutionMemoryDelta, type EvolutionArtifactMetadata, type EvolutionArtifactProducer, type EvolutionArtifactReference, @@ -44,12 +43,14 @@ export { type EvolutionJobStage, type EvolutionLeaseToken, type EvolutionLedgerEvent, + type EvolutionMemoryAddDelta, + type EvolutionMemoryDelta, type EvolutionMemoryItem, type EvolutionMemoryItemLedgerMetadata, type EvolutionMemoryProposal, + type EvolutionMemoryReference, type EvolutionMemoryRemoveDelta, type EvolutionMemoryReplaceDelta, - type EvolutionMemoryReference, type EvolutionMemoryTarget, type EvolutionModelIdentity, type EvolutionMutation, @@ -105,7 +106,6 @@ export { type ValidationIssue, type ValidationReport, type ValidationSeverity, - type ApplicableInstructionReference, } from "./contracts.ts"; export { EvolutionControlPlane, @@ -115,6 +115,12 @@ export { type EvolutionJobCreationResult, } from "./control-plane.ts"; export { DisabledPreReflectionStage } from "./counter.ts"; +export { + CURATOR_OUTPUT_SCHEMA, + StructuredDeltaCurator, + type StructuredDeltaCuratorDependencies, + type StructuredDeltaCuratorOptions, +} from "./curator.ts"; export { EvolutionExecutionError } from "./errors.ts"; export { EvolutionStorageCorruptionError, diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index e85ffcd1b..c8080218c 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -75,6 +75,7 @@ export interface CuratorModelRequest { readonly promptVersion: string; readonly idempotencyKey: string; readonly input: JsonObject; + readonly outputSchema: JsonObject; readonly tools?: never; readonly signal?: AbortSignal; } diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index 4e95f51fb..414fa1d47 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -446,6 +446,7 @@ describe("checkpointed evolution pipeline", () => { promptVersion: "curator-v1", idempotencyKey: input.idempotencyKey, input: {}, + outputSchema: {}, signal: input.signal, }); const exactDelta = { @@ -460,11 +461,13 @@ describe("checkpointed evolution pipeline", () => { summary: "Append one focused rule", scope: { repository: scope, + repositoryRevision: "abc123", materializationIds: [], traceIds: [], sessionIds: [], taskLineageIds: [], runtimeKinds: [], + runtimeVersions: [], sourceRevisions: [], harnessVersions: [FIRST_HARNESS_VERSION], applicabilityBoundaries: [], diff --git a/packages/evolution/test/curator.test.ts b/packages/evolution/test/curator.test.ts new file mode 100644 index 000000000..830b92dec --- /dev/null +++ b/packages/evolution/test/curator.test.ts @@ -0,0 +1,693 @@ +import type { MaterializedEvidence, MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import { describe, expect, it } from "vitest"; +import { + CURATOR_OUTPUT_SCHEMA, + type CuratorInput, + computeEvolutionContentDigest, + createEvolutionArtifactMetadata, + type EvidenceCorpusManifest, + type EvolutionArtifactProducer, + type EvolutionInputSnapshot, + type EvolutionMemoryItem, + type EvolutionModelResponse, + type InstructionSnapshot, + type JsonObject, + type Lesson, + type ReflectionResult, + StructuredDeltaCurator, +} from "../src/index.ts"; + +const FIRST_HARNESS_VERSION = `sha256:${"1".repeat(64)}`; +const SECOND_HARNESS_VERSION = `sha256:${"2".repeat(64)}`; +const MEMORY_ITEM_DIGEST = `sha256:${"3".repeat(64)}`; +const MODEL_CONFIGURATION_DIGEST = `sha256:${"4".repeat(64)}`; +const createdAt = "2026-08-19T12:00:00.000Z"; +const scope = { repositoryId: "diffex", scopeId: "repository" } as const; +const producer: EvolutionArtifactProducer = { + name: "fixture", + version: "1", + promptVersion: null, + model: null, +}; + +function metadata(artifactId: string, sourceIds: readonly string[], content: unknown) { + return createEvolutionArtifactMetadata({ artifactId, producer, sourceIds, scope, createdAt, content }); +} + +function sessionEvidence(evidenceId: string, position: number, sessionId: string): MaterializedEvidence { + return { + kind: "session_item", + evidenceId, + itemType: "message", + payload: { type: "message", id: `entry-${position}`, message: { role: "assistant", content: evidenceId } }, + order: { + traceSequence: position + 2, + timestamp: new Date(Date.parse(createdAt) + position).toISOString(), + sessionPosition: position, + parentSpanId: null, + causalIdentifiers: [{ name: "diffex.task.lineage", value: "task-lineage-1" }], + ambiguity: [], + }, + traceSources: [], + sessionSource: { + bindingId: "binding-1", + sessionId, + path: "/private/session.jsonl", + sessionFormat: "coding-agent-v3", + formatVersion: 3, + segmentTraceSequences: [position + 2], + entryId: `entry-${position}`, + entryOrdinal: position, + }, + }; +} + +function invocation(): MaterializedInvocation { + return { + version: 1, + tracePath: "/private/trace.jsonl", + traceId: "trace-1", + producer: { + diffexVersion: "0.2.0", + sourceRevision: "revision-1", + runtime: { kind: "node", version: "24.0.0" }, + telemetrySchemaVersions: {}, + metadata: {}, + }, + integrity: "complete", + terminationStatus: "completed", + health: null, + ordering: { + criteria: ["trace_sequence", "session_cursor", "timestamp", "span_parentage", "causal_id"], + ambiguity: [], + }, + evidence: [ + { + kind: "trace_record", + evidenceId: "activation-1", + payload: { + type: "harness_activate", + sequence: 1, + timestamp: createdAt, + harnessVersion: FIRST_HARNESS_VERSION, + reason: "initial", + }, + order: { + traceSequence: 1, + timestamp: createdAt, + causalIdentifiers: [], + ambiguity: [], + }, + traceSources: [], + }, + sessionEvidence("support-1", 0, "session-1"), + sessionEvidence("counter-1", 1, "session-1"), + ], + diagnostics: [], + }; +} + +function memoryItem(memoryItemId = "memory-1"): EvolutionMemoryItem { + return { + schemaVersion: 1, + memoryItemId, + contentDigest: MEMORY_ITEM_DIGEST, + creationRevision: FIRST_HARNESS_VERSION, + supersessionRevision: null, + ledger: { origin: "evolution", creationEventId: "event-1", supersessionEventId: null }, + }; +} + +function fixtureInput(): EvolutionInputSnapshot { + const materialized = invocation(); + const source = { + sourceId: "source-1", + materializationId: "materialization-1", + ingestionCursor: 1, + origin: "product-runtime" as const, + contentDigest: computeEvolutionContentDigest(materialized), + invocation: materialized, + }; + const memory = "# Evolution memory\n\n- Keep focused changes.\n"; + const firstInstruction = "Repository guidance.\n"; + const secondInstruction = "Nested guidance.\n"; + return { + metadata: metadata("input-1", [source.sourceId], { sourceId: source.sourceId }), + trigger: "manual", + repository: { repositoryId: "diffex", rootPath: "/repo", revision: "current-revision" }, + harness: { + harnessVersion: SECOND_HARNESS_VERSION, + baseVersion: FIRST_HARNESS_VERSION, + memory, + memoryContentDigest: computeEvolutionContentDigest(memory), + activeMemoryItems: [memoryItem()], + }, + expectedHarnessBaseVersion: SECOND_HARNESS_VERSION, + ingestionHighWaterCursor: 1, + instructions: [ + { + path: "/repo/AGENTS.md", + precedence: 10, + content: firstInstruction, + contentDigest: computeEvolutionContentDigest(firstInstruction), + }, + { + path: "/repo/packages/evolution/AGENTS.md", + precedence: 20, + content: secondInstruction, + contentDigest: computeEvolutionContentDigest(secondInstruction), + }, + ], + modelConfiguration: { + provider: "faux", + model: "fixture", + configurationDigest: MODEL_CONFIGURATION_DIGEST, + }, + pipelineVersion: "pipeline-v1", + promptVersions: { + reflector: "reflector-v1", + curator: "curator-v1", + validator: "validator-v1", + writer: "writer-v1", + }, + tokenMeasurement: { estimatorVersion: "fixture", estimatedTokens: 100 }, + evidenceSources: [source], + }; +} + +function lesson(lessonId: string, summary: string): Lesson { + const content = { lessonId, summary }; + return { + metadata: metadata(lessonId, ["support-1", "counter-1"], content), + lessonId, + kind: "successful-strategy", + summary, + preconditions: ["The task changes repository code"], + observedOutcome: "The result remained focused", + evidenceIds: ["support-1"], + counterevidenceIds: ["counter-1"], + scope, + applicabilityBoundaries: ["Repository code changes"], + uncertainty: null, + actionableTakeaway: "Keep changes focused", + }; +} + +function reflection(): ReflectionResult { + const finalLesson = lesson("lesson-final", "Keep changes focused"); + const intermediateLesson = lesson("lesson-intermediate", "Intermediate observation"); + return { + nodes: [], + lessons: [intermediateLesson, finalLesson], + coverage: { + metadata: metadata("coverage-1", ["source-1"], { finalLessonIds: [finalLesson.lessonId] }), + manifestDigest: "corpus-digest", + entries: [], + rootNodeIds: [], + finalLessonIds: [finalLesson.lessonId], + }, + finalLessonIds: [finalLesson.lessonId], + }; +} + +function corpus(): EvidenceCorpusManifest { + return { + metadata: metadata("corpus-1", ["source-1"], { documents: ["document-1"] }), + documents: [ + { + documentId: "document-1", + sourceId: "source-1", + materializationId: "materialization-1", + contentDigest: "source-digest", + materializedInvocationVersion: 1, + traceId: "trace-1", + integrity: "complete", + terminationStatus: "completed", + evidenceIds: ["activation-1", "support-1", "counter-1"], + semanticUnits: [], + diagnostics: [], + orderingAmbiguity: [], + }, + ], + excludedSources: [], + }; +} + +function instructions(input: EvolutionInputSnapshot): InstructionSnapshot { + return { + metadata: metadata("instructions-1", [input.metadata.artifactId], input.instructions), + documents: input.instructions, + }; +} + +function curatorInput(execute: CuratorInput["model"]["execute"]): CuratorInput { + const input = fixtureInput(); + return { + input, + corpus: corpus(), + instructions: instructions(input), + reflection: reflection(), + model: { execute }, + idempotencyKey: "job-1:curating", + }; +} + +function novelRelationship(): JsonObject { + return { classification: "novel", relevantEntries: [], strengthening: null }; +} + +function proposal( + action: "ADD" | "DEFER" | "REMOVE" | "REPLACE", + target: JsonObject, + exactDelta: JsonObject | null, + overrides: Readonly> = {}, +) { + return { + target, + action, + summary: `${action} focused guidance`, + exactDelta, + expectedBenefit: "Future changes remain focused", + acceptanceCriterion: "The guidance is independently addressable", + risk: "The guidance could be applied too broadly", + rollbackDescription: "Reverse this exact delta", + curatorConfidence: 0.85, + uncertainty: null, + counterexamples: ["Generated files require broad mechanical updates"], + conflictingDecisionKeys: [], + finding: null, + ...overrides, + }; +} + +function response(decisions: readonly JsonObject[]): EvolutionModelResponse { + return { output: { decisions }, model: { provider: "faux", model: "curator-fixture" } }; +} + +function decision( + decisionKey: string, + proposalValue: JsonObject | null, + relationship: JsonObject = novelRelationship(), +) { + return { decisionKey, lessonIds: ["lesson-final"], relationship, proposal: proposalValue }; +} + +describe("structured delta curator", () => { + it("emits a provenance-rich ADD from final lessons and ordered snapshots", async () => { + const capturedInputs: JsonObject[] = []; + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + const input = curatorInput(async (request) => { + if (request.stage !== "curator") throw new Error("Unexpected Reflector request"); + capturedInputs.push(request.input); + expect(request.outputSchema).toBe(CURATOR_OUTPUT_SCHEMA); + expect("tools" in request).toBe(false); + return response([ + decision( + "focused-add", + proposal( + "ADD", + { kind: "evolution-memory" }, + { + operation: "ADD", + destination: { section: "Engineering", anchor: null }, + content: "Keep repository changes focused.", + intendedScope: ["Repository code changes"], + }, + ), + ), + ]); + }); + + const first = await curator.curate(input); + const second = await curator.curate(input); + + expect(capturedInputs[0]?.lessons).toEqual([expect.objectContaining({ lessonId: "lesson-final" })]); + expect(capturedInputs[0]?.instructions).toEqual(input.instructions.documents); + expect(capturedInputs[0]?.memoryCatalog).toMatchObject({ + renderedMemory: input.input.harness.memory, + baseDigest: input.input.harness.memoryContentDigest, + activeItems: [expect.objectContaining({ memoryItemId: "memory-1" })], + }); + expect(first).toHaveLength(1); + expect(first[0]).toMatchObject({ + disposition: "actionable", + target: { kind: "evolution-memory", expectedHarnessVersion: SECOND_HARNESS_VERSION }, + action: "ADD", + expectedBaseDigest: input.input.harness.memoryContentDigest, + lessonIds: ["lesson-final"], + supportingEvidenceIds: ["support-1"], + contradictingEvidenceIds: ["counter-1"], + supportingTraceIds: ["trace-1"], + contradictingTraceIds: ["trace-1"], + scope: { + repository: scope, + repositoryRevision: "current-revision", + materializationIds: ["materialization-1"], + traceIds: ["trace-1"], + sessionIds: ["session-1"], + taskLineageIds: ["task-lineage-1"], + runtimeKinds: ["node"], + runtimeVersions: ["24.0.0"], + sourceRevisions: ["revision-1"], + harnessVersions: [FIRST_HARNESS_VERSION], + applicabilityBoundaries: ["Repository code changes"], + }, + metadata: { + producer: { + name: "diffex-evolution-curator", + version: "1.2.3", + promptVersion: "curator-v1", + model: { + provider: "faux", + model: "curator-fixture", + configurationDigest: MODEL_CONFIGURATION_DIGEST, + }, + }, + }, + }); + expect(second[0]?.proposalId).toBe(first[0]?.proposalId); + expect(second[0]?.metadata.contentDigest).toBe(first[0]?.metadata.contentDigest); + }); + + it("emits exact removal and atomic replacement deltas", async () => { + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + const proposals = await curator.curate( + curatorInput(() => + Promise.resolve( + response([ + decision( + "remove-rule", + proposal( + "REMOVE", + { kind: "evolution-memory" }, + { + operation: "REMOVE", + memoryItemId: "memory-1", + reason: "Counterevidence shows the rule is harmful", + }, + ), + ), + decision( + "replace-rule", + proposal( + "REPLACE", + { kind: "evolution-memory" }, + { + operation: "REPLACE", + removal: { + memoryItemId: "memory-1", + reason: "Narrow the existing rule", + }, + addition: { + destination: { section: null, anchor: "memory-1" }, + content: "Keep repository code changes focused.", + intendedScope: ["Repository code changes"], + }, + }, + ), + ), + ]), + ), + ), + ); + + expect(proposals[0]).toMatchObject({ + action: "REMOVE", + exactDelta: { memoryItemId: "memory-1", expectedContentDigest: MEMORY_ITEM_DIGEST }, + }); + expect(proposals[1]).toMatchObject({ + action: "REPLACE", + exactDelta: { + operation: "REPLACE", + removal: { memoryItemId: "memory-1", expectedContentDigest: MEMORY_ITEM_DIGEST }, + addition: { content: "Keep repository code changes focused." }, + }, + }); + }); + + it("preserves skill and source findings as non-executable deferred proposals", async () => { + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + const proposals = await curator.curate( + curatorInput(() => + Promise.resolve( + response([ + decision( + "defer-skill", + proposal("DEFER", { kind: "skill", skillName: "review" }, null, { + finding: "The review skill needs a narrower verification contract", + }), + ), + decision( + "defer-source", + proposal("DEFER", { kind: "orchestration-source", sourcePath: "src/runner.ts" }, null, { + finding: "The runner should isolate retries", + }), + ), + ]), + ), + ), + ); + + expect(proposals).toEqual([ + expect.objectContaining({ + disposition: "deferred", + target: { kind: "skill", skillName: "review" }, + action: "DEFER", + exactDelta: null, + expectedBaseDigest: null, + }), + expect.objectContaining({ + disposition: "deferred", + target: { kind: "orchestration-source", sourcePath: "src/runner.ts" }, + action: "DEFER", + exactDelta: null, + }), + ]); + }); + + it("resolves model decision conflicts to stable proposal IDs", async () => { + const addDelta = { + operation: "ADD", + destination: { section: "Engineering", anchor: null }, + content: "Keep changes focused.", + intendedScope: ["Repository code changes"], + }; + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + const proposals = await curator.curate( + curatorInput(() => + Promise.resolve( + response([ + decision( + "broad-rule", + proposal("ADD", { kind: "evolution-memory" }, addDelta, { + conflictingDecisionKeys: ["narrow-rule"], + }), + ), + decision("narrow-rule", proposal("ADD", { kind: "evolution-memory" }, addDelta)), + ]), + ), + ), + ); + + expect(proposals[0]?.conflictingProposalIds).toEqual([proposals[1]?.proposalId]); + }); + + it("suppresses duplicate and conflicting decisions and retains strengthening references", async () => { + const input = curatorInput(() => + Promise.resolve( + response([ + decision("duplicate-rule", null, { + classification: "duplicate", + relevantEntries: [ + { kind: "applicable-instruction", path: "/repo/AGENTS.md", locator: "Repository guidance" }, + ], + strengthening: null, + }), + decision("conflicting-rule", null, { + classification: "conflicts", + relevantEntries: [ + { kind: "evolution-memory", memoryItemId: "memory-1", locator: "Keep focused changes" }, + ], + strengthening: null, + }), + decision( + "strengthen-rule", + proposal( + "ADD", + { kind: "evolution-memory" }, + { + operation: "ADD", + destination: { section: "Engineering", anchor: null }, + content: "Run focused verification after repository changes.", + intendedScope: ["Repository code changes"], + }, + ), + { + classification: "strengthens", + relevantEntries: [ + { + kind: "applicable-instruction", + path: "/repo/packages/evolution/AGENTS.md", + locator: "Nested guidance", + }, + ], + strengthening: { kind: "enforcement", description: "Adds a verification requirement" }, + }, + ), + ]), + ), + ); + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + + const proposals = await curator.curate(input); + + expect(proposals).toHaveLength(1); + expect(proposals[0]).toMatchObject({ + instructionRelationship: { + classification: "strengthens", + relevantEntries: [ + { + kind: "applicable-instruction", + path: "/repo/packages/evolution/AGENTS.md", + precedence: 20, + contentDigest: input.instructions.documents[1]?.contentDigest, + }, + ], + strengthening: { kind: "enforcement" }, + }, + }); + }); + + it("retries invalid structured output without changing the idempotency key", async () => { + const keys: string[] = []; + let calls = 0; + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + const proposals = await curator.curate( + curatorInput((request) => { + keys.push(request.idempotencyKey); + calls += 1; + if (calls === 1) { + return Promise.resolve( + response([{ ...decision("invalid", null), lessonIds: ["lesson-intermediate"], proposal: null }]), + ); + } + return Promise.resolve( + response([ + decision( + "valid", + proposal( + "ADD", + { kind: "evolution-memory" }, + { + operation: "ADD", + destination: { section: "Engineering", anchor: null }, + content: "Keep changes focused.", + intendedScope: ["Repository code changes"], + }, + ), + ), + ]), + ); + }), + ); + + expect(proposals).toHaveLength(1); + expect(keys).toEqual(["job-1:curating", "job-1:curating"]); + }); + + it("rejects unsupported writer routes after bounded attempts", async () => { + let calls = 0; + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + const input = curatorInput(() => { + calls += 1; + return Promise.resolve( + response([ + decision( + "invalid-skill-write", + proposal( + "ADD", + { kind: "skill", skillName: "review" }, + { + operation: "ADD", + destination: { section: "Review", anchor: null }, + content: "Write directly to the skill.", + intendedScope: ["review"], + }, + ), + ), + ]), + ); + }); + + await expect(curator.curate(input)).rejects.toMatchObject({ + reasonCode: "curation-invalid-response", + retryable: false, + }); + expect(calls).toBe(2); + }); + + it("rejects unknown instruction, memory-item, and evidence references", async () => { + const curator = new StructuredDeltaCurator({ version: "1.2.3" }, { maxResponseAttempts: 1 }); + const addProposal = proposal( + "ADD", + { kind: "evolution-memory" }, + { + operation: "ADD", + destination: { section: "Engineering", anchor: null }, + content: "Keep changes focused.", + intendedScope: ["Repository code changes"], + }, + ); + const unknownInstruction = curatorInput(() => + Promise.resolve( + response([ + decision("unknown-instruction", addProposal, { + classification: "strengthens", + relevantEntries: [{ kind: "applicable-instruction", path: "/repo/missing.md", locator: "Missing" }], + strengthening: { kind: "precision", description: "Adds exact guidance" }, + }), + ]), + ), + ); + const unknownMemoryItem = curatorInput(() => + Promise.resolve( + response([ + decision( + "unknown-memory", + proposal( + "REMOVE", + { kind: "evolution-memory" }, + { + operation: "REMOVE", + memoryItemId: "missing-memory", + reason: "Remove stale guidance", + }, + ), + ), + ]), + ), + ); + const unknownEvidenceBase = curatorInput(() => + Promise.resolve(response([decision("unknown-evidence", addProposal)])), + ); + const unknownEvidence: CuratorInput = { + ...unknownEvidenceBase, + reflection: { + ...unknownEvidenceBase.reflection, + lessons: unknownEvidenceBase.reflection.lessons.map((item) => + item.lessonId === "lesson-final" ? { ...item, evidenceIds: ["missing-evidence"] } : item, + ), + }, + }; + + await expect(curator.curate(unknownInstruction)).rejects.toMatchObject({ + reasonCode: "curation-invalid-response", + }); + await expect(curator.curate(unknownMemoryItem)).rejects.toMatchObject({ + reasonCode: "curation-invalid-response", + }); + await expect(curator.curate(unknownEvidence)).rejects.toMatchObject({ + reasonCode: "curation-invalid-response", + }); + }); +});