diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index b18b11989..5baae6c5c 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -10,6 +10,7 @@ export const EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION = 1 as const; export const COUNTER_OBSERVATION_SCHEMA_VERSION = 1 as const; export const COUNTER_REPORT_SCHEMA_VERSION = 1 as const; export const SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION = 1 as const; +export const SKILL_EVALUATION_SCHEMA_VERSION = 1 as const; export type JsonPrimitive = boolean | number | string | null; export type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[]; @@ -652,6 +653,121 @@ export interface SkillDraftValidationReport { readonly memoryRelationshipDigest: string; } +export type SkillEvaluationCaseKind = "hard-negative" | "intended" | "unrelated"; + +export type SkillEvaluationCaseOrigin = "held-out" | "source-replay"; + +export type SkillEvaluationVerifierKind = "environment" | "test" | "tool"; + +export interface SkillEvaluationVerifier { + readonly kind: SkillEvaluationVerifierKind; + readonly name: string; + readonly version: string; + readonly configurationDigest: string; +} + +export interface SkillEvaluationCase { + readonly caseId: string; + readonly taskFamily: string; + readonly kind: SkillEvaluationCaseKind; + readonly origin: SkillEvaluationCaseOrigin; + readonly taskLineageId: string; + readonly sourceTraceId: string | null; + readonly prompt: string; + readonly expectedSkillIds: readonly string[]; + readonly confusableSkillIds: readonly string[]; + readonly verifier: SkillEvaluationVerifier; +} + +export interface SkillEvaluationComponentConfiguration { + readonly name: string; + readonly version: string; + readonly configurationDigest: string; +} + +export interface SkillEvaluationRuntimeConfiguration { + readonly model: EvolutionModelIdentity; + readonly runtime: SkillEvaluationComponentConfiguration; + readonly retriever: SkillEvaluationComponentConfiguration; + readonly executor: SkillEvaluationComponentConfiguration; +} + +export interface SkillEvaluationPoolSkill { + readonly skillId: string; + readonly name: string; + readonly description: string; + readonly source: "candidate" | "catalog"; + readonly version: string; + readonly contentDigest: string; + readonly content: string; +} + +export interface SkillRetrievalEvaluationResult { + readonly caseId: string; + readonly selectedSkillIds: readonly string[]; + readonly scores: readonly { + readonly skillId: string; + readonly score: number; + }[]; +} + +export interface SkillBehaviorEvaluationOutcome { + readonly passed: boolean; + readonly score: number; + readonly appliedSkillIds: readonly string[]; + readonly verifier: SkillEvaluationVerifier; + readonly details: JsonObject; +} + +export interface SkillBehaviorEvaluationResult { + readonly caseId: string; + readonly baseline: SkillBehaviorEvaluationOutcome; + readonly fullPool: SkillBehaviorEvaluationOutcome; +} + +export interface SkillEvaluationGate { + readonly passed: boolean; + readonly reasonCodes: readonly string[]; +} + +export interface SkillEvaluationMetrics { + readonly exactRetrievalAccuracy: number; + readonly sourceReplayReproducibility: number; + readonly heldOutIntendedBaselineScore: number; + readonly heldOutIntendedFullPoolScore: number; + readonly heldOutIntendedImprovement: number; + readonly negativeRegressionCount: number; + readonly misapplicationCount: number; +} + +export interface SkillEvaluationReport { + readonly artifactKind: "skill-evaluation"; + readonly schemaVersion: typeof SKILL_EVALUATION_SCHEMA_VERSION; + readonly metadata: EvolutionArtifactMetadata; + readonly evaluationId: string; + readonly groupId: string; + readonly evaluationIdempotencyKey: string; + readonly proposalRevisionIds: readonly string[]; + readonly draft: EvolutionArtifactReference; + readonly validation: EvolutionArtifactReference; + readonly evaluatorVersion: string; + readonly assurance: "retrieval-and-deterministic-behavior-gates"; + readonly disposition: "accepted" | "quarantined"; + readonly harnessVersion: HarnessVersion; + readonly candidateSkill: SkillEvaluationPoolSkill; + readonly neighboringSkills: readonly SkillEvaluationPoolSkill[]; + readonly configuration: SkillEvaluationRuntimeConfiguration; + readonly cases: readonly SkillEvaluationCase[]; + readonly retrievalResults: readonly SkillRetrievalEvaluationResult[]; + readonly behaviorResults: readonly SkillBehaviorEvaluationResult[]; + readonly metrics: SkillEvaluationMetrics; + readonly gates: { + readonly suite: SkillEvaluationGate; + readonly retrieval: SkillEvaluationGate; + readonly behavior: SkillEvaluationGate; + }; +} + export interface SkillCandidateDispositionRecord { readonly eventId: string; readonly from: SkillCandidateState | null; @@ -670,6 +786,7 @@ export interface SkillCandidateGroup { readonly priorityScore: EvolutionArtifactReference | null; readonly draft: EvolutionArtifactReference | null; readonly validation: EvolutionArtifactReference | null; + readonly evaluation: EvolutionArtifactReference | null; readonly synthesisIdempotencyKey: string; readonly validationIdempotencyKey: string; readonly evaluationIdempotencyKey: string; @@ -713,6 +830,16 @@ export interface SkillCandidateValidationRecordedEvent extends SkillCandidateReg readonly idempotencyKey: string; } +export interface SkillCandidateEvaluationRecordedEvent extends SkillCandidateRegistryEventBase { + readonly type: "skill-candidate-evaluation-recorded"; + readonly groupId: string; + readonly draft: EvolutionArtifactReference; + readonly validation: EvolutionArtifactReference; + readonly evaluation: EvolutionArtifactReference; + readonly proposalRevisionIds: readonly string[]; + readonly idempotencyKey: string; +} + export interface SkillCandidateStateTransitionedEvent extends SkillCandidateRegistryEventBase { readonly type: "skill-candidate-state-transitioned"; readonly groupId: string; @@ -721,7 +848,13 @@ export interface SkillCandidateStateTransitionedEvent extends SkillCandidateRegi readonly reasonCodes: readonly string[]; } -export type SkillCandidateWorkStage = "consolidating" | "scoring" | "synthesizing" | "validating" | "completed"; +export type SkillCandidateWorkStage = + | "consolidating" + | "scoring" + | "synthesizing" + | "validating" + | "evaluating" + | "completed"; export interface SkillCandidateWorkCheckpointedEvent extends SkillCandidateRegistryEventBase { readonly type: "skill-candidate-work-checkpointed"; @@ -748,6 +881,7 @@ export type SkillCandidateRegistryEvent = | SkillCandidatePriorityScoredEvent | SkillCandidateDraftRecordedEvent | SkillCandidateValidationRecordedEvent + | SkillCandidateEvaluationRecordedEvent | SkillCandidateStateTransitionedEvent | SkillCandidateWorkCheckpointedEvent | SkillCandidateWorkFailedEvent; diff --git a/packages/evolution/src/filesystem-store.ts b/packages/evolution/src/filesystem-store.ts index 0afcdec24..894916c6d 100644 --- a/packages/evolution/src/filesystem-store.ts +++ b/packages/evolution/src/filesystem-store.ts @@ -49,6 +49,7 @@ const JOB_STAGES = new Set([ const ACTIVE_JOB_STAGES = new Set([...JOB_STAGES].filter((stage) => stage !== "completed")); const SKILL_CANDIDATE_REGISTRY_EVENT_TYPES = new Set([ "skill-candidate-draft-recorded", + "skill-candidate-evaluation-recorded", "skill-candidate-priority-scored", "skill-candidate-proposal-recorded", "skill-candidate-state-transitioned", @@ -67,7 +68,14 @@ const SKILL_CANDIDATE_STATES = new Set([ "rejected", "retired", ]); -const SKILL_CANDIDATE_WORK_STAGES = new Set(["consolidating", "scoring", "synthesizing", "validating", "completed"]); +const SKILL_CANDIDATE_WORK_STAGES = new Set([ + "consolidating", + "scoring", + "synthesizing", + "validating", + "evaluating", + "completed", +]); const JOB_OUTCOMES = new Set([ "active", "completed", @@ -284,6 +292,16 @@ function isCandidateRegistryEventContent(value: Readonly typeof value.idempotencyKey === "string" ); } + if (value.type === "skill-candidate-evaluation-recorded") { + return ( + typeof value.groupId === "string" && + isArtifactReference(value.draft) && + isArtifactReference(value.validation) && + isArtifactReference(value.evaluation) && + isStringArray(value.proposalRevisionIds) && + typeof value.idempotencyKey === "string" + ); + } if (value.type === "skill-candidate-validation-recorded") { return ( typeof value.groupId === "string" && diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 2f0adf1ea..9651644e7 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -121,10 +121,14 @@ export { type ScoredProposal, type ScoredProposalDisposition, SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, + SKILL_EVALUATION_SCHEMA_VERSION, type SkillabilityClassification, + type SkillBehaviorEvaluationOutcome, + type SkillBehaviorEvaluationResult, type SkillCandidate, type SkillCandidateDispositionRecord, type SkillCandidateDraftRecordedEvent, + type SkillCandidateEvaluationRecordedEvent, type SkillCandidateEvidenceContribution, type SkillCandidateFailureHandling, type SkillCandidateGroup, @@ -150,7 +154,19 @@ export { type SkillDraft, type SkillDraftValidationIssue, type SkillDraftValidationReport, + type SkillEvaluationCase, + type SkillEvaluationCaseKind, + type SkillEvaluationCaseOrigin, + type SkillEvaluationComponentConfiguration, + type SkillEvaluationGate, + type SkillEvaluationMetrics, + type SkillEvaluationPoolSkill, + type SkillEvaluationReport, + type SkillEvaluationRuntimeConfiguration, + type SkillEvaluationVerifier, + type SkillEvaluationVerifierKind, type SkillMemoryRelationshipClassification, + type SkillRetrievalEvaluationResult, type SkillTarget, type ValidationIssue, type ValidationReport, @@ -253,6 +269,8 @@ export type { Reflector, ReflectorInput, SkillCandidateDraftRecordingInput, + SkillCandidateEvaluationRecordingInput, + SkillCandidateEvaluationStartInput, SkillCandidateRegistry, SkillCandidateRegistryAppendResult, SkillCandidateRegistryConsolidationInput, @@ -266,6 +284,15 @@ export type { SkillDraftValidationEnvironmentSource, SkillDraftValidator, SkillDraftValidatorInput, + SkillEvaluationCaseSource, + SkillEvaluationCaseSourceInput, + SkillEvaluationEnvironment, + SkillEvaluationEnvironmentSource, + SkillEvaluationExecutionInput, + SkillEvaluationExecutor, + SkillEvaluationSelectionInput, + SkillEvaluator, + SkillEvaluatorInput, SkillSynthesizer, SkillSynthesizerInput, SuccessfulMaterialization, @@ -340,6 +367,11 @@ export { type DeterministicSkillDraftValidatorDependencies, type DeterministicSkillDraftValidatorOptions, } from "./skill-draft-validator.ts"; +export { + DeterministicSkillEvaluator, + type DeterministicSkillEvaluatorDependencies, + SKILL_EVALUATOR_VERSION, +} from "./skill-evaluator.ts"; export { StructuredSkillSynthesizer, type StructuredSkillSynthesizerDependencies, diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index d4fc846d1..5daa690c9 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -34,6 +34,7 @@ import type { ProceduralInformationLossFinding, ReflectionNode, ScoredProposal, + SkillBehaviorEvaluationOutcome, SkillCandidate, SkillCandidateGroup, SkillCandidatePriorityScore, @@ -41,6 +42,11 @@ import type { SkillCandidateRegistrySnapshot, SkillDraft, SkillDraftValidationReport, + SkillEvaluationCase, + SkillEvaluationPoolSkill, + SkillEvaluationReport, + SkillEvaluationRuntimeConfiguration, + SkillRetrievalEvaluationResult, ValidationReport, } from "./contracts.ts"; @@ -63,6 +69,7 @@ export type EvolutionArtifact = | SkillCandidatePriorityScore | SkillDraft | SkillDraftValidationReport + | SkillEvaluationReport | DeferredSkillCandidateFinding | ValidationReport; @@ -330,10 +337,33 @@ export interface SkillCandidateValidationRecordingInput { readonly createdAt: string; } +export interface SkillCandidateEvaluationStartInput { + readonly scope: EvolutionScope; + readonly groupId: string; + readonly draft: EvolutionArtifactReference; + readonly validation: EvolutionArtifactReference; + readonly proposalRevisionIds: readonly string[]; + readonly idempotencyKey: string; + readonly createdAt: string; +} + +export interface SkillCandidateEvaluationRecordingInput { + readonly scope: EvolutionScope; + readonly groupId: string; + readonly draft: EvolutionArtifactReference; + readonly validation: EvolutionArtifactReference; + readonly evaluation: EvolutionArtifactReference; + readonly proposalRevisionIds: readonly string[]; + readonly idempotencyKey: string; + readonly createdAt: string; +} + export interface SkillCandidateRegistry { consolidate(input: SkillCandidateRegistryConsolidationInput): Promise; recordDraft(input: SkillCandidateDraftRecordingInput): Promise; recordValidation(input: SkillCandidateValidationRecordingInput): Promise; + startEvaluation(input: SkillCandidateEvaluationStartInput): Promise; + recordEvaluation(input: SkillCandidateEvaluationRecordingInput): Promise; list(scope: EvolutionScope): Promise; } @@ -381,6 +411,67 @@ export interface SkillDraftValidator { validate(input: SkillDraftValidatorInput): Promise; } +export interface SkillEvaluationEnvironment { + readonly workspace: { + readonly repositoryId: string; + readonly directoryId?: string; + }; + readonly harnessVersion: string; + readonly skills: readonly SkillEvaluationPoolSkill[]; +} + +export interface SkillEvaluationCaseSourceInput { + readonly group: SkillCandidateGroup; + readonly draft: SkillDraft; + readonly validation: SkillDraftValidationReport; + readonly candidates: readonly SkillCandidate[]; + readonly environment: SkillEvaluationEnvironment; + readonly candidateSkill: SkillEvaluationPoolSkill; + readonly signal?: AbortSignal; +} + +export interface SkillEvaluationCaseSource { + build(input: SkillEvaluationCaseSourceInput): Promise; +} + +export interface SkillEvaluationEnvironmentSource { + capture( + workspace: SkillCandidateGroup["profile"]["workspace"], + signal?: AbortSignal, + ): Promise; +} + +export interface SkillEvaluationSelectionInput { + readonly evaluationIdempotencyKey: string; + readonly harnessVersion: string; + readonly testCase: SkillEvaluationCase; + readonly skills: readonly SkillEvaluationPoolSkill[]; + readonly signal?: AbortSignal; +} + +export interface SkillEvaluationExecutionInput extends SkillEvaluationSelectionInput { + readonly arm: "baseline" | "full-pool"; +} + +export interface SkillEvaluationExecutor { + readonly configuration: SkillEvaluationRuntimeConfiguration; + select(input: SkillEvaluationSelectionInput): Promise; + execute(input: SkillEvaluationExecutionInput): Promise; +} + +export interface SkillEvaluatorInput { + readonly group: SkillCandidateGroup; + readonly draft: SkillDraft; + readonly validation: SkillDraftValidationReport; + readonly idempotencyKey: string; + readonly createdAt: string; + readonly signal?: AbortSignal; +} + +export interface SkillEvaluator { + evaluate(input: SkillEvaluatorInput): Promise; +} + export interface EvolutionJobStore { create(job: EvolutionJob, lease: EvolutionLeaseToken): Promise; save(job: EvolutionJob, expectedRevision: number, lease: EvolutionLeaseToken): Promise; diff --git a/packages/evolution/src/skill-candidate-registry.ts b/packages/evolution/src/skill-candidate-registry.ts index 10f1eca46..c0be5df21 100644 --- a/packages/evolution/src/skill-candidate-registry.ts +++ b/packages/evolution/src/skill-candidate-registry.ts @@ -18,11 +18,14 @@ import type { SkillCandidateStateTransitionedEvent, SkillDraft, SkillDraftValidationReport, + SkillEvaluationReport, } from "./contracts.ts"; -import { SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION } from "./contracts.ts"; +import { SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, SKILL_EVALUATION_SCHEMA_VERSION } from "./contracts.ts"; import type { EvolutionArtifactStore, SkillCandidateDraftRecordingInput, + SkillCandidateEvaluationRecordingInput, + SkillCandidateEvaluationStartInput, SkillCandidateRegistry, SkillCandidateRegistryConsolidationInput, SkillCandidateRegistryConsolidationResult, @@ -65,6 +68,8 @@ interface MutableGroup { readonly draftProposalRevisionIds: string[]; validation: EvolutionArtifactReference | null; readonly validationProposalRevisionIds: string[]; + evaluation: EvolutionArtifactReference | null; + readonly evaluationProposalRevisionIds: string[]; readonly scoredProposalRevisionIds: string[]; } @@ -273,6 +278,8 @@ function materializeGroups(events: readonly SkillCandidateRegistryEvent[]): Map< draftProposalRevisionIds: [], validation: null, validationProposalRevisionIds: [], + evaluation: null, + evaluationProposalRevisionIds: [], scoredProposalRevisionIds: [], }); } else if ( @@ -348,6 +355,37 @@ function materializeGroups(events: readonly SkillCandidateRegistryEvent[]): Map< ); continue; } + if (event.type === "skill-candidate-evaluation-recorded") { + const group = groups.get(event.groupId); + if (group === undefined) throw new Error(`Candidate evaluation references unknown group "${event.groupId}"`); + if ( + !sameValues( + event.proposalRevisionIds, + group.proposalRevisions.map((revision) => revision.revisionId), + ) || + group.draft?.artifactId !== event.draft.artifactId || + group.draft.contentDigest !== event.draft.contentDigest || + group.validation?.artifactId !== event.validation.artifactId || + group.validation.contentDigest !== event.validation.contentDigest + ) { + throw new Error(`Candidate evaluation for group "${event.groupId}" has stale prerequisites`); + } + if ( + group.evaluation !== null && + sameValues(group.evaluationProposalRevisionIds, event.proposalRevisionIds) && + (group.evaluation.artifactId !== event.evaluation.artifactId || + group.evaluation.contentDigest !== event.evaluation.contentDigest) + ) { + throw new Error(`Candidate group "${event.groupId}" has conflicting evaluation reports`); + } + group.evaluation = event.evaluation; + group.evaluationProposalRevisionIds.splice( + 0, + group.evaluationProposalRevisionIds.length, + ...event.proposalRevisionIds, + ); + continue; + } if (event.type === "skill-candidate-state-transitioned") { const group = groups.get(event.groupId); if (group === undefined) throw new Error(`Candidate transition references unknown group "${event.groupId}"`); @@ -394,6 +432,12 @@ function immutableGroup(group: MutableGroup): SkillCandidateGroup { ) ? group.validation : null, + evaluation: sameValues( + group.evaluationProposalRevisionIds, + proposalRevisions.map((revision) => revision.revisionId), + ) + ? group.evaluation + : null, synthesisIdempotencyKey: stableId("skill-synthesis", { groupId: group.groupId, revisionSetDigest }), validationIdempotencyKey: stableId("skill-validation", { groupId: group.groupId, @@ -452,6 +496,38 @@ function assertValidation( } } +function assertEvaluation( + evaluation: SkillEvaluationReport, + reference: EvolutionArtifactReference, + group: SkillCandidateGroup, + input: SkillCandidateEvaluationRecordingInput, +): void { + const gatesPassed = + evaluation.gates.suite.passed && evaluation.gates.retrieval.passed && evaluation.gates.behavior.passed; + if ( + evaluation.artifactKind !== "skill-evaluation" || + evaluation.schemaVersion !== SKILL_EVALUATION_SCHEMA_VERSION || + evaluation.evaluationId === "" || + evaluation.metadata.artifactId !== evaluation.evaluationId || + evaluation.metadata.contentDigest !== computeEvolutionContentDigest(contentWithoutMetadata(evaluation)) || + reference.artifactId !== evaluation.evaluationId || + reference.contentDigest !== computeEvolutionContentDigest(evaluation) || + evaluation.groupId !== group.groupId || + evaluation.draft.artifactId !== input.draft.artifactId || + evaluation.draft.contentDigest !== input.draft.contentDigest || + evaluation.validation.artifactId !== input.validation.artifactId || + evaluation.validation.contentDigest !== input.validation.contentDigest || + evaluation.evaluationIdempotencyKey !== group.evaluationIdempotencyKey || + evaluation.evaluationIdempotencyKey !== input.idempotencyKey || + evaluation.disposition !== (gatesPassed ? "accepted" : "quarantined") || + !sameValues(evaluation.proposalRevisionIds, input.proposalRevisionIds) || + evaluation.metadata.scope.repositoryId !== input.scope.repositoryId || + evaluation.metadata.scope.scopeId !== input.scope.scopeId + ) { + throw new Error(`Skill evaluation for candidate group "${group.groupId}" has an invalid immutable identity`); + } +} + function assertCandidate( candidate: SkillCandidate, reference: EvolutionArtifactReference, @@ -599,8 +675,13 @@ export class DeterministicSkillCandidatePriorityScorer { readonly reasonCodes: readonly string[]; } { const hasStaleArtifacts = - (group.state === "drafted" || group.state === "structurally-valid" || group.state === "rejected") && + (group.state === "drafted" || + group.state === "structurally-valid" || + group.state === "evaluation-pending" || + group.state === "accepted" || + group.state === "rejected") && group.draft === null; + if (group.evaluation !== null) return { state: group.state, reasonCodes: [] }; if (!hasStaleArtifacts && !MANAGED_QUALIFICATION_STATES.has(group.state)) { return { state: group.state, reasonCodes: [] }; } @@ -925,6 +1006,192 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry throw new Error("Skill candidate registry changed too often to record validation"); } + async startEvaluation(input: SkillCandidateEvaluationStartInput): Promise { + for (let attempt = 0; attempt < this.#maxCommitAttempts; attempt += 1) { + const snapshot = await this.#store.loadSkillCandidateRegistry(input.scope); + const mutable = materializeGroups(snapshot.events).get(input.groupId); + if (mutable === undefined) throw new Error(`Unknown candidate group "${input.groupId}"`); + const group = immutableGroup(mutable); + if ( + !sameValues( + input.proposalRevisionIds, + group.proposalRevisions.map((revision) => revision.revisionId), + ) || + group.draft?.artifactId !== input.draft.artifactId || + group.draft.contentDigest !== input.draft.contentDigest || + group.validation?.artifactId !== input.validation.artifactId || + group.validation.contentDigest !== input.validation.contentDigest || + input.idempotencyKey !== group.evaluationIdempotencyKey + ) { + throw new Error(`Candidate group "${input.groupId}" changed before evaluation started`); + } + if (group.state === "evaluation-pending") return group; + if (group.state !== "structurally-valid") { + throw new Error(`Candidate group "${input.groupId}" is not eligible to start evaluation`); + } + const validation = await this.#artifacts.get(input.validation); + if ( + !("artifactKind" in validation) || + validation.artifactKind !== "skill-draft-validation" || + validation.disposition !== "accepted" || + computeEvolutionContentDigest(validation) !== input.validation.contentDigest + ) { + throw new Error(`Candidate group "${input.groupId}" does not have accepted validation`); + } + const transition = transitionEvent( + input.groupId, + "structurally-valid", + "evaluation-pending", + ["retrieval-and-behavior-evaluation-started"], + nextSequence(snapshot.events, 0), + input.idempotencyKey, + this.#producer, + input.scope, + input.createdAt, + ); + const committed = await this.#store.appendSkillCandidateRegistryEvents(input.scope, snapshot.revision, [ + transition, + ]); + if (committed === "committed") { + mutable.state = "evaluation-pending"; + mutable.dispositionHistory.push({ + eventId: transition.eventId, + from: transition.from, + to: transition.to, + reasonCodes: transition.reasonCodes, + createdAt: input.createdAt, + }); + return immutableGroup(mutable); + } + } + throw new Error("Skill candidate registry changed too often to start evaluation"); + } + + async recordEvaluation(input: SkillCandidateEvaluationRecordingInput): Promise { + for (let attempt = 0; attempt < this.#maxCommitAttempts; attempt += 1) { + const snapshot = await this.#store.loadSkillCandidateRegistry(input.scope); + const mutable = materializeGroups(snapshot.events).get(input.groupId); + if (mutable === undefined) throw new Error(`Unknown candidate group "${input.groupId}"`); + let group = immutableGroup(mutable); + if ( + !sameValues( + input.proposalRevisionIds, + group.proposalRevisions.map((revision) => revision.revisionId), + ) || + group.draft?.artifactId !== input.draft.artifactId || + group.draft.contentDigest !== input.draft.contentDigest || + group.validation?.artifactId !== input.validation.artifactId || + group.validation.contentDigest !== input.validation.contentDigest + ) { + throw new Error(`Candidate group "${input.groupId}" changed before evaluation was recorded`); + } + if (group.state !== "evaluation-pending" && group.state !== "accepted" && group.state !== "quarantined") { + throw new Error(`Candidate group "${input.groupId}" is not eligible to record evaluation`); + } + const artifact = await this.#artifacts.get(input.evaluation); + if (!("artifactKind" in artifact) || artifact.artifactKind !== "skill-evaluation") { + throw new Error(`Candidate group "${input.groupId}" evaluation artifact is invalid`); + } + assertEvaluation(artifact, input.evaluation, group, input); + const validation = await this.#artifacts.get(input.validation); + if ( + !("artifactKind" in validation) || + validation.artifactKind !== "skill-draft-validation" || + validation.disposition !== "accepted" || + validation.validatedHarnessVersion !== artifact.harnessVersion + ) { + throw new Error(`Candidate group "${input.groupId}" evaluation uses stale validation`); + } + if ( + group.evaluation !== null && + (group.evaluation.artifactId !== input.evaluation.artifactId || + group.evaluation.contentDigest !== input.evaluation.contentDigest) + ) { + throw new Error(`Candidate group "${input.groupId}" already has a different evaluation report`); + } + const targetState = artifact.disposition; + if (group.state !== "evaluation-pending" && group.state !== targetState) { + throw new Error(`Candidate group "${input.groupId}" evaluation disposition conflicts with its state`); + } + const planned: SkillCandidateRegistryEvent[] = []; + if (group.evaluation === null) { + const registrySequence = nextSequence(snapshot.events, planned.length); + const content = { + schemaVersion: SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, + eventId: stableId("skill-candidate-event", { + idempotencyKey: input.idempotencyKey, + registrySequence, + type: "skill-candidate-evaluation-recorded", + groupId: input.groupId, + }), + registrySequence, + type: "skill-candidate-evaluation-recorded" as const, + groupId: input.groupId, + draft: input.draft, + validation: input.validation, + evaluation: input.evaluation, + proposalRevisionIds: input.proposalRevisionIds, + idempotencyKey: input.idempotencyKey, + }; + planned.push( + withEventMetadata( + content, + this.#producer, + [input.groupId, input.draft.artifactId, input.validation.artifactId, input.evaluation.artifactId], + input.scope, + input.createdAt, + ), + ); + mutable.evaluation = input.evaluation; + mutable.evaluationProposalRevisionIds.splice( + 0, + mutable.evaluationProposalRevisionIds.length, + ...input.proposalRevisionIds, + ); + group = immutableGroup(mutable); + } + if (group.state === "evaluation-pending") { + const failedReasons = uniqueSorted([ + ...artifact.gates.suite.reasonCodes, + ...artifact.gates.retrieval.reasonCodes, + ...artifact.gates.behavior.reasonCodes, + ]); + const transition = transitionEvent( + input.groupId, + "evaluation-pending", + targetState, + targetState === "accepted" + ? ["retrieval-and-behavior-gates-passed"] + : failedReasons.length === 0 + ? ["evaluation-gates-not-passed"] + : failedReasons, + nextSequence(snapshot.events, planned.length), + input.idempotencyKey, + this.#producer, + input.scope, + input.createdAt, + ); + planned.push(transition); + mutable.state = targetState; + mutable.dispositionHistory.push({ + eventId: transition.eventId, + from: transition.from, + to: transition.to, + reasonCodes: transition.reasonCodes, + createdAt: input.createdAt, + }); + } + if (planned.length === 0) return immutableGroup(mutable); + const committed = await this.#store.appendSkillCandidateRegistryEvents( + input.scope, + snapshot.revision, + planned, + ); + if (committed === "committed") return immutableGroup(mutable); + } + throw new Error("Skill candidate registry changed too often to record evaluation"); + } + async consolidate( input: SkillCandidateRegistryConsolidationInput, ): Promise { @@ -967,6 +1234,8 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry draftProposalRevisionIds: [], validation: null, validationProposalRevisionIds: [], + evaluation: null, + evaluationProposalRevisionIds: [], scoredProposalRevisionIds: [], }; groups.set(groupId, group); diff --git a/packages/evolution/src/skill-candidate-worker.ts b/packages/evolution/src/skill-candidate-worker.ts index eb488af25..b11dac62e 100644 --- a/packages/evolution/src/skill-candidate-worker.ts +++ b/packages/evolution/src/skill-candidate-worker.ts @@ -9,6 +9,7 @@ import type { SkillCandidateWorkCheckpointedEvent, SkillCandidateWorkFailedEvent, SkillCandidateWorkStage, + SkillDraftValidationReport, } from "./contracts.ts"; import { SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION } from "./contracts.ts"; import type { @@ -21,6 +22,7 @@ import type { SkillCandidateRegistry, SkillCandidateRegistryStore, SkillDraftValidator, + SkillEvaluator, SkillSynthesizer, } from "./ports.ts"; import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; @@ -55,6 +57,7 @@ export interface SkillCandidateRegistryWorkerDependencies { readonly registry: SkillCandidateRegistry; readonly synthesizer: SkillSynthesizer; readonly validator: SkillDraftValidator; + readonly evaluator?: SkillEvaluator; readonly model: EvolutionModelExecutor; } @@ -144,6 +147,10 @@ function isSkillDraft( return "artifactKind" in artifact && artifact.artifactKind === "skill-draft"; } +function isSkillValidation(artifact: EvolutionArtifact): artifact is SkillDraftValidationReport { + return "artifactKind" in artifact && artifact.artifactKind === "skill-draft-validation"; +} + function retryDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number { return Math.min(maxDelayMs, baseDelayMs * 2 ** Math.max(0, attempt - 1)); } @@ -452,7 +459,11 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor attempt, draftedGroups.map((group) => group.groupId), ); - const validatedGroupIds: string[] = []; + const evaluationGroups = new Map( + result.groups + .filter((group) => group.state === "structurally-valid" || group.state === "evaluation-pending") + .map((group) => [group.groupId, group]), + ); for (const group of draftedGroups) { signal?.throwIfAborted(); if (group.draft === null) throw new Error(`Drafted candidate group "${group.groupId}" has no draft`); @@ -469,7 +480,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor }); validationReference = await this.#dependencies.artifacts.put(validation); } - await this.#dependencies.registry.recordValidation({ + const validated = await this.#dependencies.registry.recordValidation({ scope: job.metadata.scope, groupId: group.groupId, draft: group.draft, @@ -478,10 +489,57 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor idempotencyKey: group.validationIdempotencyKey, createdAt: this.#dependencies.clock.now(), }); - validatedGroupIds.push(group.groupId); + if (validated.state === "structurally-valid") evaluationGroups.set(validated.groupId, validated); + } + await this.#appendCheckpoint(job, "evaluating", attempt, [...evaluationGroups.keys()]); + const evaluatedGroupIds: string[] = []; + if (this.#dependencies.evaluator !== undefined) { + for (const evaluationGroup of evaluationGroups.values()) { + signal?.throwIfAborted(); + if (evaluationGroup.draft === null || evaluationGroup.validation === null) { + throw new Error(`Candidate group "${evaluationGroup.groupId}" lacks evaluation prerequisites`); + } + const draftArtifact = await this.#dependencies.artifacts.get(evaluationGroup.draft); + const validationArtifact = await this.#dependencies.artifacts.get(evaluationGroup.validation); + if (!isSkillDraft(draftArtifact) || !isSkillValidation(validationArtifact)) { + throw new Error(`Candidate group "${evaluationGroup.groupId}" evaluation prerequisites are invalid`); + } + const pending = await this.#dependencies.registry.startEvaluation({ + scope: job.metadata.scope, + groupId: evaluationGroup.groupId, + draft: evaluationGroup.draft, + validation: evaluationGroup.validation, + proposalRevisionIds: evaluationGroup.proposalRevisions.map((revision) => revision.revisionId), + idempotencyKey: evaluationGroup.evaluationIdempotencyKey, + createdAt: this.#dependencies.clock.now(), + }); + let evaluationReference = pending.evaluation; + if (evaluationReference === null) { + const evaluation = await this.#dependencies.evaluator.evaluate({ + group: pending, + draft: draftArtifact, + validation: validationArtifact, + idempotencyKey: pending.evaluationIdempotencyKey, + createdAt: this.#dependencies.clock.now(), + signal, + }); + evaluationReference = await this.#dependencies.artifacts.put(evaluation); + } + await this.#dependencies.registry.recordEvaluation({ + scope: job.metadata.scope, + groupId: pending.groupId, + draft: evaluationGroup.draft, + validation: evaluationGroup.validation, + evaluation: evaluationReference, + proposalRevisionIds: pending.proposalRevisions.map((revision) => revision.revisionId), + idempotencyKey: pending.evaluationIdempotencyKey, + createdAt: this.#dependencies.clock.now(), + }); + evaluatedGroupIds.push(pending.groupId); + } } await this.#appendCheckpoint(job, "completed", attempt, [ - ...new Set([...processedGroupIds, ...validatedGroupIds]), + ...new Set([...processedGroupIds, ...evaluationGroups.keys(), ...evaluatedGroupIds]), ]); } catch (error) { if (signal?.aborted) throw error; diff --git a/packages/evolution/src/skill-evaluator.ts b/packages/evolution/src/skill-evaluator.ts new file mode 100644 index 000000000..9b159207d --- /dev/null +++ b/packages/evolution/src/skill-evaluator.ts @@ -0,0 +1,500 @@ +import { validateHarnessSkillContent } from "@earendil-works/diffex-harness-state"; +import type { + EvolutionArtifactProducer, + EvolutionArtifactReference, + SkillBehaviorEvaluationOutcome, + SkillBehaviorEvaluationResult, + SkillCandidate, + SkillEvaluationCase, + SkillEvaluationGate, + SkillEvaluationMetrics, + SkillEvaluationPoolSkill, + SkillEvaluationReport, + SkillRetrievalEvaluationResult, +} from "./contracts.ts"; +import { SKILL_EVALUATION_SCHEMA_VERSION } from "./contracts.ts"; +import type { + EvolutionArtifact, + EvolutionArtifactStore, + SkillEvaluationCaseSource, + SkillEvaluationEnvironmentSource, + SkillEvaluationExecutor, + SkillEvaluator, + SkillEvaluatorInput, +} from "./ports.ts"; +import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; + +export const SKILL_EVALUATOR_VERSION = "skill-evaluator-v1"; + +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; + +export interface DeterministicSkillEvaluatorDependencies { + readonly version: string; + readonly artifacts: EvolutionArtifactStore; + readonly cases: SkillEvaluationCaseSource; + readonly environment: SkillEvaluationEnvironmentSource; + readonly executor: SkillEvaluationExecutor; +} + +function uniqueSorted(values: readonly string[]): string[] { + return [...new Set(values)].sort((left, right) => left.localeCompare(right)); +} + +function sameValues(left: readonly string[], right: readonly string[]): boolean { + const normalizedLeft = uniqueSorted(left); + const normalizedRight = uniqueSorted(right); + return ( + normalizedLeft.length === normalizedRight.length && + normalizedLeft.every((value, index) => value === normalizedRight[index]) + ); +} + +function sameReference(left: EvolutionArtifactReference | null, right: EvolutionArtifactReference): boolean { + return left?.artifactId === right.artifactId && left.contentDigest === right.contentDigest; +} + +function contentWithoutMetadata(value: T): Omit { + const { metadata: _metadata, ...content } = value; + return content; +} + +function isSkillCandidate(artifact: EvolutionArtifact): artifact is SkillCandidate { + return "artifactKind" in artifact && artifact.artifactKind === "skill-candidate"; +} + +function mean(values: readonly number[]): number { + return values.length === 0 ? 0 : values.reduce((total, value) => total + value, 0) / values.length; +} + +function gate(reasonCodes: readonly string[]): SkillEvaluationGate { + const normalized = uniqueSorted(reasonCodes); + return { passed: normalized.length === 0, reasonCodes: normalized }; +} + +function candidateSkill(input: SkillEvaluatorInput): SkillEvaluationPoolSkill { + const frontmatter = validateHarnessSkillContent(`skills/${input.draft.proposedName}/SKILL.md`, input.draft.content); + return { + skillId: input.draft.draftId, + name: frontmatter.name, + description: frontmatter.description, + source: "candidate", + version: input.draft.metadata.contentDigest, + contentDigest: computeEvolutionContentDigest(input.draft.content), + content: input.draft.content, + }; +} + +function suiteIssues( + input: SkillEvaluatorInput, + testCases: readonly SkillEvaluationCase[], + skills: readonly SkillEvaluationPoolSkill[], + candidate: SkillEvaluationPoolSkill, +): string[] { + const issues: string[] = []; + const caseIds = new Set(); + const lineages = new Set(); + const poolIds = new Set(skills.map((skill) => skill.skillId)); + const sourceLineages = new Set([ + ...input.group.evidence.supportingTaskLineageIds, + ...input.group.evidence.contradictingTaskLineageIds, + ]); + const sourceFamilies = new Set( + testCases + .filter((testCase) => testCase.kind === "intended" && testCase.origin === "source-replay") + .map((testCase) => testCase.taskFamily), + ); + const heldOutFamilies = new Set( + testCases + .filter((testCase) => testCase.kind === "intended" && testCase.origin === "held-out") + .map((testCase) => testCase.taskFamily), + ); + const replayCases = testCases.filter((testCase) => testCase.origin === "source-replay"); + const replayLineages = new Set(replayCases.map((testCase) => testCase.taskLineageId)); + const replayTraceIds = new Set(replayCases.flatMap((testCase) => testCase.sourceTraceId ?? [])); + + if (sourceFamilies.size === 0) issues.push("source-replay-cases-missing"); + if (input.group.evidence.supportingTaskLineageIds.some((lineageId) => !replayLineages.has(lineageId))) { + issues.push("source-replay-lineage-coverage-incomplete"); + } + if (input.draft.sourceTraceIds.some((traceId) => !replayTraceIds.has(traceId))) { + issues.push("source-replay-trace-coverage-incomplete"); + } + if (heldOutFamilies.size === 0) issues.push("held-out-intended-cases-missing"); + if (!testCases.some((testCase) => testCase.kind === "hard-negative" && testCase.origin === "held-out")) { + issues.push("held-out-hard-negative-cases-missing"); + } + if (!testCases.some((testCase) => testCase.kind === "unrelated" && testCase.origin === "held-out")) { + issues.push("held-out-unrelated-cases-missing"); + } + for (const taskFamily of sourceFamilies) { + if (!heldOutFamilies.has(taskFamily)) issues.push("supported-task-family-lacks-held-out-case"); + } + + for (const testCase of testCases) { + if (testCase.caseId.trim() === "" || caseIds.has(testCase.caseId)) issues.push("case-identity-invalid"); + caseIds.add(testCase.caseId); + if (testCase.taskFamily.trim() === "" || testCase.prompt.trim() === "") issues.push("case-content-invalid"); + if (testCase.taskLineageId.trim() === "" || lineages.has(testCase.taskLineageId)) { + issues.push("case-lineage-invalid"); + } + lineages.add(testCase.taskLineageId); + if (testCase.verifier.name.trim() === "" || testCase.verifier.version.trim() === "") { + issues.push("deterministic-verifier-invalid"); + } + if (testCase.origin === "source-replay") { + if ( + testCase.kind !== "intended" || + testCase.sourceTraceId === null || + !input.draft.sourceTraceIds.includes(testCase.sourceTraceId) || + !input.group.evidence.supportingTaskLineageIds.includes(testCase.taskLineageId) + ) { + issues.push("source-replay-provenance-invalid"); + } + } else if (testCase.sourceTraceId !== null || sourceLineages.has(testCase.taskLineageId)) { + issues.push("held-out-lineage-invalid"); + } + if (testCase.kind === "intended") { + if (!testCase.expectedSkillIds.includes(candidate.skillId)) issues.push("intended-expectation-invalid"); + } else if (testCase.expectedSkillIds.includes(candidate.skillId)) { + issues.push("negative-expectation-invalid"); + } + if (testCase.kind === "hard-negative" && testCase.confusableSkillIds.length === 0) { + issues.push("hard-negative-neighbor-missing"); + } + if ( + testCase.expectedSkillIds.some((skillId) => !poolIds.has(skillId)) || + testCase.confusableSkillIds.some((skillId) => skillId === candidate.skillId || !poolIds.has(skillId)) + ) { + issues.push("case-skill-reference-invalid"); + } + } + return uniqueSorted(issues); +} + +function assertSelectionResult( + result: SkillRetrievalEvaluationResult, + testCase: SkillEvaluationCase, + pool: readonly SkillEvaluationPoolSkill[], +): void { + const poolIds = new Set(pool.map((skill) => skill.skillId)); + if ( + result.caseId !== testCase.caseId || + new Set(result.selectedSkillIds).size !== result.selectedSkillIds.length || + result.selectedSkillIds.some((skillId) => !poolIds.has(skillId)) || + new Set(result.scores.map((score) => score.skillId)).size !== result.scores.length || + result.scores.some( + (score) => !poolIds.has(score.skillId) || !Number.isFinite(score.score) || score.score < 0 || score.score > 1, + ) + ) { + throw new Error(`Retrieval result for evaluation case "${testCase.caseId}" is invalid`); + } +} + +function assertBehaviorOutcome( + outcome: SkillBehaviorEvaluationOutcome, + testCase: SkillEvaluationCase, + pool: readonly SkillEvaluationPoolSkill[], +): void { + const poolIds = new Set(pool.map((skill) => skill.skillId)); + if ( + !Number.isFinite(outcome.score) || + outcome.score < 0 || + outcome.score > 1 || + new Set(outcome.appliedSkillIds).size !== outcome.appliedSkillIds.length || + outcome.appliedSkillIds.some((skillId) => !poolIds.has(skillId)) || + outcome.verifier.kind !== testCase.verifier.kind || + outcome.verifier.name !== testCase.verifier.name || + outcome.verifier.version !== testCase.verifier.version || + outcome.verifier.configurationDigest !== testCase.verifier.configurationDigest + ) { + throw new Error(`Behavior result for evaluation case "${testCase.caseId}" is invalid`); + } +} + +function retrievalGate( + testCases: readonly SkillEvaluationCase[], + results: readonly SkillRetrievalEvaluationResult[], + candidateSkillId: string, +): SkillEvaluationGate { + const reasons: string[] = []; + for (const result of results) { + const testCase = testCases.find((candidate) => candidate.caseId === result.caseId); + if (testCase === undefined) throw new Error(`Unknown retrieval evaluation case "${result.caseId}"`); + const selectedCandidate = result.selectedSkillIds.includes(candidateSkillId); + if (testCase.kind === "intended" && !selectedCandidate) reasons.push("intended-case-not-retrieved"); + if (testCase.kind !== "intended" && selectedCandidate) reasons.push("candidate-retrieved-for-negative-case"); + } + return gate(reasons); +} + +function behaviorGate( + testCases: readonly SkillEvaluationCase[], + results: readonly SkillBehaviorEvaluationResult[], + candidateSkillId: string, +): SkillEvaluationGate { + const reasons: string[] = []; + const heldOutIntended: SkillBehaviorEvaluationResult[] = []; + for (const result of results) { + const testCase = testCases.find((candidate) => candidate.caseId === result.caseId); + if (testCase === undefined) throw new Error(`Unknown behavior evaluation case "${result.caseId}"`); + if (testCase.origin === "source-replay" && !result.fullPool.passed) { + reasons.push("source-replay-not-reproduced"); + } + if (testCase.origin === "held-out" && testCase.kind === "intended") { + heldOutIntended.push(result); + if (result.fullPool.score < result.baseline.score) reasons.push("held-out-intended-regression"); + if (!result.fullPool.appliedSkillIds.includes(candidateSkillId)) + reasons.push("intended-candidate-not-applied"); + } + if (testCase.kind !== "intended") { + if (result.fullPool.score < result.baseline.score || (result.baseline.passed && !result.fullPool.passed)) { + reasons.push("negative-case-regression"); + } + if (result.fullPool.appliedSkillIds.includes(candidateSkillId)) reasons.push("candidate-misapplied"); + } + } + if ( + heldOutIntended.length === 0 || + mean(heldOutIntended.map((result) => result.fullPool.score)) <= + mean(heldOutIntended.map((result) => result.baseline.score)) + ) { + reasons.push("held-out-intended-improvement-missing"); + } + return gate(reasons); +} + +function metrics( + testCases: readonly SkillEvaluationCase[], + retrieval: readonly SkillRetrievalEvaluationResult[], + behavior: readonly SkillBehaviorEvaluationResult[], + candidateSkillId: string, +): SkillEvaluationMetrics { + const exactSelections = retrieval.filter((result) => { + const testCase = testCases.find((candidate) => candidate.caseId === result.caseId); + return testCase !== undefined && sameValues(result.selectedSkillIds, testCase.expectedSkillIds); + }); + const sourceReplay = behavior.filter((result) => + testCases.some((testCase) => testCase.caseId === result.caseId && testCase.origin === "source-replay"), + ); + const heldOutIntended = behavior.filter((result) => + testCases.some( + (testCase) => + testCase.caseId === result.caseId && testCase.origin === "held-out" && testCase.kind === "intended", + ), + ); + const negative = behavior.filter((result) => + testCases.some((testCase) => testCase.caseId === result.caseId && testCase.kind !== "intended"), + ); + const baseline = mean(heldOutIntended.map((result) => result.baseline.score)); + const fullPool = mean(heldOutIntended.map((result) => result.fullPool.score)); + return { + exactRetrievalAccuracy: retrieval.length === 0 ? 0 : exactSelections.length / retrieval.length, + sourceReplayReproducibility: + sourceReplay.length === 0 + ? 0 + : sourceReplay.filter((result) => result.fullPool.passed).length / sourceReplay.length, + heldOutIntendedBaselineScore: baseline, + heldOutIntendedFullPoolScore: fullPool, + heldOutIntendedImprovement: fullPool - baseline, + negativeRegressionCount: negative.filter( + (result) => + result.fullPool.score < result.baseline.score || (result.baseline.passed && !result.fullPool.passed), + ).length, + misapplicationCount: negative.filter((result) => result.fullPool.appliedSkillIds.includes(candidateSkillId)) + .length, + }; +} + +export class DeterministicSkillEvaluator implements SkillEvaluator { + readonly #dependencies: DeterministicSkillEvaluatorDependencies; + readonly #producer: EvolutionArtifactProducer; + + constructor(dependencies: DeterministicSkillEvaluatorDependencies) { + this.#dependencies = dependencies; + this.#producer = { + name: "diffex-skill-evaluator", + version: dependencies.version, + promptVersion: null, + model: dependencies.executor.configuration.model, + }; + } + + async evaluate(input: SkillEvaluatorInput): Promise { + input.signal?.throwIfAborted(); + if ( + input.group.state !== "evaluation-pending" || + input.idempotencyKey !== input.group.evaluationIdempotencyKey || + !sameReference(input.group.draft, { + artifactId: input.draft.draftId, + contentDigest: computeEvolutionContentDigest(input.draft), + }) || + !sameReference(input.group.validation, { + artifactId: input.validation.validationId, + contentDigest: computeEvolutionContentDigest(input.validation), + }) || + input.validation.disposition !== "accepted" + ) { + throw new Error(`Candidate group "${input.group.groupId}" is not ready for evaluation`); + } + const candidates: SkillCandidate[] = []; + for (const revision of input.group.proposalRevisions) { + const artifact = await this.#dependencies.artifacts.get(revision.candidate); + if ( + !isSkillCandidate(artifact) || + artifact.candidateId !== revision.candidateId || + artifact.metadata.contentDigest !== computeEvolutionContentDigest(contentWithoutMetadata(artifact)) || + computeEvolutionContentDigest(artifact) !== revision.candidate.contentDigest + ) { + throw new Error(`Candidate revision "${revision.revisionId}" is invalid during evaluation`); + } + candidates.push(artifact); + } + const environment = await this.#dependencies.environment.capture(input.group.profile.workspace, input.signal); + if ( + environment.workspace.repositoryId !== input.group.profile.workspace.repositoryId || + environment.workspace.directoryId !== input.group.profile.workspace.directoryId || + environment.harnessVersion !== input.validation.validatedHarnessVersion + ) { + throw new Error("Skill evaluation environment no longer matches deterministic validation"); + } + const candidate = candidateSkill(input); + const configuration = this.#dependencies.executor.configuration; + if ( + configuration.model.provider.trim() === "" || + configuration.model.model.trim() === "" || + !DIGEST_PATTERN.test(configuration.model.configurationDigest) || + [configuration.runtime, configuration.retriever, configuration.executor].some( + (component) => + component.name.trim() === "" || + component.version.trim() === "" || + !DIGEST_PATTERN.test(component.configurationDigest), + ) + ) { + throw new Error("Skill evaluation runtime configuration is invalid"); + } + if ( + new Set(environment.skills.map((skill) => skill.skillId)).size !== environment.skills.length || + environment.skills.some( + (skill) => + skill.source !== "catalog" || + skill.skillId === candidate.skillId || + skill.name.trim() === "" || + skill.description.trim() === "", + ) + ) { + throw new Error("Skill evaluation environment contains an invalid skill pool"); + } + const testCases = [ + ...(await this.#dependencies.cases.build({ + group: input.group, + draft: input.draft, + validation: input.validation, + candidates, + environment, + candidateSkill: candidate, + signal: input.signal, + })), + ].sort((left, right) => left.caseId.localeCompare(right.caseId)); + const fullPool = [candidate, ...environment.skills]; + const suite = gate(suiteIssues(input, testCases, fullPool, candidate)); + const retrievalResults: SkillRetrievalEvaluationResult[] = []; + const behaviorResults: SkillBehaviorEvaluationResult[] = []; + if (suite.passed) { + for (const testCase of testCases) { + input.signal?.throwIfAborted(); + const selection = await this.#dependencies.executor.select({ + evaluationIdempotencyKey: input.idempotencyKey, + harnessVersion: environment.harnessVersion, + testCase, + skills: fullPool, + signal: input.signal, + }); + assertSelectionResult(selection, testCase, fullPool); + retrievalResults.push(selection); + const baseline = await this.#dependencies.executor.execute({ + evaluationIdempotencyKey: input.idempotencyKey, + harnessVersion: environment.harnessVersion, + testCase, + skills: environment.skills, + arm: "baseline", + signal: input.signal, + }); + assertBehaviorOutcome(baseline, testCase, environment.skills); + const candidateAvailable = await this.#dependencies.executor.execute({ + evaluationIdempotencyKey: input.idempotencyKey, + harnessVersion: environment.harnessVersion, + testCase, + skills: fullPool, + arm: "full-pool", + signal: input.signal, + }); + assertBehaviorOutcome(candidateAvailable, testCase, fullPool); + behaviorResults.push({ caseId: testCase.caseId, baseline, fullPool: candidateAvailable }); + } + } + const retrieval = suite.passed + ? retrievalGate(testCases, retrievalResults, candidate.skillId) + : gate(["evaluation-suite-invalid"]); + const behavior = suite.passed + ? behaviorGate(testCases, behaviorResults, candidate.skillId) + : gate(["evaluation-suite-invalid"]); + const neighboringSkillIds = new Set(testCases.flatMap((testCase) => testCase.confusableSkillIds)); + const neighboringSkills = environment.skills.filter((skill) => neighboringSkillIds.has(skill.skillId)); + const draftReference = { + artifactId: input.draft.draftId, + contentDigest: computeEvolutionContentDigest(input.draft), + }; + const validationReference = { + artifactId: input.validation.validationId, + contentDigest: computeEvolutionContentDigest(input.validation), + }; + const content = { + artifactKind: "skill-evaluation" as const, + schemaVersion: SKILL_EVALUATION_SCHEMA_VERSION, + evaluationId: `skill-evaluation-${computeEvolutionContentDigest({ + groupId: input.group.groupId, + idempotencyKey: input.idempotencyKey, + draftReference, + validationReference, + configuration, + testCases, + retrievalResults, + behaviorResults, + }).slice("sha256:".length, "sha256:".length + 32)}`, + groupId: input.group.groupId, + evaluationIdempotencyKey: input.idempotencyKey, + proposalRevisionIds: input.group.proposalRevisions.map((revision) => revision.revisionId), + draft: draftReference, + validation: validationReference, + evaluatorVersion: SKILL_EVALUATOR_VERSION, + assurance: "retrieval-and-deterministic-behavior-gates" as const, + disposition: + suite.passed && retrieval.passed && behavior.passed ? ("accepted" as const) : ("quarantined" as const), + harnessVersion: environment.harnessVersion, + candidateSkill: candidate, + neighboringSkills, + configuration, + cases: testCases, + retrievalResults, + behaviorResults, + metrics: metrics(testCases, retrievalResults, behaviorResults, candidate.skillId), + gates: { suite, retrieval, behavior }, + }; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: content.evaluationId, + producer: this.#producer, + sourceIds: uniqueSorted([ + input.draft.draftId, + input.validation.validationId, + ...input.draft.sourceCandidateIds, + ...testCases.map((testCase) => testCase.caseId), + ]), + scope: input.draft.metadata.scope, + createdAt: input.createdAt, + content, + }), + ...content, + }; + } +} diff --git a/packages/evolution/test/filesystem-store.test.ts b/packages/evolution/test/filesystem-store.test.ts index 04692b3c5..1c0983887 100644 --- a/packages/evolution/test/filesystem-store.test.ts +++ b/packages/evolution/test/filesystem-store.test.ts @@ -137,6 +137,33 @@ function candidateWorkEvent(eventId: string, registrySequence: number): SkillCan }; } +function candidateEvaluationEvent(eventId: string, registrySequence: number): SkillCandidateRegistryEvent { + const reference = (artifactId: string) => ({ artifactId, contentDigest: `sha256:${"a".repeat(64)}` }); + const content = { + schemaVersion: SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, + eventId, + registrySequence, + type: "skill-candidate-evaluation-recorded" as const, + groupId: "group-1", + draft: reference("draft-1"), + validation: reference("validation-1"), + evaluation: reference("evaluation-1"), + proposalRevisionIds: ["proposal-revision-1"], + idempotencyKey: "evaluation-key-1", + }; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: eventId, + producer, + sourceIds: ["group-1"], + scope, + createdAt: "2026-08-19T12:00:00.000Z", + content, + }), + ...content, + }; +} + async function scopeLease(store: FileEvolutionPersistence, ownerId: string): Promise { const lease = await store.acquire( "scope", @@ -207,13 +234,14 @@ describe("filesystem evolution persistence", () => { const peer = new FileEvolutionPersistence(root); const first = candidateWorkEvent("candidate-event-1", 1); - expect(await store.appendSkillCandidateRegistryEvents(scope, 0, [first])).toBe("committed"); - expect(await peer.loadSkillCandidateRegistry(scope)).toEqual({ revision: 1, events: [first] }); + const evaluation = candidateEvaluationEvent("candidate-event-2", 2); + expect(await store.appendSkillCandidateRegistryEvents(scope, 0, [first, evaluation])).toBe("committed"); + expect(await peer.loadSkillCandidateRegistry(scope)).toEqual({ revision: 2, events: [first, evaluation] }); expect( - await peer.appendSkillCandidateRegistryEvents(scope, 0, [candidateWorkEvent("candidate-event-2", 1)]), + await peer.appendSkillCandidateRegistryEvents(scope, 0, [candidateWorkEvent("candidate-event-3", 1)]), ).toBe("conflict"); await expect( - peer.appendSkillCandidateRegistryEvents(scope, 1, [candidateWorkEvent("candidate-event-2", 3)]), + peer.appendSkillCandidateRegistryEvents(scope, 2, [candidateWorkEvent("candidate-event-3", 4)]), ).rejects.toThrow("contiguous sequence"); }); diff --git a/packages/evolution/test/skill-candidate-registry.test.ts b/packages/evolution/test/skill-candidate-registry.test.ts index 5f106c76c..e9165f0dc 100644 --- a/packages/evolution/test/skill-candidate-registry.test.ts +++ b/packages/evolution/test/skill-candidate-registry.test.ts @@ -17,6 +17,7 @@ import { type SkillCandidateRegistryStore, type SkillDraft, type SkillDraftValidationReport, + type SkillEvaluationReport, } from "../src/index.ts"; const createdAt = "2026-08-25T12:00:00.000Z"; @@ -479,6 +480,91 @@ describe("persistent skill candidate registry", () => { reasonCodes: ["deterministic-draft-validation-passed"], }); + const evaluationStart = { + scope, + groupId: group.groupId, + draft: draftReference, + validation: validationReference, + proposalRevisionIds: draft.proposalRevisionIds, + idempotencyKey: validated.evaluationIdempotencyKey, + createdAt: "2026-08-26T12:03:00.000Z", + }; + const pending = await fixture.registry.startEvaluation(evaluationStart); + const pendingRetry = await fixture.registry.startEvaluation(evaluationStart); + expect(pending).toMatchObject({ state: "evaluation-pending", evaluation: null }); + expect(pendingRetry).toMatchObject({ state: "evaluation-pending", evaluation: null }); + const configurationDigest = `sha256:${"6".repeat(64)}`; + const evaluationContent = { + artifactKind: "skill-evaluation" as const, + schemaVersion: 1 as const, + evaluationId: "skill-evaluation-1", + groupId: group.groupId, + evaluationIdempotencyKey: pending.evaluationIdempotencyKey, + proposalRevisionIds: draft.proposalRevisionIds, + draft: draftReference, + validation: validationReference, + evaluatorVersion: "skill-evaluator-v1", + assurance: "retrieval-and-deterministic-behavior-gates" as const, + disposition: "accepted" as const, + harnessVersion: secondInput.expectedHarnessBaseVersion, + candidateSkill: { + skillId: draft.draftId, + name: draft.proposedName, + description: "Run a focused check after repository changes.", + source: "candidate" as const, + version: draft.metadata.contentDigest, + contentDigest: computeEvolutionContentDigest(draft.content), + content: draft.content, + }, + neighboringSkills: [], + configuration: { + model: { provider: "faux", model: "fixture", configurationDigest }, + runtime: { name: "node", version: "24", configurationDigest }, + retriever: { name: "fixture", version: "1", configurationDigest }, + executor: { name: "fixture", version: "1", configurationDigest }, + }, + cases: [], + retrievalResults: [], + behaviorResults: [], + metrics: { + exactRetrievalAccuracy: 1, + sourceReplayReproducibility: 1, + heldOutIntendedBaselineScore: 0, + heldOutIntendedFullPoolScore: 1, + heldOutIntendedImprovement: 1, + negativeRegressionCount: 0, + misapplicationCount: 0, + }, + gates: { + suite: { passed: true, reasonCodes: [] }, + retrieval: { passed: true, reasonCodes: [] }, + behavior: { passed: true, reasonCodes: [] }, + }, + }; + const evaluation: SkillEvaluationReport = { + metadata: metadata( + evaluationContent.evaluationId, + [draft.draftId, validation.validationId], + evaluationContent, + ), + ...evaluationContent, + }; + const evaluationReference = await fixture.artifacts.put(evaluation); + const evaluationRequest = { ...evaluationStart, evaluation: evaluationReference }; + store.interruptNextMultiAppend = true; + await expect(fixture.registry.recordEvaluation(evaluationRequest)).rejects.toThrow( + "interrupted draft registry append", + ); + const accepted = await fixture.registry.recordEvaluation(evaluationRequest); + const acceptedRetry = await fixture.registry.recordEvaluation(evaluationRequest); + expect(accepted).toMatchObject({ state: "accepted", evaluation: evaluationReference }); + expect(acceptedRetry).toMatchObject({ state: "accepted", evaluation: evaluationReference }); + expect(accepted.dispositionHistory.at(-1)).toMatchObject({ + from: "evaluation-pending", + to: "accepted", + reasonCodes: ["retrieval-and-behavior-gates-passed"], + }); + const thirdInput = input([evidence("support-c", "task-c", 0)], "trace-c"); const thirdCandidate = candidate({ candidateId: "candidate-c", @@ -495,10 +581,16 @@ describe("persistent skill candidate registry", () => { createdAt: "2026-08-27T12:00:00.000Z", }); - expect(revised.groups[0]).toMatchObject({ state: "evidence-qualified", draft: null }); + expect(revised.groups[0]).toMatchObject({ + state: "evidence-qualified", + draft: null, + validation: null, + evaluation: null, + }); expect(revised.groups[0]?.synthesisIdempotencyKey).not.toBe(recorded.synthesisIdempotencyKey); + expect(revised.groups[0]?.evaluationIdempotencyKey).not.toBe(accepted.evaluationIdempotencyKey); expect(revised.groups[0]?.dispositionHistory.at(-1)).toMatchObject({ - from: "structurally-valid", + from: "accepted", to: "evidence-qualified", reasonCodes: ["proposal-revision-set-changed"], }); diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index acb55f39f..0f94436e6 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -13,6 +13,8 @@ import { PollingSkillCandidateWorkSource, type SkillCandidate, type SkillCandidateDraftRecordingInput, + type SkillCandidateEvaluationRecordingInput, + type SkillCandidateEvaluationStartInput, type SkillCandidateGroup, type SkillCandidateRegistry, type SkillCandidateRegistryAppendResult, @@ -25,6 +27,8 @@ import { type SkillDraft, type SkillDraftValidationReport, type SkillDraftValidatorInput, + type SkillEvaluationReport, + type SkillEvaluatorInput, type SkillSynthesizerInput, } from "../src/index.ts"; @@ -180,6 +184,7 @@ function qualifiedGroup(candidateReference: EvolutionArtifactReference): SkillCa priorityScore: null, draft: null, validation: null, + evaluation: null, synthesisIdempotencyKey: "skill-synthesis-qualified", validationIdempotencyKey: "skill-validation-qualified", evaluationIdempotencyKey: "skill-evaluation-qualified", @@ -247,6 +252,8 @@ class FixtureRegistry implements SkillCandidateRegistry { readonly requests: SkillCandidateRegistryConsolidationInput[] = []; readonly draftRequests: SkillCandidateDraftRecordingInput[] = []; readonly validationRequests: SkillCandidateValidationRecordingInput[] = []; + readonly evaluationStartRequests: SkillCandidateEvaluationStartInput[] = []; + readonly evaluationRequests: SkillCandidateEvaluationRecordingInput[] = []; readonly groups: SkillCandidateGroup[] = []; failuresRemaining = 0; @@ -272,7 +279,26 @@ class FixtureRegistry implements SkillCandidateRegistry { this.validationRequests.push(input); const group = this.groups.find((candidateGroup) => candidateGroup.groupId === input.groupId); if (group === undefined) throw new Error("Unknown group"); - return { ...group, state: "structurally-valid", validation: input.validation }; + return { ...group, state: "structurally-valid", draft: input.draft, validation: input.validation }; + } + + async startEvaluation(input: SkillCandidateEvaluationStartInput): Promise { + this.evaluationStartRequests.push(input); + const group = this.groups.find((candidateGroup) => candidateGroup.groupId === input.groupId); + if (group === undefined) throw new Error("Unknown group"); + return { + ...group, + state: "evaluation-pending", + draft: input.draft, + validation: input.validation, + }; + } + + async recordEvaluation(input: SkillCandidateEvaluationRecordingInput): Promise { + this.evaluationRequests.push(input); + const group = this.groups.find((candidateGroup) => candidateGroup.groupId === input.groupId); + if (group === undefined) throw new Error("Unknown group"); + return { ...group, state: "accepted", evaluation: input.evaluation }; } async list(): Promise { @@ -280,7 +306,7 @@ class FixtureRegistry implements SkillCandidateRegistry { } } -function fixture() { +function fixture(withEvaluator = false) { const input = inputSnapshot(); const skillCandidate = candidate(); const inputReference = reference(input); @@ -291,6 +317,7 @@ function fixture() { const registry = new FixtureRegistry(); const synthesisRequests: SkillSynthesizerInput[] = []; const validationRequests: SkillDraftValidatorInput[] = []; + const evaluationRequests: SkillEvaluatorInput[] = []; let clockTime = now; const clock = { now: () => clockTime }; return { @@ -300,6 +327,7 @@ function fixture() { registry, synthesisRequests, validationRequests, + evaluationRequests, clock, setClock(value: string) { clockTime = value; @@ -360,6 +388,80 @@ function fixture() { return { metadata: metadata(content.validationId, [request.draft.draftId], content), ...content }; }, }, + ...(withEvaluator + ? { + evaluator: { + async evaluate(request: SkillEvaluatorInput): Promise { + evaluationRequests.push(request); + const draftReference = reference(request.draft); + const validationReference = reference(request.validation); + const content = { + artifactKind: "skill-evaluation" as const, + schemaVersion: 1 as const, + evaluationId: "evaluation-1", + groupId: request.group.groupId, + evaluationIdempotencyKey: request.idempotencyKey, + proposalRevisionIds: request.group.proposalRevisions.map((revision) => revision.revisionId), + draft: draftReference, + validation: validationReference, + evaluatorVersion: "skill-evaluator-v1", + assurance: "retrieval-and-deterministic-behavior-gates" as const, + disposition: "accepted" as const, + harnessVersion, + candidateSkill: { + skillId: request.draft.draftId, + name: request.draft.proposedName, + description: "Fixture skill.", + source: "candidate" as const, + version: request.draft.metadata.contentDigest, + contentDigest: computeEvolutionContentDigest(request.draft.content), + content: request.draft.content, + }, + neighboringSkills: [], + configuration: { + model: { + provider: "faux", + model: "fixture", + configurationDigest: `sha256:${"4".repeat(64)}`, + }, + runtime: { name: "node", version: "24", configurationDigest: `sha256:${"5".repeat(64)}` }, + retriever: { + name: "fixture", + version: "1", + configurationDigest: `sha256:${"6".repeat(64)}`, + }, + executor: { + name: "fixture", + version: "1", + configurationDigest: `sha256:${"7".repeat(64)}`, + }, + }, + cases: [], + retrievalResults: [], + behaviorResults: [], + metrics: { + exactRetrievalAccuracy: 1, + sourceReplayReproducibility: 1, + heldOutIntendedBaselineScore: 0, + heldOutIntendedFullPoolScore: 1, + heldOutIntendedImprovement: 1, + negativeRegressionCount: 0, + misapplicationCount: 0, + }, + gates: { + suite: { passed: true, reasonCodes: [] }, + retrieval: { passed: true, reasonCodes: [] }, + behavior: { passed: true, reasonCodes: [] }, + }, + }; + return { + metadata: metadata(content.evaluationId, [request.draft.draftId], content), + ...content, + }; + }, + }, + } + : {}), model: { maxProviderCallsPerExecution: 20, execute: () => Promise.reject(new Error("Fixture synthesizer does not call the model")), @@ -393,7 +495,7 @@ describe("skill candidate registry worker", () => { data.store.events.flatMap((event) => event.type === "skill-candidate-work-checkpointed" ? [event.stage] : [], ), - ).toEqual(["consolidating", "scoring", "synthesizing", "validating", "completed"]); + ).toEqual(["consolidating", "scoring", "synthesizing", "validating", "evaluating", "completed"]); expect( data.store.events .filter((event) => event.type === "skill-candidate-work-checkpointed") @@ -404,6 +506,7 @@ describe("skill candidate registry worker", () => { expect.stringMatching(/^skill-candidate-work-stage-/), expect.stringMatching(/^skill-candidate-work-stage-/), expect.stringMatching(/^skill-candidate-work-stage-/), + expect.stringMatching(/^skill-candidate-work-stage-/), ]); }); @@ -443,6 +546,35 @@ describe("skill candidate registry worker", () => { ]); }); + it("evaluates structurally valid drafts and records the gated disposition", async () => { + const data = fixture(true); + const candidateReference = data.job.checkpoints[0]?.outputArtifacts[0]; + if (candidateReference === undefined) throw new Error("Expected candidate reference"); + data.registry.groups.push(qualifiedGroup(candidateReference)); + + await data.worker.process(data.job.jobId); + + expect(data.evaluationRequests).toEqual([ + expect.objectContaining({ + group: expect.objectContaining({ groupId: "group-qualified", state: "evaluation-pending" }), + idempotencyKey: "skill-evaluation-qualified", + }), + ]); + expect(data.registry.evaluationStartRequests).toEqual([ + expect.objectContaining({ + groupId: "group-qualified", + idempotencyKey: "skill-evaluation-qualified", + }), + ]); + expect(data.registry.evaluationRequests).toEqual([ + expect.objectContaining({ + groupId: "group-qualified", + idempotencyKey: "skill-evaluation-qualified", + evaluation: expect.objectContaining({ artifactId: "evaluation-1" }), + }), + ]); + }); + it("persists bounded retry timing without marking failed work complete", async () => { const data = fixture(); data.registry.failuresRemaining = 1; diff --git a/packages/evolution/test/skill-draft-validator.test.ts b/packages/evolution/test/skill-draft-validator.test.ts index 35a36df6c..b4d6af2a5 100644 --- a/packages/evolution/test/skill-draft-validator.test.ts +++ b/packages/evolution/test/skill-draft-validator.test.ts @@ -391,6 +391,7 @@ function fixture( priorityScore: null, draft: draftReference, validation: null, + evaluation: null, synthesisIdempotencyKey: "skill-synthesis-1", validationIdempotencyKey: "skill-validation-1", evaluationIdempotencyKey: "skill-evaluation-1", diff --git a/packages/evolution/test/skill-evaluator.test.ts b/packages/evolution/test/skill-evaluator.test.ts new file mode 100644 index 000000000..94ef76d74 --- /dev/null +++ b/packages/evolution/test/skill-evaluator.test.ts @@ -0,0 +1,532 @@ +import { describe, expect, it } from "vitest"; +import { + computeEvolutionContentDigest, + createEvolutionArtifactMetadata, + DeterministicSkillEvaluator, + type EvolutionArtifact, + type EvolutionArtifactProducer, + type EvolutionArtifactReference, + type EvolutionArtifactStore, + type EvolutionScope, + type SkillBehaviorEvaluationOutcome, + type SkillCandidate, + type SkillCandidateGroup, + type SkillDraft, + type SkillDraftValidationReport, + type SkillEvaluationCase, + type SkillEvaluationExecutionInput, + type SkillEvaluationPoolSkill, + type SkillEvaluationSelectionInput, + type SkillEvaluationVerifier, +} from "../src/index.ts"; + +const createdAt = "2026-08-25T12:00:00.000Z"; +const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; +const harnessVersion = `sha256:${"2".repeat(64)}`; +const configurationDigest = `sha256:${"3".repeat(64)}`; +const producer: EvolutionArtifactProducer = { + name: "fixture", + version: "1", + promptVersion: null, + model: null, +}; +const verifier: SkillEvaluationVerifier = { + kind: "test", + name: "fixture-verifier", + version: "1", + configurationDigest, +}; + +function metadata(artifactId: string, sourceIds: readonly string[], content: unknown) { + return createEvolutionArtifactMetadata({ artifactId, producer, sourceIds, scope, createdAt, content }); +} + +function reference(artifact: EvolutionArtifact): EvolutionArtifactReference { + return { + artifactId: artifact.metadata.artifactId, + contentDigest: computeEvolutionContentDigest(artifact), + }; +} + +function candidate(candidateId: string, lineageId: string, traceId: string): SkillCandidate { + const content = { + artifactKind: "skill-candidate" as const, + candidateId, + operation: "CREATE" as const, + workspace: { + repositoryId: "diffex", + directoryId: "directory-1", + rootPath: "/repo", + repositoryRevision: "revision-1", + evolutionScope: scope, + }, + proposedName: "focused-verification", + existingSkillIds: [], + sourceLessonIds: [`lesson-${candidateId}`], + sourceProceduralObservationIds: [], + supportingEvidenceIds: [`evidence-${candidateId}`], + contradictingEvidenceIds: [], + supportingTraceIds: [traceId], + contradictingTraceIds: [], + taskLineageIds: [lineageId], + skillability: { + classification: "verification-routine" as const, + rationale: "The evidence describes a repeatable verification routine.", + }, + trigger: { + description: "A repository change requires focused verification.", + positiveExamples: ["Verify a focused repository change."], + negativeExamples: ["No repository files changed."], + }, + preconditions: ["A focused test exists."], + procedure: { + steps: [{ sequence: 1, action: "Run the focused test.", condition: null }], + requiredTools: ["shell"], + environment: ["Repository checkout"], + }, + failureHandling: [{ signal: "The test fails.", mitigation: "Inspect the failure.", recovery: "Fix and rerun." }], + applicabilityBoundaries: ["Repositories with focused tests"], + verificationContract: { checks: ["The test passes."], completionCondition: "The test exits successfully." }, + memoryRelationship: { + classification: "independent" as const, + memoryProposalIds: [], + rationale: "The procedure is independent.", + }, + }; + return { metadata: metadata(candidateId, content.sourceLessonIds, content), ...content }; +} + +class MemoryArtifactStore implements EvolutionArtifactStore { + readonly artifacts = new Map(); + + constructor(artifacts: readonly EvolutionArtifact[]) { + for (const artifact of artifacts) this.artifacts.set(reference(artifact).contentDigest, artifact); + } + + async put(artifact: EvolutionArtifact): Promise { + const artifactReference = reference(artifact); + this.artifacts.set(artifactReference.contentDigest, artifact); + return artifactReference; + } + + async get(artifactReference: EvolutionArtifactReference): Promise { + const artifact = this.artifacts.get(artifactReference.contentDigest); + if (artifact === undefined) throw new Error("Unknown artifact"); + return artifact; + } +} + +function fixture( + options: { + readonly cases?: (candidateSkillId: string, neighborSkillId: string) => readonly SkillEvaluationCase[]; + readonly behavior?: ( + input: SkillEvaluationExecutionInput, + candidateSkillId: string, + ) => SkillBehaviorEvaluationOutcome; + } = {}, +) { + const first = candidate("candidate-a", "task-a", "trace-a"); + const second = candidate("candidate-b", "task-b", "trace-b"); + const revisions = [first, second].map((item, index) => ({ + revisionId: `revision-${index + 1}`, + candidateId: item.candidateId, + candidate: reference(item), + sourceEvolutionJobId: `job-${index + 1}`, + createdAt, + evidence: { + supportingEvidenceIds: item.supportingEvidenceIds, + contradictingEvidenceIds: [], + supportingTraceIds: item.supportingTraceIds, + contradictingTraceIds: [], + supportingTaskLineageIds: item.taskLineageIds, + contradictingTaskLineageIds: [], + }, + })); + const draftContent = { + artifactKind: "skill-draft" as const, + draftId: "draft-1", + groupId: "group-1", + synthesisIdempotencyKey: "synthesis-1", + proposalRevisionIds: revisions.map((revision) => revision.revisionId), + sourceCandidateIds: [first.candidateId, second.candidateId], + sourceEvidenceIds: [...first.supportingEvidenceIds, ...second.supportingEvidenceIds], + sourceTraceIds: ["trace-a", "trace-b"], + sourceSkillVersions: [], + repositoryInstructionDigests: [], + operation: "CREATE" as const, + proposedName: "focused-verification", + existingSkillIds: [], + workspace: second.workspace, + content: + "---\nname: focused-verification\ndescription: Run focused verification for repository changes.\n---\n\nRun the focused verification procedure.\n", + }; + const draft: SkillDraft = { + metadata: metadata(draftContent.draftId, draftContent.sourceCandidateIds, draftContent), + ...draftContent, + }; + const draftReference = reference(draft); + const validationContent = { + artifactKind: "skill-draft-validation" as const, + validationId: "validation-1", + groupId: "group-1", + draft: draftReference, + validatorVersion: "validator-1", + assurance: "structure-provenance-scope-and-security-only" as const, + disposition: "accepted" as const, + issues: [], + validatedHarnessVersion: harnessVersion, + validatedSkillVersions: [], + canonicalInventory: ["skills/focused-verification/SKILL.md"], + memoryRelationshipDigest: computeEvolutionContentDigest([]), + }; + const validation: SkillDraftValidationReport = { + metadata: metadata(validationContent.validationId, [draft.draftId], validationContent), + ...validationContent, + }; + const validationReference = reference(validation); + const group: SkillCandidateGroup = { + groupId: "group-1", + profile: { + workspace: { repositoryId: "diffex", directoryId: "directory-1", evolutionScope: scope }, + existingSkillIds: [], + skillability: "verification-routine", + triggerTerms: ["focused", "verification"], + proceduralIntentTerms: ["focused", "test"], + applicabilityTerms: ["repository"], + }, + proposalRevisions: revisions, + state: "evaluation-pending", + dispositionHistory: [], + evidence: { + supportingEvidenceIds: [...first.supportingEvidenceIds, ...second.supportingEvidenceIds], + contradictingEvidenceIds: [], + supportingTraceIds: ["trace-a", "trace-b"], + contradictingTraceIds: [], + supportingTaskLineageIds: ["task-a", "task-b"], + contradictingTaskLineageIds: [], + }, + priorityScore: null, + draft: draftReference, + validation: validationReference, + evaluation: null, + synthesisIdempotencyKey: "synthesis-1", + validationIdempotencyKey: "validation-key-1", + evaluationIdempotencyKey: "evaluation-key-1", + }; + const neighbor: SkillEvaluationPoolSkill = { + skillId: "neighbor-1", + name: "broad-verification", + description: "Run broad verification for release changes.", + source: "catalog", + version: "1", + contentDigest: computeEvolutionContentDigest("neighbor"), + content: "---\nname: broad-verification\ndescription: Run broad verification for release changes.\n---\n", + }; + const requests: Array = []; + const defaultCases = (candidateSkillId: string, neighborSkillId: string): readonly SkillEvaluationCase[] => [ + { + caseId: "replay-a", + taskFamily: "focused-change", + kind: "intended", + origin: "source-replay", + taskLineageId: "task-a", + sourceTraceId: "trace-a", + prompt: "Replay the original focused verification task.", + expectedSkillIds: [candidateSkillId], + confusableSkillIds: [neighborSkillId], + verifier, + }, + { + caseId: "replay-b", + taskFamily: "focused-change", + kind: "intended", + origin: "source-replay", + taskLineageId: "task-b", + sourceTraceId: "trace-b", + prompt: "Replay the second focused verification task.", + expectedSkillIds: [candidateSkillId], + confusableSkillIds: [neighborSkillId], + verifier, + }, + { + caseId: "held-out-positive", + taskFamily: "focused-change", + kind: "intended", + origin: "held-out", + taskLineageId: "held-out-a", + sourceTraceId: null, + prompt: "Verify a different focused repository change.", + expectedSkillIds: [candidateSkillId], + confusableSkillIds: [neighborSkillId], + verifier, + }, + { + caseId: "held-out-hard-negative", + taskFamily: "release-change", + kind: "hard-negative", + origin: "held-out", + taskLineageId: "held-out-b", + sourceTraceId: null, + prompt: "Verify a release with the broad verification procedure.", + expectedSkillIds: [neighborSkillId], + confusableSkillIds: [neighborSkillId], + verifier, + }, + { + caseId: "held-out-unrelated", + taskFamily: "documentation", + kind: "unrelated", + origin: "held-out", + taskLineageId: "held-out-c", + sourceTraceId: null, + prompt: "Rewrite a paragraph without changing code.", + expectedSkillIds: [], + confusableSkillIds: [], + verifier, + }, + ]; + const evaluator = new DeterministicSkillEvaluator({ + version: "1", + artifacts: new MemoryArtifactStore([first, second, draft, validation]), + cases: { + async build(input) { + return (options.cases ?? defaultCases)(input.candidateSkill.skillId, neighbor.skillId); + }, + }, + environment: { + async capture() { + return { + workspace: { repositoryId: "diffex", directoryId: "directory-1" }, + harnessVersion, + skills: [neighbor], + }; + }, + }, + executor: { + configuration: { + model: { provider: "faux", model: "fixture", configurationDigest }, + runtime: { name: "node", version: "24.0.0", configurationDigest }, + retriever: { name: "fixture-retriever", version: "1", configurationDigest }, + executor: { name: "fixture-executor", version: "1", configurationDigest }, + }, + async select(input) { + requests.push(input); + return { + caseId: input.testCase.caseId, + selectedSkillIds: input.testCase.expectedSkillIds, + scores: input.testCase.expectedSkillIds.map((skillId) => ({ skillId, score: 1 })), + }; + }, + async execute(input) { + requests.push(input); + const custom = options.behavior?.(input, draft.draftId); + if (custom !== undefined) return custom; + const intended = input.testCase.kind === "intended"; + return { + passed: input.arm === "full-pool" || !intended, + score: intended ? (input.arm === "full-pool" ? 1 : 0) : 1, + appliedSkillIds: intended && input.arm === "full-pool" ? [draft.draftId] : [], + verifier, + details: { deterministic: true }, + }; + }, + }, + }); + return { evaluator, group, draft, validation, requests }; +} + +describe("DeterministicSkillEvaluator", () => { + it("accepts only paired retrieval and behavioral improvement with held-out non-regression", async () => { + const data = fixture(); + + const report = await data.evaluator.evaluate({ + group: data.group, + draft: data.draft, + validation: data.validation, + idempotencyKey: data.group.evaluationIdempotencyKey, + createdAt, + }); + + expect(report).toMatchObject({ + artifactKind: "skill-evaluation", + disposition: "accepted", + harnessVersion, + configuration: { + model: { provider: "faux", model: "fixture", configurationDigest }, + runtime: { name: "node", version: "24.0.0", configurationDigest }, + }, + gates: { + suite: { passed: true, reasonCodes: [] }, + retrieval: { passed: true, reasonCodes: [] }, + behavior: { passed: true, reasonCodes: [] }, + }, + metrics: { + exactRetrievalAccuracy: 1, + sourceReplayReproducibility: 1, + heldOutIntendedImprovement: 1, + negativeRegressionCount: 0, + misapplicationCount: 0, + }, + }); + expect(report.cases).toHaveLength(5); + expect(report.retrievalResults).toHaveLength(5); + expect(report.behaviorResults).toHaveLength(5); + expect(data.requests).toHaveLength(15); + }); + + it("quarantines exact retrieval when downstream intended behavior does not improve", async () => { + const data = fixture({ + behavior(input, candidateSkillId) { + const intended = input.testCase.kind === "intended"; + return { + passed: true, + score: 1, + appliedSkillIds: intended && input.arm === "full-pool" ? [candidateSkillId] : [], + verifier, + details: { deterministic: true }, + }; + }, + }); + + const report = await data.evaluator.evaluate({ + group: data.group, + draft: data.draft, + validation: data.validation, + idempotencyKey: data.group.evaluationIdempotencyKey, + createdAt, + }); + + expect(report.metrics.exactRetrievalAccuracy).toBe(1); + expect(report.disposition).toBe("quarantined"); + expect(report.gates.retrieval.passed).toBe(true); + expect(report.gates.behavior).toEqual({ + passed: false, + reasonCodes: ["held-out-intended-improvement-missing"], + }); + }); + + it("quarantines negative-case regressions and candidate misapplication", async () => { + const data = fixture({ + behavior(input, candidateSkillId) { + if (input.testCase.kind === "hard-negative" && input.arm === "full-pool") { + return { + passed: false, + score: 0, + appliedSkillIds: [candidateSkillId], + verifier, + details: { deterministic: true }, + }; + } + const intended = input.testCase.kind === "intended"; + return { + passed: input.arm === "full-pool" || !intended, + score: intended ? (input.arm === "full-pool" ? 1 : 0) : 1, + appliedSkillIds: intended && input.arm === "full-pool" ? [candidateSkillId] : [], + verifier, + details: { deterministic: true }, + }; + }, + }); + + const report = await data.evaluator.evaluate({ + group: data.group, + draft: data.draft, + validation: data.validation, + idempotencyKey: data.group.evaluationIdempotencyKey, + createdAt, + }); + + expect(report.disposition).toBe("quarantined"); + expect(report.gates.behavior).toEqual({ + passed: false, + reasonCodes: ["candidate-misapplied", "negative-case-regression"], + }); + expect(report.metrics).toMatchObject({ negativeRegressionCount: 1, misapplicationCount: 1 }); + }); + + it("quarantines suites whose held-out lineage overlaps source evidence without executing cases", async () => { + const data = fixture({ + cases(candidateSkillId, neighborSkillId) { + return [ + { + caseId: "replay-a", + taskFamily: "focused-change", + kind: "intended", + origin: "source-replay", + taskLineageId: "task-a", + sourceTraceId: "trace-a", + prompt: "Replay the original task.", + expectedSkillIds: [candidateSkillId], + confusableSkillIds: [neighborSkillId], + verifier, + }, + { + caseId: "replay-b", + taskFamily: "focused-change", + kind: "intended", + origin: "source-replay", + taskLineageId: "task-b", + sourceTraceId: "trace-b", + prompt: "Replay the second original task.", + expectedSkillIds: [candidateSkillId], + confusableSkillIds: [neighborSkillId], + verifier, + }, + { + caseId: "held-out-positive", + taskFamily: "focused-change", + kind: "intended", + origin: "held-out", + taskLineageId: "task-b", + sourceTraceId: null, + prompt: "Overlapping held-out task.", + expectedSkillIds: [candidateSkillId], + confusableSkillIds: [neighborSkillId], + verifier, + }, + { + caseId: "hard-negative", + taskFamily: "release", + kind: "hard-negative", + origin: "held-out", + taskLineageId: "held-out-b", + sourceTraceId: null, + prompt: "Use the neighboring skill.", + expectedSkillIds: [neighborSkillId], + confusableSkillIds: [neighborSkillId], + verifier, + }, + { + caseId: "unrelated", + taskFamily: "docs", + kind: "unrelated", + origin: "held-out", + taskLineageId: "held-out-c", + sourceTraceId: null, + prompt: "Edit prose.", + expectedSkillIds: [], + confusableSkillIds: [], + verifier, + }, + ]; + }, + }); + + const report = await data.evaluator.evaluate({ + group: data.group, + draft: data.draft, + validation: data.validation, + idempotencyKey: data.group.evaluationIdempotencyKey, + createdAt, + }); + + expect(report.disposition).toBe("quarantined"); + expect(report.gates.suite).toEqual({ + passed: false, + reasonCodes: ["case-lineage-invalid", "held-out-lineage-invalid"], + }); + expect(report.retrievalResults).toEqual([]); + expect(report.behaviorResults).toEqual([]); + expect(data.requests).toEqual([]); + }); +}); diff --git a/packages/evolution/test/skill-synthesizer.test.ts b/packages/evolution/test/skill-synthesizer.test.ts index 5211df783..521ab253d 100644 --- a/packages/evolution/test/skill-synthesizer.test.ts +++ b/packages/evolution/test/skill-synthesizer.test.ts @@ -381,6 +381,7 @@ function fixture(state: SkillCandidateGroup["state"] = "evidence-qualified") { priorityScore: null, draft: null, validation: null, + evaluation: null, synthesisIdempotencyKey: "skill-synthesis-1", validationIdempotencyKey: "skill-validation-1", evaluationIdempotencyKey: "skill-evaluation-1",