diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index cf6351482..11eaaa2a0 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -12,6 +12,7 @@ import { classifyEvolutionEvidence, computeEvolutionContentDigest, DeterministicProposalRanker, + DeterministicSkillDraftValidator, EvolutionBackgroundWorker, EvolutionControlPlane, EvolutionControlService, @@ -879,6 +880,33 @@ export async function createCodingAgentEvolutionRuntime( artifacts: persistence, jobs: persistence, }); + const skillValidator = new DeterministicSkillDraftValidator({ + version: VERSION, + artifacts: persistence, + jobs: persistence, + environment: { + async capture(workspace, signal) { + signal?.throwIfAborted(); + if ( + workspace.repositoryId !== scope.repositoryId || + workspace.directoryId !== currentDirectory.directoryId + ) { + throw new Error("Skill validation workspace does not match the repository runtime"); + } + const harnessVersion = options.session.activeHarnessVersion; + if (harnessVersion === undefined) throw new Error("Skill validation requires an active session harness"); + return { + workspace: { repositoryId: scope.repositoryId, directoryId: currentDirectory.directoryId }, + harnessVersion, + skillVersions: options.resourceLoader.getSkills().skills.map((skill) => ({ + skillId: skill.skillId, + version: skill.version, + contentDigest: skill.contentDigest, + })), + }; + }, + }, + }); const candidateWorker = new SkillCandidateBackgroundWorker({ clock, source: new PollingSkillCandidateWorkSource({ clock, jobs: persistence, store: persistence }), @@ -890,6 +918,7 @@ export async function createCodingAgentEvolutionRuntime( store: persistence, registry: candidateRegistry, synthesizer: skillSynthesizer, + validator: skillValidator, model, }), priority, diff --git a/packages/evolution/RECOVERY.md b/packages/evolution/RECOVERY.md index f669fcc7d..888ad1d5a 100644 --- a/packages/evolution/RECOVERY.md +++ b/packages/evolution/RECOVERY.md @@ -26,7 +26,7 @@ Recovery never treats partial output as completed work and never feeds evolution | Successful rollback selection | The recorded immutable parent is still present and the evolved revision is current. | Select the parent with compare-and-swap and refresh on the next prompt boundary. | Existing in-flight prompts stay pinned to their admitted revision. | | Unsupported skill or orchestration target | The target registry has no enabled writer route. | Preserve the finding as deferred evidence with no executable payload. | Disabled target kinds cannot reach filesystem mutation or activation. | | Interrupted skill-candidate consolidation | A completed evolution job has checkpointed candidates but no completed candidate-work event. | Resume from append-only registry events with the same source-job idempotency key; repair missing scores or transitions before completion. | Candidate proposal revisions remain immutable, retries do not add recurrence, and partial registry appends cannot activate a skill. | -| Interrupted skill synthesis | An evidence-qualified group has no draft, or a draft-record event exists without its `drafted` transition. | Resume with the revision-bound synthesis idempotency key; reuse an already recorded immutable draft and repair the missing transition before completing candidate work. | Only evidence-qualified groups invoke synthesis, and a draft remains outside the active harness with no writer or activation path. | +| Interrupted skill synthesis or draft validation | An evidence-qualified group has no draft, a draft-record event lacks its `drafted` transition, or a validation-record event lacks its terminal transition. | Resume with the revision-bound idempotency keys; reuse recorded immutable artifacts and repair the missing transition before completing candidate work. | Only evidence-qualified groups invoke synthesis, and only deterministically accepted drafts reach `structurally-valid`; no draft has a writer or activation path. | ## Operational observability @@ -37,7 +37,7 @@ These operational records remain outside materialized product-runtime evidence, ## Verification boundaries -The deterministic validator proves provenance, recurrence, immutable closure, score bounds, and structural eligibility. +Deterministic validation proves provenance, recurrence, immutable closure, score bounds, structural eligibility, workspace and version freshness, and bounded sensitive-data checks. Candidate priority scoring separately orders evaluation work; it cannot skip later gates, accept a candidate, or activate a skill. -It does not prove that natural-language evidence semantically entails a proposal. +It does not prove semantic usefulness, that natural-language evidence entails a proposal, or the absence of all semantically unsafe guidance. Post-activation outcome evaluation and automatic regression rollback are intentionally outside this recovery contract. diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index d268a585e..defbe56d4 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -608,6 +608,27 @@ export interface SkillDraft { readonly content: string; } +export interface SkillDraftValidationIssue { + readonly code: string; + readonly message: string; +} + +export interface SkillDraftValidationReport { + readonly artifactKind: "skill-draft-validation"; + readonly metadata: EvolutionArtifactMetadata; + readonly validationId: string; + readonly groupId: string; + readonly draft: EvolutionArtifactReference; + readonly validatorVersion: string; + readonly assurance: "structure-provenance-scope-and-security-only"; + readonly disposition: "accepted" | "rejected"; + readonly issues: readonly SkillDraftValidationIssue[]; + readonly validatedHarnessVersion: HarnessVersion; + readonly validatedSkillVersions: readonly string[]; + readonly canonicalInventory: readonly string[]; + readonly memoryRelationshipDigest: string; +} + export interface SkillCandidateDispositionRecord { readonly eventId: string; readonly from: SkillCandidateState | null; @@ -625,7 +646,9 @@ export interface SkillCandidateGroup { readonly evidence: SkillCandidateEvidenceContribution; readonly priorityScore: EvolutionArtifactReference | null; readonly draft: EvolutionArtifactReference | null; + readonly validation: EvolutionArtifactReference | null; readonly synthesisIdempotencyKey: string; + readonly validationIdempotencyKey: string; readonly evaluationIdempotencyKey: string; } @@ -658,6 +681,15 @@ export interface SkillCandidateDraftRecordedEvent extends SkillCandidateRegistry readonly idempotencyKey: string; } +export interface SkillCandidateValidationRecordedEvent extends SkillCandidateRegistryEventBase { + readonly type: "skill-candidate-validation-recorded"; + readonly groupId: string; + readonly draft: EvolutionArtifactReference; + readonly validation: EvolutionArtifactReference; + readonly proposalRevisionIds: readonly string[]; + readonly idempotencyKey: string; +} + export interface SkillCandidateStateTransitionedEvent extends SkillCandidateRegistryEventBase { readonly type: "skill-candidate-state-transitioned"; readonly groupId: string; @@ -666,7 +698,7 @@ export interface SkillCandidateStateTransitionedEvent extends SkillCandidateRegi readonly reasonCodes: readonly string[]; } -export type SkillCandidateWorkStage = "consolidating" | "scoring" | "synthesizing" | "completed"; +export type SkillCandidateWorkStage = "consolidating" | "scoring" | "synthesizing" | "validating" | "completed"; export interface SkillCandidateWorkCheckpointedEvent extends SkillCandidateRegistryEventBase { readonly type: "skill-candidate-work-checkpointed"; @@ -692,6 +724,7 @@ export type SkillCandidateRegistryEvent = | SkillCandidateProposalRecordedEvent | SkillCandidatePriorityScoredEvent | SkillCandidateDraftRecordedEvent + | SkillCandidateValidationRecordedEvent | SkillCandidateStateTransitionedEvent | SkillCandidateWorkCheckpointedEvent | SkillCandidateWorkFailedEvent; diff --git a/packages/evolution/src/filesystem-store.ts b/packages/evolution/src/filesystem-store.ts index 22dbd6a97..0afcdec24 100644 --- a/packages/evolution/src/filesystem-store.ts +++ b/packages/evolution/src/filesystem-store.ts @@ -52,6 +52,7 @@ const SKILL_CANDIDATE_REGISTRY_EVENT_TYPES = new Set([ "skill-candidate-priority-scored", "skill-candidate-proposal-recorded", "skill-candidate-state-transitioned", + "skill-candidate-validation-recorded", "skill-candidate-work-checkpointed", "skill-candidate-work-failed", ]); @@ -66,7 +67,7 @@ const SKILL_CANDIDATE_STATES = new Set([ "rejected", "retired", ]); -const SKILL_CANDIDATE_WORK_STAGES = new Set(["consolidating", "scoring", "synthesizing", "completed"]); +const SKILL_CANDIDATE_WORK_STAGES = new Set(["consolidating", "scoring", "synthesizing", "validating", "completed"]); const JOB_OUTCOMES = new Set([ "active", "completed", @@ -283,6 +284,15 @@ function isCandidateRegistryEventContent(value: Readonly typeof value.idempotencyKey === "string" ); } + if (value.type === "skill-candidate-validation-recorded") { + return ( + typeof value.groupId === "string" && + isArtifactReference(value.draft) && + isArtifactReference(value.validation) && + isStringArray(value.proposalRevisionIds) && + typeof value.idempotencyKey === "string" + ); + } if (value.type === "skill-candidate-priority-scored") { return ( typeof value.groupId === "string" && diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index fbd0d5b1e..fb0448589 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -138,12 +138,15 @@ export { type SkillCandidateRegistrySnapshot, type SkillCandidateState, type SkillCandidateStateTransitionedEvent, + type SkillCandidateValidationRecordedEvent, type SkillCandidateWorkCheckpointedEvent, type SkillCandidateWorkFailedEvent, type SkillCandidateWorkStage, type SkillCandidateWorkspaceScope, type SkillCuratorToolCallLedgerEvent, type SkillDraft, + type SkillDraftValidationIssue, + type SkillDraftValidationReport, type SkillMemoryRelationshipClassification, type SkillTarget, type ValidationIssue, @@ -252,9 +255,14 @@ export type { SkillCandidateRegistryConsolidationInput, SkillCandidateRegistryConsolidationResult, SkillCandidateRegistryStore, + SkillCandidateValidationRecordingInput, SkillCurationResult, SkillCurator, SkillCuratorInput, + SkillDraftValidationEnvironment, + SkillDraftValidationEnvironmentSource, + SkillDraftValidator, + SkillDraftValidatorInput, SkillSynthesizer, SkillSynthesizerInput, SuccessfulMaterialization, @@ -324,6 +332,11 @@ export { SkillCuratorToolRegistry, type SkillCuratorToolRegistryOptions, } from "./skill-curator-tools.ts"; +export { + DeterministicSkillDraftValidator, + type DeterministicSkillDraftValidatorDependencies, + type DeterministicSkillDraftValidatorOptions, +} from "./skill-draft-validator.ts"; export { StructuredSkillSynthesizer, type StructuredSkillSynthesizerDependencies, diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 4f412a446..b16918777 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -39,6 +39,7 @@ import type { SkillCandidateRegistryEvent, SkillCandidateRegistrySnapshot, SkillDraft, + SkillDraftValidationReport, ValidationReport, } from "./contracts.ts"; @@ -59,6 +60,7 @@ export type EvolutionArtifact = | SkillCandidate | SkillCandidatePriorityScore | SkillDraft + | SkillDraftValidationReport | DeferredSkillCandidateFinding | ValidationReport; @@ -315,9 +317,20 @@ export interface SkillCandidateDraftRecordingInput { readonly createdAt: string; } +export interface SkillCandidateValidationRecordingInput { + readonly scope: EvolutionScope; + readonly groupId: string; + readonly draft: EvolutionArtifactReference; + readonly validation: EvolutionArtifactReference; + readonly proposalRevisionIds: readonly string[]; + readonly idempotencyKey: string; + readonly createdAt: string; +} + export interface SkillCandidateRegistry { consolidate(input: SkillCandidateRegistryConsolidationInput): Promise; recordDraft(input: SkillCandidateDraftRecordingInput): Promise; + recordValidation(input: SkillCandidateValidationRecordingInput): Promise; list(scope: EvolutionScope): Promise; } @@ -333,6 +346,38 @@ export interface SkillSynthesizer { synthesize(input: SkillSynthesizerInput): Promise; } +export interface SkillDraftValidationEnvironment { + readonly workspace: { + readonly repositoryId: string; + readonly directoryId?: string; + }; + readonly harnessVersion: string; + readonly skillVersions: readonly { + readonly skillId: string; + readonly version: string; + readonly contentDigest: string; + }[]; +} + +export interface SkillDraftValidationEnvironmentSource { + capture( + workspace: SkillCandidateGroup["profile"]["workspace"], + signal?: AbortSignal, + ): Promise; +} + +export interface SkillDraftValidatorInput { + readonly group: SkillCandidateGroup; + readonly draft: SkillDraft; + readonly idempotencyKey: string; + readonly createdAt: string; + readonly signal?: AbortSignal; +} + +export interface SkillDraftValidator { + validate(input: SkillDraftValidatorInput): Promise; +} + export interface EvolutionJobStore { create(job: EvolutionJob, lease: EvolutionLeaseToken): Promise; save(job: EvolutionJob, expectedRevision: number, lease: EvolutionLeaseToken): Promise; diff --git a/packages/evolution/src/skill-candidate-registry.ts b/packages/evolution/src/skill-candidate-registry.ts index 86b59cafb..10f1eca46 100644 --- a/packages/evolution/src/skill-candidate-registry.ts +++ b/packages/evolution/src/skill-candidate-registry.ts @@ -17,6 +17,7 @@ import type { SkillCandidateState, SkillCandidateStateTransitionedEvent, SkillDraft, + SkillDraftValidationReport, } from "./contracts.ts"; import { SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION } from "./contracts.ts"; import type { @@ -26,6 +27,7 @@ import type { SkillCandidateRegistryConsolidationInput, SkillCandidateRegistryConsolidationResult, SkillCandidateRegistryStore, + SkillCandidateValidationRecordingInput, } from "./ports.ts"; import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; @@ -61,6 +63,8 @@ interface MutableGroup { priorityScore: EvolutionArtifactReference | null; draft: EvolutionArtifactReference | null; readonly draftProposalRevisionIds: string[]; + validation: EvolutionArtifactReference | null; + readonly validationProposalRevisionIds: string[]; readonly scoredProposalRevisionIds: string[]; } @@ -267,6 +271,8 @@ function materializeGroups(events: readonly SkillCandidateRegistryEvent[]): Map< priorityScore: null, draft: null, draftProposalRevisionIds: [], + validation: null, + validationProposalRevisionIds: [], scoredProposalRevisionIds: [], }); } else if ( @@ -313,6 +319,35 @@ function materializeGroups(events: readonly SkillCandidateRegistryEvent[]): Map< group.draftProposalRevisionIds.splice(0, group.draftProposalRevisionIds.length, ...event.proposalRevisionIds); continue; } + if (event.type === "skill-candidate-validation-recorded") { + const group = groups.get(event.groupId); + if (group === undefined) throw new Error(`Candidate validation references unknown group "${event.groupId}"`); + if ( + !sameValues( + event.proposalRevisionIds, + group.proposalRevisions.map((revision) => revision.revisionId), + ) || + group.draft?.artifactId !== event.draft.artifactId || + group.draft.contentDigest !== event.draft.contentDigest + ) { + throw new Error(`Candidate validation for group "${event.groupId}" has a stale draft or revision set`); + } + if ( + group.validation !== null && + sameValues(group.validationProposalRevisionIds, event.proposalRevisionIds) && + (group.validation.artifactId !== event.validation.artifactId || + group.validation.contentDigest !== event.validation.contentDigest) + ) { + throw new Error(`Candidate group "${event.groupId}" has conflicting validation reports`); + } + group.validation = event.validation; + group.validationProposalRevisionIds.splice( + 0, + group.validationProposalRevisionIds.length, + ...event.proposalRevisionIds, + ); + continue; + } if (event.type === "skill-candidate-state-transitioned") { const group = groups.get(event.groupId); if (group === undefined) throw new Error(`Candidate transition references unknown group "${event.groupId}"`); @@ -353,7 +388,18 @@ function immutableGroup(group: MutableGroup): SkillCandidateGroup { ) ? group.draft : null, + validation: sameValues( + group.validationProposalRevisionIds, + proposalRevisions.map((revision) => revision.revisionId), + ) + ? group.validation + : null, synthesisIdempotencyKey: stableId("skill-synthesis", { groupId: group.groupId, revisionSetDigest }), + validationIdempotencyKey: stableId("skill-validation", { + groupId: group.groupId, + revisionSetDigest, + draft: group.draft, + }), evaluationIdempotencyKey: stableId("skill-evaluation", { groupId: group.groupId, revisionSetDigest }), }; } @@ -382,6 +428,30 @@ function assertDraft( } } +function assertValidation( + validation: SkillDraftValidationReport, + reference: EvolutionArtifactReference, + group: SkillCandidateGroup, + input: SkillCandidateValidationRecordingInput, +): void { + if ( + validation.artifactKind !== "skill-draft-validation" || + validation.validationId === "" || + validation.metadata.artifactId !== validation.validationId || + validation.metadata.contentDigest !== computeEvolutionContentDigest(contentWithoutMetadata(validation)) || + reference.artifactId !== validation.validationId || + reference.contentDigest !== computeEvolutionContentDigest(validation) || + validation.groupId !== group.groupId || + validation.draft.artifactId !== input.draft.artifactId || + validation.draft.contentDigest !== input.draft.contentDigest || + input.idempotencyKey !== group.validationIdempotencyKey || + validation.metadata.scope.repositoryId !== input.scope.repositoryId || + validation.metadata.scope.scopeId !== input.scope.scopeId + ) { + throw new Error(`Skill validation for candidate group "${group.groupId}" has an invalid immutable identity`); + } +} + function assertCandidate( candidate: SkillCandidate, reference: EvolutionArtifactReference, @@ -528,8 +598,10 @@ export class DeterministicSkillCandidatePriorityScorer { readonly state: SkillCandidateState; readonly reasonCodes: readonly string[]; } { - const hasStaleDraft = group.state === "drafted" && group.draft === null; - if (!hasStaleDraft && !MANAGED_QUALIFICATION_STATES.has(group.state)) { + const hasStaleArtifacts = + (group.state === "drafted" || group.state === "structurally-valid" || group.state === "rejected") && + group.draft === null; + if (!hasStaleArtifacts && !MANAGED_QUALIFICATION_STATES.has(group.state)) { return { state: group.state, reasonCodes: [] }; } const support = group.evidence.supportingTaskLineageIds.length; @@ -540,7 +612,7 @@ export class DeterministicSkillCandidatePriorityScorer { if (support >= this.#minimumIndependentTasks) { return { state: "evidence-qualified", - reasonCodes: [hasStaleDraft ? "proposal-revision-set-changed" : "independent-recurrence-qualified"], + reasonCodes: [hasStaleArtifacts ? "proposal-revision-set-changed" : "independent-recurrence-qualified"], }; } return { state: "proposed", reasonCodes: ["awaiting-independent-recurrence"] }; @@ -744,6 +816,115 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry throw new Error("Skill candidate registry changed too often to record a draft"); } + async recordValidation(input: SkillCandidateValidationRecordingInput): Promise { + for (let attempt = 0; attempt < this.#maxCommitAttempts; attempt += 1) { + const snapshot = await this.#store.loadSkillCandidateRegistry(input.scope); + const groups = materializeGroups(snapshot.events); + const mutable = groups.get(input.groupId); + if (mutable === undefined) throw new Error(`Unknown candidate group "${input.groupId}"`); + let group = immutableGroup(mutable); + if ( + !sameValues( + input.proposalRevisionIds, + group.proposalRevisions.map((revision) => revision.revisionId), + ) || + group.draft?.artifactId !== input.draft.artifactId || + group.draft.contentDigest !== input.draft.contentDigest + ) { + throw new Error(`Candidate group "${input.groupId}" changed before validation was recorded`); + } + if (group.state !== "drafted" && group.state !== "structurally-valid" && group.state !== "rejected") { + throw new Error(`Candidate group "${input.groupId}" is not eligible to record validation`); + } + const artifact = await this.#artifacts.get(input.validation); + if (!("artifactKind" in artifact) || artifact.artifactKind !== "skill-draft-validation") { + throw new Error(`Candidate group "${input.groupId}" validation artifact is invalid`); + } + assertValidation(artifact, input.validation, group, input); + if ( + group.validation !== null && + (group.validation.artifactId !== input.validation.artifactId || + group.validation.contentDigest !== input.validation.contentDigest) + ) { + throw new Error(`Candidate group "${input.groupId}" already has a different validation report`); + } + const targetState = artifact.disposition === "accepted" ? "structurally-valid" : "rejected"; + if (group.state !== "drafted" && group.state !== targetState) { + throw new Error(`Candidate group "${input.groupId}" validation disposition conflicts with its state`); + } + const planned: SkillCandidateRegistryEvent[] = []; + if (group.validation === null) { + const registrySequence = nextSequence(snapshot.events, planned.length); + const content = { + schemaVersion: SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, + eventId: stableId("skill-candidate-event", { + idempotencyKey: input.idempotencyKey, + registrySequence, + type: "skill-candidate-validation-recorded", + groupId: input.groupId, + }), + registrySequence, + type: "skill-candidate-validation-recorded" as const, + groupId: input.groupId, + draft: input.draft, + validation: input.validation, + proposalRevisionIds: input.proposalRevisionIds, + idempotencyKey: input.idempotencyKey, + }; + planned.push( + withEventMetadata( + content, + this.#producer, + [input.groupId, input.draft.artifactId, input.validation.artifactId], + input.scope, + input.createdAt, + ), + ); + mutable.validation = input.validation; + mutable.validationProposalRevisionIds.splice( + 0, + mutable.validationProposalRevisionIds.length, + ...input.proposalRevisionIds, + ); + group = immutableGroup(mutable); + } + if (group.state === "drafted") { + const reasonCodes = + artifact.disposition === "accepted" + ? ["deterministic-draft-validation-passed"] + : uniqueSorted(artifact.issues.map((validationIssue) => validationIssue.code)); + const transition = transitionEvent( + input.groupId, + "drafted", + targetState, + reasonCodes, + nextSequence(snapshot.events, planned.length), + input.idempotencyKey, + this.#producer, + input.scope, + input.createdAt, + ); + planned.push(transition); + mutable.state = targetState; + mutable.dispositionHistory.push({ + eventId: transition.eventId, + from: transition.from, + to: transition.to, + reasonCodes: transition.reasonCodes, + createdAt: input.createdAt, + }); + } + if (planned.length === 0) return immutableGroup(mutable); + const committed = await this.#store.appendSkillCandidateRegistryEvents( + input.scope, + snapshot.revision, + planned, + ); + if (committed === "committed") return immutableGroup(mutable); + } + throw new Error("Skill candidate registry changed too often to record validation"); + } + async consolidate( input: SkillCandidateRegistryConsolidationInput, ): Promise { @@ -784,6 +965,8 @@ export class PersistentSkillCandidateRegistry implements SkillCandidateRegistry priorityScore: null, draft: null, draftProposalRevisionIds: [], + validation: null, + validationProposalRevisionIds: [], scoredProposalRevisionIds: [], }; groups.set(groupId, group); diff --git a/packages/evolution/src/skill-candidate-worker.ts b/packages/evolution/src/skill-candidate-worker.ts index 8cdd7c5ce..eb488af25 100644 --- a/packages/evolution/src/skill-candidate-worker.ts +++ b/packages/evolution/src/skill-candidate-worker.ts @@ -20,6 +20,7 @@ import type { InteractivePriorityGate, SkillCandidateRegistry, SkillCandidateRegistryStore, + SkillDraftValidator, SkillSynthesizer, } from "./ports.ts"; import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; @@ -53,6 +54,7 @@ export interface SkillCandidateRegistryWorkerDependencies { readonly store: SkillCandidateRegistryStore; readonly registry: SkillCandidateRegistry; readonly synthesizer: SkillSynthesizer; + readonly validator: SkillDraftValidator; readonly model: EvolutionModelExecutor; } @@ -136,6 +138,12 @@ function isSkillCandidate(artifact: EvolutionArtifact): artifact is SkillCandida return "artifactKind" in artifact && artifact.artifactKind === "skill-candidate"; } +function isSkillDraft( + artifact: EvolutionArtifact, +): artifact is Extract { + return "artifactKind" in artifact && artifact.artifactKind === "skill-draft"; +} + function retryDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number { return Math.min(maxDelayMs, baseDelayMs * 2 ** Math.max(0, attempt - 1)); } @@ -413,7 +421,7 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor attempt, synthesisTargets.map((group) => group.groupId), ); - const draftedGroupIds: string[] = []; + const draftedGroups = result.groups.filter((group) => group.state === "drafted"); for (const group of synthesisTargets) { signal?.throwIfAborted(); let draftReference = group.draft; @@ -427,18 +435,53 @@ export class SkillCandidateRegistryWorker implements SkillCandidateWorkProcessor }); draftReference = await this.#dependencies.artifacts.put(draft); } - await this.#dependencies.registry.recordDraft({ + draftedGroups.push( + await this.#dependencies.registry.recordDraft({ + scope: job.metadata.scope, + groupId: group.groupId, + draft: draftReference, + proposalRevisionIds: group.proposalRevisions.map((revision) => revision.revisionId), + idempotencyKey: group.synthesisIdempotencyKey, + createdAt: this.#dependencies.clock.now(), + }), + ); + } + await this.#appendCheckpoint( + job, + "validating", + attempt, + draftedGroups.map((group) => group.groupId), + ); + const validatedGroupIds: string[] = []; + for (const group of draftedGroups) { + signal?.throwIfAborted(); + if (group.draft === null) throw new Error(`Drafted candidate group "${group.groupId}" has no draft`); + const artifact = await this.#dependencies.artifacts.get(group.draft); + if (!isSkillDraft(artifact)) throw new Error(`Candidate group "${group.groupId}" draft is invalid`); + let validationReference = group.validation; + if (validationReference === null) { + const validation = await this.#dependencies.validator.validate({ + group, + draft: artifact, + idempotencyKey: group.validationIdempotencyKey, + createdAt: this.#dependencies.clock.now(), + signal, + }); + validationReference = await this.#dependencies.artifacts.put(validation); + } + await this.#dependencies.registry.recordValidation({ scope: job.metadata.scope, groupId: group.groupId, - draft: draftReference, + draft: group.draft, + validation: validationReference, proposalRevisionIds: group.proposalRevisions.map((revision) => revision.revisionId), - idempotencyKey: group.synthesisIdempotencyKey, + idempotencyKey: group.validationIdempotencyKey, createdAt: this.#dependencies.clock.now(), }); - draftedGroupIds.push(group.groupId); + validatedGroupIds.push(group.groupId); } await this.#appendCheckpoint(job, "completed", attempt, [ - ...new Set([...processedGroupIds, ...draftedGroupIds]), + ...new Set([...processedGroupIds, ...validatedGroupIds]), ]); } catch (error) { if (signal?.aborted) throw error; diff --git a/packages/evolution/src/skill-draft-validator.ts b/packages/evolution/src/skill-draft-validator.ts new file mode 100644 index 000000000..8f6015a7c --- /dev/null +++ b/packages/evolution/src/skill-draft-validator.ts @@ -0,0 +1,463 @@ +import { validateHarnessSkillContent } from "@earendil-works/diffex-harness-state"; +import type { + EvolutionArtifactProducer, + EvolutionArtifactReference, + EvolutionInputSnapshot, + EvolutionSkillCatalogEntry, + SkillCandidate, + SkillCandidateGroup, + SkillDraftValidationIssue, + SkillDraftValidationReport, +} from "./contracts.ts"; +import type { + EvolutionArtifact, + EvolutionArtifactStore, + EvolutionJobStore, + SkillDraftValidationEnvironment, + SkillDraftValidationEnvironmentSource, + SkillDraftValidator, + SkillDraftValidatorInput, +} from "./ports.ts"; +import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; + +const DEFAULT_MAX_CONTENT_BYTES = 64 * 1024; +const MINIMUM_INDEPENDENT_TASKS = 2; +const SECRET_PATTERNS = [ + /-----BEGIN (?:OPENSSH |RSA |EC |DSA )?PRIVATE KEY-----/, + /\bAKIA[0-9A-Z]{16}\b/, + /\bgh[pousr]_[A-Za-z0-9]{30,}\b/, + /\bxox[baprs]-[A-Za-z0-9-]{20,}\b/, + /\b(?:api[_ -]?key|password|secret|token)\s*[:=]\s*["']?[A-Za-z0-9_+./=-]{16,}/i, +] as const; +const MARKDOWN_TARGET_PATTERN = /!?\[[^\]]*\]\(([^)\s]+)(?:\s+["'][^"']*["'])?\)/gu; +const HTML_TARGET_PATTERN = /\b(?:href|src)\s*=\s*["']([^"']+)["']/giu; + +interface CandidateSource { + readonly candidate: SkillCandidate; + readonly input: EvolutionInputSnapshot; + readonly selectedMemoryProposalIds: readonly string[]; + readonly resultingHarnessVersion: string | null; +} + +export interface DeterministicSkillDraftValidatorDependencies { + readonly version: string; + readonly artifacts: EvolutionArtifactStore; + readonly jobs: EvolutionJobStore; + readonly environment: SkillDraftValidationEnvironmentSource; +} + +export interface DeterministicSkillDraftValidatorOptions { + readonly maxContentBytes?: number; +} + +function uniqueSorted(values: readonly string[]): string[] { + return [...new Set(values)].sort((left, right) => left.localeCompare(right)); +} + +function sameValues(left: readonly string[], right: readonly string[]): boolean { + const normalizedLeft = uniqueSorted(left); + const normalizedRight = uniqueSorted(right); + return ( + normalizedLeft.length === normalizedRight.length && + normalizedLeft.every((value, index) => value === normalizedRight[index]) + ); +} + +function sameReference(left: EvolutionArtifactReference | null, right: EvolutionArtifactReference): boolean { + return left?.artifactId === right.artifactId && left.contentDigest === right.contentDigest; +} + +function contentWithoutMetadata(value: T): Omit { + const { metadata: _metadata, ...content } = value; + return content; +} + +function isInputSnapshot(artifact: EvolutionArtifact): artifact is EvolutionInputSnapshot { + return "evidenceSources" in artifact && "repository" in artifact && "harness" in artifact && "trigger" in artifact; +} + +function isSkillCandidate(artifact: EvolutionArtifact): artifact is SkillCandidate { + return "artifactKind" in artifact && artifact.artifactKind === "skill-candidate"; +} + +function normalizedText(value: string): string { + return value.normalize("NFKC").trim().replace(/\s+/gu, " ").toLocaleLowerCase("en-US"); +} + +function localReferenceTargets(content: string): string[] { + const targets = [...content.matchAll(MARKDOWN_TARGET_PATTERN), ...content.matchAll(HTML_TARGET_PATTERN)].flatMap( + (match) => match[1] ?? [], + ); + return uniqueSorted( + targets.filter((target) => { + if (target.startsWith("#") || target.startsWith("//")) return false; + return !/^[a-z][a-z0-9+.-]*:/iu.test(target); + }), + ); +} + +function issue(code: string, message: string): SkillDraftValidationIssue { + return { code, message }; +} + +function workspaceMatches(group: SkillCandidateGroup, environment: SkillDraftValidationEnvironment): boolean { + return ( + group.profile.workspace.repositoryId === environment.workspace.repositoryId && + group.profile.workspace.directoryId === environment.workspace.directoryId + ); +} + +function skillVersionKey(value: { + readonly skillId: string; + readonly version: string; + readonly contentDigest: string; +}): string { + return `${value.skillId}\u0000${value.version}\u0000${value.contentDigest}`; +} + +function latestCatalogVersions(source: CandidateSource): readonly EvolutionSkillCatalogEntry[] { + return source.candidate.existingSkillIds.flatMap((skillId) => { + const matches = source.input.skillCatalog.filter((skill) => skill.skillId === skillId); + return matches.length === 1 ? matches : []; + }); +} + +export class DeterministicSkillDraftValidator implements SkillDraftValidator { + readonly #dependencies: DeterministicSkillDraftValidatorDependencies; + readonly #producer: EvolutionArtifactProducer; + readonly #maxContentBytes: number; + + constructor( + dependencies: DeterministicSkillDraftValidatorDependencies, + options: DeterministicSkillDraftValidatorOptions = {}, + ) { + this.#dependencies = dependencies; + this.#producer = { + name: "diffex-skill-draft-validator", + version: dependencies.version, + promptVersion: null, + model: null, + }; + this.#maxContentBytes = options.maxContentBytes ?? DEFAULT_MAX_CONTENT_BYTES; + if (!Number.isSafeInteger(this.#maxContentBytes) || this.#maxContentBytes < 1) { + throw new Error("maxContentBytes must be a positive integer"); + } + } + + async validate(input: SkillDraftValidatorInput): Promise { + input.signal?.throwIfAborted(); + const issues: SkillDraftValidationIssue[] = []; + const add = (code: string, message: string) => issues.push(issue(code, message)); + const draftReference = { + artifactId: input.draft.metadata.artifactId, + contentDigest: computeEvolutionContentDigest(input.draft), + }; + if ( + input.draft.artifactKind !== "skill-draft" || + input.draft.draftId === "" || + input.draft.metadata.artifactId !== input.draft.draftId || + input.draft.metadata.contentDigest !== computeEvolutionContentDigest(contentWithoutMetadata(input.draft)) || + input.draft.groupId !== input.group.groupId || + input.draft.synthesisIdempotencyKey !== input.group.synthesisIdempotencyKey || + !sameReference(input.group.draft, draftReference) || + input.group.state !== "drafted" + ) { + add("draft-identity-invalid", "The draft does not match the immutable drafted candidate group"); + } + if ( + !sameValues( + input.draft.proposalRevisionIds, + input.group.proposalRevisions.map((revision) => revision.revisionId), + ) + ) { + add("proposal-revision-closure-invalid", "The draft does not retain the complete proposal revision set"); + } + + const sources: CandidateSource[] = []; + for (const revision of input.group.proposalRevisions) { + input.signal?.throwIfAborted(); + const artifact = await this.#dependencies.artifacts.get(revision.candidate); + if ( + !isSkillCandidate(artifact) || + artifact.candidateId !== revision.candidateId || + artifact.metadata.artifactId !== revision.candidateId || + artifact.metadata.contentDigest !== computeEvolutionContentDigest(contentWithoutMetadata(artifact)) || + computeEvolutionContentDigest(artifact) !== revision.candidate.contentDigest + ) { + add("candidate-provenance-invalid", `Candidate revision "${revision.revisionId}" is not immutable`); + continue; + } + const job = await this.#dependencies.jobs.load(revision.sourceEvolutionJobId); + if (job === null || job.outcome !== "completed" || job.stage !== "completed") { + add("source-job-unavailable", `Source evolution job "${revision.sourceEvolutionJobId}" is unavailable`); + continue; + } + const inputArtifact = await this.#dependencies.artifacts.get(job.inputSnapshot); + if (!isInputSnapshot(inputArtifact)) { + add("source-snapshot-invalid", `Source evolution job "${job.jobId}" has an invalid input snapshot`); + continue; + } + const checkpoint = job.checkpoints.find((candidate) => candidate.stage === "skill-curating"); + const selectedMemoryProposalIds = + checkpoint !== undefined && Array.isArray(checkpoint.state.selectedMemoryProposalIds) + ? checkpoint.state.selectedMemoryProposalIds.filter( + (proposalId): proposalId is string => typeof proposalId === "string", + ) + : []; + sources.push({ + candidate: artifact, + input: inputArtifact, + selectedMemoryProposalIds, + resultingHarnessVersion: job.completion?.resultingHarnessVersion ?? null, + }); + } + + const candidates = sources.map((source) => source.candidate); + if ( + !sameValues( + input.draft.sourceCandidateIds, + candidates.map((candidate) => candidate.candidateId), + ) + ) { + add("candidate-closure-invalid", "The draft source candidate closure is incomplete"); + } + const expectedEvidenceIds = uniqueSorted( + candidates.flatMap((candidate) => [...candidate.supportingEvidenceIds, ...candidate.contradictingEvidenceIds]), + ); + if (!sameValues(input.draft.sourceEvidenceIds, expectedEvidenceIds)) { + add("evidence-closure-invalid", "The draft source evidence closure is incomplete"); + } + const declaredTraceIds = uniqueSorted( + candidates.flatMap((candidate) => [...candidate.supportingTraceIds, ...candidate.contradictingTraceIds]), + ); + const derivedTraceIds = uniqueSorted( + sources.flatMap((source) => { + const citedEvidenceIds = new Set([ + ...source.candidate.supportingEvidenceIds, + ...source.candidate.contradictingEvidenceIds, + ]); + return source.input.evidenceSources.flatMap((evidenceSource) => + evidenceSource.invocation.evidence.some((evidence) => citedEvidenceIds.has(evidence.evidenceId)) + ? (evidenceSource.invocation.traceId ?? []) + : [], + ); + }), + ); + if (!sameValues(declaredTraceIds, derivedTraceIds)) { + add("candidate-trace-closure-invalid", "The candidate trace closure does not match its cited evidence"); + } + if (!sameValues(input.draft.sourceTraceIds, derivedTraceIds)) { + add("trace-closure-invalid", "The draft source trace closure is incomplete"); + } + const expectedInstructionDigests = uniqueSorted( + sources.flatMap((source) => source.input.instructions.map((instruction) => instruction.contentDigest)), + ); + if (!sameValues(input.draft.repositoryInstructionDigests, expectedInstructionDigests)) { + add("instruction-closure-invalid", "The draft repository instruction closure is incomplete"); + } + const expectedSkillVersions = uniqueSorted( + sources.flatMap((source) => + source.candidate.existingSkillIds.flatMap((skillId) => { + const matches = source.input.skillCatalog.filter((skill) => skill.skillId === skillId); + if (matches.length !== 1) { + add( + "source-skill-provenance-invalid", + `Candidate "${source.candidate.candidateId}" does not resolve skill "${skillId}" uniquely`, + ); + } + return matches.map((skill) => skillVersionKey(skill)); + }), + ), + ); + if (!sameValues(input.draft.sourceSkillVersions.map(skillVersionKey), expectedSkillVersions)) { + add("source-skill-closure-invalid", "The draft source skill version closure is incomplete"); + } + const expectedMetadataSourceIds = uniqueSorted([ + ...input.draft.sourceCandidateIds, + ...input.draft.sourceEvidenceIds, + ...input.draft.sourceTraceIds, + ...input.draft.sourceSkillVersions.map((skill) => skill.skillId), + ...input.draft.repositoryInstructionDigests, + ]); + if ( + !sameValues(input.draft.metadata.sourceIds, expectedMetadataSourceIds) || + input.draft.metadata.scope.repositoryId !== input.group.profile.workspace.evolutionScope.repositoryId || + input.draft.metadata.scope.scopeId !== input.group.profile.workspace.evolutionScope.scopeId + ) { + add("draft-provenance-invalid", "The draft metadata does not retain its complete source and scope closure"); + } + + const derivedSupportingLineages = new Set(); + const derivedContradictingLineages = new Set(); + for (const source of sources) { + const evidenceById = new Map( + source.input.evidenceSources.flatMap((evidenceSource) => + evidenceSource.invocation.evidence.map((evidence) => [evidence.evidenceId, evidence] as const), + ), + ); + const derive = (evidenceIds: readonly string[], target: Set) => { + for (const evidenceId of evidenceIds) { + const evidence = evidenceById.get(evidenceId); + if (evidence === undefined) { + add( + "evidence-source-missing", + `Candidate "${source.candidate.candidateId}" cites unavailable evidence`, + ); + continue; + } + for (const identifier of evidence.order.causalIdentifiers) { + const name = identifier.name.toLocaleLowerCase("en-US"); + if (name.includes("task") && name.includes("lineage")) target.add(identifier.value); + } + } + }; + const supporting = new Set(); + const contradicting = new Set(); + derive(source.candidate.supportingEvidenceIds, supporting); + derive(source.candidate.contradictingEvidenceIds, contradicting); + if (!sameValues(source.candidate.taskLineageIds, [...supporting, ...contradicting])) { + add( + "candidate-lineage-closure-invalid", + `Candidate "${source.candidate.candidateId}" has invalid task lineage closure`, + ); + } + for (const lineageId of supporting) derivedSupportingLineages.add(lineageId); + for (const lineageId of contradicting) derivedContradictingLineages.add(lineageId); + if ( + source.candidate.memoryRelationship.memoryProposalIds.some( + (proposalId) => !source.selectedMemoryProposalIds.includes(proposalId), + ) + ) { + add( + "memory-relationship-stale", + `Candidate "${source.candidate.candidateId}" cites memory guidance that was not selected`, + ); + } + } + if ( + derivedSupportingLineages.size < MINIMUM_INDEPENDENT_TASKS || + !sameValues(input.group.evidence.supportingTaskLineageIds, [...derivedSupportingLineages]) || + !sameValues(input.group.evidence.contradictingTaskLineageIds, [...derivedContradictingLineages]) + ) { + add("independent-recurrence-invalid", "The final draft does not retain independently recurring evidence"); + } + + const environment = await this.#dependencies.environment.capture(input.group.profile.workspace, input.signal); + if (!workspaceMatches(input.group, environment)) { + add("workspace-scope-mismatch", "The active workspace does not match the candidate workspace"); + } + const latest = sources.at(-1); + if (latest === undefined) { + add("final-candidate-missing", "The final candidate revision is unavailable"); + } else { + const expectedHarnessVersion = latest.resultingHarnessVersion ?? latest.input.expectedHarnessBaseVersion; + if (environment.harnessVersion !== expectedHarnessVersion) { + add("harness-base-stale", "The active harness no longer matches the final candidate base"); + } + if ( + input.draft.operation !== latest.candidate.operation || + input.draft.proposedName !== latest.candidate.proposedName || + !sameValues(input.draft.existingSkillIds, latest.candidate.existingSkillIds) || + input.draft.workspace.repositoryId !== latest.candidate.workspace.repositoryId || + input.draft.workspace.directoryId !== latest.candidate.workspace.directoryId || + input.draft.workspace.rootPath !== latest.candidate.workspace.rootPath || + input.draft.workspace.repositoryRevision !== latest.candidate.workspace.repositoryRevision || + input.draft.workspace.evolutionScope.repositoryId !== + latest.candidate.workspace.evolutionScope.repositoryId || + input.draft.workspace.evolutionScope.scopeId !== latest.candidate.workspace.evolutionScope.scopeId + ) { + add("final-candidate-mismatch", "The draft does not match the final candidate operation and scope"); + } + const activeVersions = new Map(environment.skillVersions.map((skill) => [skill.skillId, skill])); + for (const frozen of latestCatalogVersions(latest)) { + const active = activeVersions.get(frozen.skillId); + if ( + active === undefined || + active.version !== frozen.version || + active.contentDigest !== frozen.contentDigest + ) { + add("source-skill-stale", `Source skill "${frozen.skillId}" is no longer at its frozen version`); + } + } + const normalizedContent = normalizedText(input.draft.content); + for (const boundary of latest.candidate.applicabilityBoundaries) { + if (!normalizedContent.includes(normalizedText(boundary))) { + add("applicability-boundary-missing", `The draft omits applicability boundary "${boundary}"`); + } + } + for (const counterexample of latest.candidate.trigger.negativeExamples) { + if (!normalizedContent.includes(normalizedText(counterexample))) { + add("counterexample-missing", `The draft omits counterexample "${counterexample}"`); + } + } + for (const acceptanceCriterion of [ + ...latest.candidate.verificationContract.checks, + latest.candidate.verificationContract.completionCondition, + ]) { + if (!normalizedContent.includes(normalizedText(acceptanceCriterion))) { + add("acceptance-criterion-missing", `The draft omits acceptance criterion "${acceptanceCriterion}"`); + } + } + } + + const inventoryPath = `skills/${input.draft.proposedName}/SKILL.md`; + try { + validateHarnessSkillContent(inventoryPath, input.draft.content); + } catch (error) { + add("skill-structure-invalid", error instanceof Error ? error.message : "The skill structure is invalid"); + } + if (Buffer.byteLength(input.draft.content, "utf8") > this.#maxContentBytes) { + add("skill-content-too-large", `The skill exceeds the ${this.#maxContentBytes}-byte size limit`); + } + if (input.draft.content.includes("\u0000")) add("skill-content-invalid", "The skill contains a NUL byte"); + if (SECRET_PATTERNS.some((pattern) => pattern.test(input.draft.content))) { + add("sensitive-data-detected", "The skill violates the sensitive-data policy"); + } + const localTargets = localReferenceTargets(input.draft.content); + if (localTargets.length > 0) { + add( + "undeclared-reference", + `Instruction-only skill drafts cannot reference undeclared local files: ${localTargets.join(", ")}`, + ); + } + + const sortedIssues = [...issues].sort( + (left, right) => left.code.localeCompare(right.code) || left.message.localeCompare(right.message), + ); + const content = { + artifactKind: "skill-draft-validation" as const, + validationId: `skill-draft-validation-${computeEvolutionContentDigest({ + groupId: input.group.groupId, + draft: draftReference, + harnessVersion: environment.harnessVersion, + issues: sortedIssues, + }).slice("sha256:".length, "sha256:".length + 32)}`, + groupId: input.group.groupId, + draft: draftReference, + validatorVersion: this.#dependencies.version, + assurance: "structure-provenance-scope-and-security-only" as const, + disposition: sortedIssues.length === 0 ? ("accepted" as const) : ("rejected" as const), + issues: sortedIssues, + validatedHarnessVersion: environment.harnessVersion, + validatedSkillVersions: uniqueSorted(environment.skillVersions.map(skillVersionKey)), + canonicalInventory: [inventoryPath], + memoryRelationshipDigest: computeEvolutionContentDigest( + candidates.map((candidate) => ({ + candidateId: candidate.candidateId, + memoryRelationship: candidate.memoryRelationship, + })), + ), + }; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: content.validationId, + producer: this.#producer, + sourceIds: uniqueSorted([input.draft.draftId, ...input.draft.sourceCandidateIds]), + scope: input.draft.metadata.scope, + createdAt: input.createdAt, + content, + }), + ...content, + }; + } +} diff --git a/packages/evolution/src/skill-synthesizer.ts b/packages/evolution/src/skill-synthesizer.ts index 707a644d9..f195bd171 100644 --- a/packages/evolution/src/skill-synthesizer.ts +++ b/packages/evolution/src/skill-synthesizer.ts @@ -272,6 +272,7 @@ function modelInput( removeTaskSpecificPathsSecretsAndTransientValues: true, removeUnsupportedGeneralizations: true, retainExplicitCompatibilityRequirements: true, + retainCandidateApplicabilityBoundariesNegativeTriggersAndVerificationContractVerbatim: true, requiredContent: [ "Agent Skills frontmatter with a specific name and description", "recognizable use conditions", diff --git a/packages/evolution/test/skill-candidate-registry.test.ts b/packages/evolution/test/skill-candidate-registry.test.ts index 9799d8b6f..5f106c76c 100644 --- a/packages/evolution/test/skill-candidate-registry.test.ts +++ b/packages/evolution/test/skill-candidate-registry.test.ts @@ -16,6 +16,7 @@ import { type SkillCandidateRegistrySnapshot, type SkillCandidateRegistryStore, type SkillDraft, + type SkillDraftValidationReport, } from "../src/index.ts"; const createdAt = "2026-08-25T12:00:00.000Z"; @@ -437,6 +438,46 @@ describe("persistent skill candidate registry", () => { to: "drafted", reasonCodes: ["outcome-aware-draft-recorded"], }); + const validationContent = { + artifactKind: "skill-draft-validation" as const, + validationId: "skill-validation-1", + groupId: group.groupId, + draft: draftReference, + validatorVersion: "validator-v1", + assurance: "structure-provenance-scope-and-security-only" as const, + disposition: "accepted" as const, + issues: [], + validatedHarnessVersion: secondInput.expectedHarnessBaseVersion, + validatedSkillVersions: [], + canonicalInventory: ["skills/focused-check/SKILL.md"], + memoryRelationshipDigest: computeEvolutionContentDigest([]), + }; + const validation: SkillDraftValidationReport = { + metadata: metadata(validationContent.validationId, [draft.draftId], validationContent), + ...validationContent, + }; + const validationReference = await fixture.artifacts.put(validation); + const validationRequest = { + scope, + groupId: group.groupId, + draft: draftReference, + validation: validationReference, + proposalRevisionIds: draft.proposalRevisionIds, + idempotencyKey: recorded.validationIdempotencyKey, + createdAt: "2026-08-26T12:02:00.000Z", + }; + store.interruptNextMultiAppend = true; + await expect(fixture.registry.recordValidation(validationRequest)).rejects.toThrow( + "interrupted draft registry append", + ); + const validated = await fixture.registry.recordValidation(validationRequest); + + expect(validated).toMatchObject({ state: "structurally-valid", validation: validationReference }); + expect(validated.dispositionHistory.at(-1)).toMatchObject({ + from: "drafted", + to: "structurally-valid", + reasonCodes: ["deterministic-draft-validation-passed"], + }); const thirdInput = input([evidence("support-c", "task-c", 0)], "trace-c"); const thirdCandidate = candidate({ @@ -457,7 +498,7 @@ describe("persistent skill candidate registry", () => { expect(revised.groups[0]).toMatchObject({ state: "evidence-qualified", draft: null }); expect(revised.groups[0]?.synthesisIdempotencyKey).not.toBe(recorded.synthesisIdempotencyKey); expect(revised.groups[0]?.dispositionHistory.at(-1)).toMatchObject({ - from: "drafted", + from: "structurally-valid", to: "evidence-qualified", reasonCodes: ["proposal-revision-set-changed"], }); diff --git a/packages/evolution/test/skill-candidate-worker.test.ts b/packages/evolution/test/skill-candidate-worker.test.ts index bd7ac197f..acb55f39f 100644 --- a/packages/evolution/test/skill-candidate-worker.test.ts +++ b/packages/evolution/test/skill-candidate-worker.test.ts @@ -21,11 +21,15 @@ import { type SkillCandidateRegistrySnapshot, type SkillCandidateRegistryStore, SkillCandidateRegistryWorker, + type SkillCandidateValidationRecordingInput, type SkillDraft, + type SkillDraftValidationReport, + type SkillDraftValidatorInput, type SkillSynthesizerInput, } from "../src/index.ts"; const now = "2026-08-25T12:00:00.000Z"; +const harnessVersion = `sha256:${"2".repeat(64)}`; const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; const producer: EvolutionArtifactProducer = { name: "fixture", @@ -50,13 +54,13 @@ function inputSnapshot(): EvolutionInputSnapshot { revision: "revision-1", }, harness: { - harnessVersion: `sha256:${"2".repeat(64)}`, + harnessVersion, baseVersion: `sha256:${"1".repeat(64)}`, memory, memoryContentDigest: computeEvolutionContentDigest(memory), activeMemoryItems: [], }, - expectedHarnessBaseVersion: `sha256:${"2".repeat(64)}`, + expectedHarnessBaseVersion: harnessVersion, ingestionHighWaterCursor: 0, instructions: [], skillCatalog: [], @@ -175,7 +179,9 @@ function qualifiedGroup(candidateReference: EvolutionArtifactReference): SkillCa }, priorityScore: null, draft: null, + validation: null, synthesisIdempotencyKey: "skill-synthesis-qualified", + validationIdempotencyKey: "skill-validation-qualified", evaluationIdempotencyKey: "skill-evaluation-qualified", }; } @@ -200,14 +206,16 @@ class MemoryJobStore implements EvolutionJobStore { } class MemoryArtifactStore implements EvolutionArtifactStore { - readonly artifacts: ReadonlyMap; + readonly artifacts: Map; constructor(artifacts: readonly EvolutionArtifact[]) { this.artifacts = new Map(artifacts.map((artifact) => [reference(artifact).contentDigest, artifact])); } async put(artifact: EvolutionArtifact): Promise { - return reference(artifact); + const artifactReference = reference(artifact); + this.artifacts.set(artifactReference.contentDigest, artifact); + return artifactReference; } async get(artifactReference: EvolutionArtifactReference): Promise { @@ -238,6 +246,7 @@ class MemoryRegistryStore implements SkillCandidateRegistryStore { class FixtureRegistry implements SkillCandidateRegistry { readonly requests: SkillCandidateRegistryConsolidationInput[] = []; readonly draftRequests: SkillCandidateDraftRecordingInput[] = []; + readonly validationRequests: SkillCandidateValidationRecordingInput[] = []; readonly groups: SkillCandidateGroup[] = []; failuresRemaining = 0; @@ -259,6 +268,13 @@ class FixtureRegistry implements SkillCandidateRegistry { return { ...group, state: "drafted", draft: input.draft }; } + async recordValidation(input: SkillCandidateValidationRecordingInput): Promise { + this.validationRequests.push(input); + const group = this.groups.find((candidateGroup) => candidateGroup.groupId === input.groupId); + if (group === undefined) throw new Error("Unknown group"); + return { ...group, state: "structurally-valid", validation: input.validation }; + } + async list(): Promise { return this.groups; } @@ -274,6 +290,7 @@ function fixture() { const store = new MemoryRegistryStore(); const registry = new FixtureRegistry(); const synthesisRequests: SkillSynthesizerInput[] = []; + const validationRequests: SkillDraftValidatorInput[] = []; let clockTime = now; const clock = { now: () => clockTime }; return { @@ -282,6 +299,7 @@ function fixture() { store, registry, synthesisRequests, + validationRequests, clock, setClock(value: string) { clockTime = value; @@ -322,6 +340,26 @@ function fixture() { return { metadata: metadata(content.draftId, content.sourceCandidateIds, content), ...content }; }, }, + validator: { + async validate(request): Promise { + validationRequests.push(request); + const content = { + artifactKind: "skill-draft-validation" as const, + validationId: "validation-1", + groupId: request.group.groupId, + draft: reference(request.draft), + validatorVersion: "validator-v1", + assurance: "structure-provenance-scope-and-security-only" as const, + disposition: "accepted" as const, + issues: [], + validatedHarnessVersion: harnessVersion, + validatedSkillVersions: [], + canonicalInventory: ["skills/fixture-skill/SKILL.md"], + memoryRelationshipDigest: computeEvolutionContentDigest([]), + }; + return { metadata: metadata(content.validationId, [request.draft.draftId], content), ...content }; + }, + }, model: { maxProviderCallsPerExecution: 20, execute: () => Promise.reject(new Error("Fixture synthesizer does not call the model")), @@ -355,7 +393,7 @@ describe("skill candidate registry worker", () => { data.store.events.flatMap((event) => event.type === "skill-candidate-work-checkpointed" ? [event.stage] : [], ), - ).toEqual(["consolidating", "scoring", "synthesizing", "completed"]); + ).toEqual(["consolidating", "scoring", "synthesizing", "validating", "completed"]); expect( data.store.events .filter((event) => event.type === "skill-candidate-work-checkpointed") @@ -365,6 +403,7 @@ describe("skill candidate registry worker", () => { expect.stringMatching(/^skill-candidate-work-stage-/), expect.stringMatching(/^skill-candidate-work-stage-/), expect.stringMatching(/^skill-candidate-work-stage-/), + expect.stringMatching(/^skill-candidate-work-stage-/), ]); }); @@ -389,6 +428,19 @@ describe("skill candidate registry worker", () => { draft: expect.objectContaining({ artifactId: "draft-1" }), }), ]); + expect(data.validationRequests).toEqual([ + expect.objectContaining({ + group: expect.objectContaining({ groupId: "group-qualified", state: "drafted" }), + idempotencyKey: "skill-validation-qualified", + }), + ]); + expect(data.registry.validationRequests).toEqual([ + expect.objectContaining({ + groupId: "group-qualified", + idempotencyKey: "skill-validation-qualified", + validation: expect.objectContaining({ artifactId: "validation-1" }), + }), + ]); }); it("persists bounded retry timing without marking failed work complete", async () => { diff --git a/packages/evolution/test/skill-draft-validator.test.ts b/packages/evolution/test/skill-draft-validator.test.ts new file mode 100644 index 000000000..992ee77a6 --- /dev/null +++ b/packages/evolution/test/skill-draft-validator.test.ts @@ -0,0 +1,475 @@ +import type { MaterializedEvidence, MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import { describe, expect, it } from "vitest"; +import { + computeEvolutionContentDigest, + createEvolutionArtifactMetadata, + DeterministicSkillDraftValidator, + type EvolutionArtifact, + type EvolutionArtifactProducer, + type EvolutionArtifactReference, + type EvolutionArtifactStore, + type EvolutionInputSnapshot, + type EvolutionJob, + type EvolutionJobStore, + type EvolutionScope, + type SkillCandidate, + type SkillCandidateGroup, + type SkillDraft, + type SkillDraftValidationEnvironment, +} from "../src/index.ts"; + +const createdAt = "2026-08-25T12:00:00.000Z"; +const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; +const harnessVersion = `sha256:${"2".repeat(64)}`; +const producer: EvolutionArtifactProducer = { + name: "fixture", + version: "1", + promptVersion: null, + model: null, +}; + +function metadata(artifactId: string, sourceIds: readonly string[], content: unknown) { + return createEvolutionArtifactMetadata({ artifactId, producer, sourceIds, scope, createdAt, content }); +} + +function reference(artifact: EvolutionArtifact): EvolutionArtifactReference { + return { artifactId: artifact.metadata.artifactId, contentDigest: computeEvolutionContentDigest(artifact) }; +} + +function evidence(evidenceId: string, lineageId: string, position: number): MaterializedEvidence { + return { + kind: "session_item", + evidenceId, + itemType: "message", + payload: { + type: "message", + id: `entry-${position}`, + message: { role: "assistant", content: evidenceId }, + }, + order: { + traceSequence: position + 1, + timestamp: createdAt, + sessionPosition: position, + parentSpanId: null, + causalIdentifiers: [{ name: "diffex.task.lineage.id", value: lineageId }], + ambiguity: [], + }, + traceSources: [], + }; +} + +function invocation(): MaterializedInvocation { + return { + version: 1, + tracePath: "/private/trace-1.jsonl", + traceId: "trace-1", + producer: null, + integrity: "complete", + terminationStatus: "completed", + health: null, + ordering: { + criteria: ["trace_sequence", "session_cursor", "timestamp", "span_parentage", "causal_id"], + ambiguity: [], + }, + evidence: [evidence("evidence-a", "task-a", 0), evidence("evidence-b", "task-b", 1)], + diagnostics: [], + }; +} + +function inputSnapshot(): EvolutionInputSnapshot { + const materialized = invocation(); + const memory = "# Evolution memory\n"; + return { + metadata: metadata("input-1", ["source-1"], { sourceId: "source-1" }), + trigger: "manual", + repository: { + repositoryId: "diffex", + directoryId: "directory-1", + rootPath: "/repo", + revision: "revision-1", + }, + harness: { + harnessVersion, + baseVersion: `sha256:${"1".repeat(64)}`, + memory, + memoryContentDigest: computeEvolutionContentDigest(memory), + activeMemoryItems: [], + }, + expectedHarnessBaseVersion: harnessVersion, + ingestionHighWaterCursor: 1, + instructions: [ + { + path: "/repo/AGENTS.md", + precedence: 0, + content: "Run focused tests.", + contentDigest: computeEvolutionContentDigest("Run focused tests."), + }, + ], + skillCatalog: [ + { + skillId: "installed:focused-test", + name: "focused-test", + description: "Run focused tests.", + sourceKind: "installed", + scope: "workspace", + source: "project-resource", + version: "version-1", + contentDigest: `sha256:${"5".repeat(64)}`, + workspace: { repositoryId: "diffex", directoryId: "directory-1" }, + disableModelInvocation: false, + contentArtifact: { artifactId: "skill-content-1", contentDigest: `sha256:${"6".repeat(64)}` }, + }, + ], + modelConfiguration: { + provider: "faux", + model: "fixture", + configurationDigest: `sha256:${"4".repeat(64)}`, + }, + pipelineVersion: "pipeline-v1", + promptVersions: { + reflector: "reflector-v1", + curator: "curator-v1", + skillCurator: "skill-curator-v1", + skillSynthesizer: "skill-synthesizer-v1", + validator: "validator-v1", + writer: "writer-v1", + }, + tokenMeasurement: { estimatorVersion: "fixture", estimatedTokens: 100 }, + evidenceSources: [ + { + sourceId: "source-1", + materializationId: "materialization-1", + ingestionCursor: 1, + origin: "product-runtime", + contentDigest: computeEvolutionContentDigest(materialized), + invocation: materialized, + }, + ], + }; +} + +function candidate(input: EvolutionInputSnapshot): SkillCandidate { + const content = { + artifactKind: "skill-candidate" as const, + candidateId: "candidate-1", + operation: "UPDATE" as const, + workspace: { + repositoryId: input.repository.repositoryId, + directoryId: input.repository.directoryId, + rootPath: input.repository.rootPath, + repositoryRevision: input.repository.revision, + evolutionScope: input.metadata.scope, + }, + proposedName: "focused-verification", + existingSkillIds: ["installed:focused-test"], + sourceLessonIds: ["lesson-1"], + sourceProceduralObservationIds: [], + supportingEvidenceIds: ["evidence-a", "evidence-b"], + contradictingEvidenceIds: [], + supportingTraceIds: ["trace-1"], + contradictingTraceIds: [], + taskLineageIds: ["task-a", "task-b"], + skillability: { + classification: "verification-routine" as const, + rationale: "The procedure has deterministic outcomes.", + }, + trigger: { + description: "A focused repository change needs verification.", + positiveExamples: ["A source file changed."], + negativeExamples: ["No repository files changed."], + }, + preconditions: ["A focused test command exists."], + procedure: { + steps: [{ sequence: 1, action: "Run the focused test.", condition: null }], + requiredTools: ["shell"], + environment: ["Repository checkout"], + }, + failureHandling: [ + { + signal: "The test fails.", + mitigation: "Inspect the diagnostic.", + recovery: "Correct the issue and rerun the test.", + }, + ], + applicabilityBoundaries: ["Repositories with a focused test command."], + verificationContract: { + checks: ["The focused test exits successfully."], + completionCondition: "The focused test passes.", + }, + memoryRelationship: { + classification: "independent" as const, + memoryProposalIds: [], + rationale: "The workflow is distinct from selected memory guidance.", + }, + }; + return { metadata: metadata(content.candidateId, content.sourceLessonIds, content), ...content }; +} + +function completedJob(input: EvolutionInputSnapshot): EvolutionJob { + const inputReference = reference(input); + const content: Omit = { + jobId: "job-1", + inputSnapshot: inputReference, + trigger: "manual", + evidenceRange: { afterCursor: 0, throughCursor: 1 }, + stage: "completed", + outcome: "completed", + reasonCode: null, + failureMessage: null, + completion: { kind: "no-op", resultingHarnessVersion: null, processedEvidenceCursor: 1 }, + resolution: null, + checkpoints: [ + { + schemaVersion: 1, + stage: "skill-curating", + idempotencyKey: "job-1:skill-curating", + outputArtifacts: [], + state: { candidateIds: ["candidate-1"], deferredFindingIds: [], selectedMemoryProposalIds: [] }, + completedAt: createdAt, + }, + ], + attempt: 1, + revision: 1, + createdAt, + updatedAt: createdAt, + terminalAt: createdAt, + retryNotBefore: null, + }; + return { metadata: metadata(content.jobId, [inputReference.artifactId], content), ...content }; +} + +class MemoryArtifactStore implements EvolutionArtifactStore { + readonly artifacts: ReadonlyMap; + + constructor(artifacts: readonly EvolutionArtifact[]) { + this.artifacts = new Map(artifacts.map((artifact) => [reference(artifact).contentDigest, artifact])); + } + + async put(artifact: EvolutionArtifact): Promise { + return reference(artifact); + } + + async get(artifactReference: EvolutionArtifactReference): Promise { + const artifact = this.artifacts.get(artifactReference.contentDigest); + if (artifact === undefined) throw new Error("Unknown artifact"); + return artifact; + } +} + +class MemoryJobStore implements EvolutionJobStore { + readonly job: EvolutionJob; + + constructor(job: EvolutionJob) { + this.job = job; + } + + async create(): Promise {} + async save(): Promise {} + async load(jobId: string): Promise { + return jobId === this.job.jobId ? this.job : null; + } + async list(): Promise { + return [this.job]; + } +} + +const validMarkdown = `--- +name: focused-verification +description: Verify focused repository changes with a deterministic test and recovery routine. +--- + +Use only for Repositories with a focused test command. +Do not use when No repository files changed. +Run the focused test and recover from deterministic failures. +Verify that The focused test exits successfully. +Finish only when The focused test passes. +`; + +function fixture( + options: { readonly content?: string; readonly environment?: Partial } = {}, +) { + const input = inputSnapshot(); + const sourceCandidate = candidate(input); + const candidateReference = reference(sourceCandidate); + const job = completedJob(input); + const draftContent = { + artifactKind: "skill-draft" as const, + draftId: "draft-1", + groupId: "group-1", + synthesisIdempotencyKey: "skill-synthesis-1", + proposalRevisionIds: ["proposal-revision-1"], + sourceCandidateIds: ["candidate-1"], + sourceEvidenceIds: ["evidence-a", "evidence-b"], + sourceTraceIds: ["trace-1"], + sourceSkillVersions: [ + { + skillId: "installed:focused-test", + version: "version-1", + contentDigest: `sha256:${"5".repeat(64)}`, + }, + ], + repositoryInstructionDigests: [input.instructions[0]!.contentDigest], + operation: sourceCandidate.operation, + proposedName: sourceCandidate.proposedName, + existingSkillIds: sourceCandidate.existingSkillIds, + workspace: sourceCandidate.workspace, + content: options.content ?? validMarkdown, + }; + const draft: SkillDraft = { + metadata: metadata( + draftContent.draftId, + [ + ...draftContent.sourceCandidateIds, + ...draftContent.sourceEvidenceIds, + ...draftContent.sourceTraceIds, + ...draftContent.sourceSkillVersions.map((skill) => skill.skillId), + ...draftContent.repositoryInstructionDigests, + ].sort((left, right) => left.localeCompare(right)), + draftContent, + ), + ...draftContent, + }; + const draftReference = reference(draft); + const group: SkillCandidateGroup = { + groupId: "group-1", + profile: { + workspace: { repositoryId: "diffex", directoryId: "directory-1", evolutionScope: scope }, + existingSkillIds: sourceCandidate.existingSkillIds, + skillability: "verification-routine", + triggerTerms: ["focused", "verification"], + proceduralIntentTerms: ["focused", "test"], + applicabilityTerms: ["repository"], + }, + proposalRevisions: [ + { + revisionId: "proposal-revision-1", + candidateId: sourceCandidate.candidateId, + candidate: candidateReference, + sourceEvolutionJobId: job.jobId, + createdAt, + evidence: { + supportingEvidenceIds: sourceCandidate.supportingEvidenceIds, + contradictingEvidenceIds: [], + supportingTraceIds: sourceCandidate.supportingTraceIds, + contradictingTraceIds: [], + supportingTaskLineageIds: sourceCandidate.taskLineageIds, + contradictingTaskLineageIds: [], + }, + }, + ], + state: "drafted", + dispositionHistory: [], + evidence: { + supportingEvidenceIds: sourceCandidate.supportingEvidenceIds, + contradictingEvidenceIds: [], + supportingTraceIds: sourceCandidate.supportingTraceIds, + contradictingTraceIds: [], + supportingTaskLineageIds: sourceCandidate.taskLineageIds, + contradictingTaskLineageIds: [], + }, + priorityScore: null, + draft: draftReference, + validation: null, + synthesisIdempotencyKey: "skill-synthesis-1", + validationIdempotencyKey: "skill-validation-1", + evaluationIdempotencyKey: "skill-evaluation-1", + }; + const environment: SkillDraftValidationEnvironment = { + workspace: { repositoryId: "diffex", directoryId: "directory-1" }, + harnessVersion, + skillVersions: [ + { + skillId: "installed:focused-test", + version: "version-1", + contentDigest: `sha256:${"5".repeat(64)}`, + }, + ], + ...options.environment, + }; + return { + draft, + group, + validator: new DeterministicSkillDraftValidator({ + version: "validator-v1", + artifacts: new MemoryArtifactStore([input, sourceCandidate]), + jobs: new MemoryJobStore(job), + environment: { capture: async () => environment }, + }), + }; +} + +describe("DeterministicSkillDraftValidator", () => { + it("accepts only a current, closed, instruction-only skill draft", async () => { + const data = fixture(); + const report = await data.validator.validate({ + group: data.group, + draft: data.draft, + idempotencyKey: "skill-validation-1", + createdAt, + }); + + expect(report).toMatchObject({ + artifactKind: "skill-draft-validation", + groupId: "group-1", + validatorVersion: "validator-v1", + assurance: "structure-provenance-scope-and-security-only", + disposition: "accepted", + issues: [], + validatedHarnessVersion: harnessVersion, + canonicalInventory: ["skills/focused-verification/SKILL.md"], + }); + }); + + it.each([ + { + name: "sensitive content", + fixture: () => fixture({ content: `${validMarkdown}\napi_key = abcdefghijklmnopqrstuvwxyz123456\n` }), + code: "sensitive-data-detected", + }, + { + name: "undeclared local references", + fixture: () => fixture({ content: `${validMarkdown}\nSee [script](scripts/run.sh).\n` }), + code: "undeclared-reference", + }, + { + name: "stale harness bases", + fixture: () => fixture({ environment: { harnessVersion: `sha256:${"9".repeat(64)}` } }), + code: "harness-base-stale", + }, + { + name: "stale source skill versions", + fixture: () => + fixture({ + environment: { + skillVersions: [ + { + skillId: "installed:focused-test", + version: "version-2", + contentDigest: `sha256:${"7".repeat(64)}`, + }, + ], + }, + }), + code: "source-skill-stale", + }, + { + name: "omitted candidate boundaries", + fixture: () => + fixture({ + content: validMarkdown.replace("Repositories with a focused test command.", "Other repositories."), + }), + code: "applicability-boundary-missing", + }, + ] as const)("rejects $name", async ({ fixture: createFixture, code }) => { + const data = createFixture(); + const report = await data.validator.validate({ + group: data.group, + draft: data.draft, + idempotencyKey: "skill-validation-1", + createdAt, + }); + + expect(report.disposition).toBe("rejected"); + expect(report.issues).toContainEqual(expect.objectContaining({ code })); + }); +}); diff --git a/packages/evolution/test/skill-synthesizer.test.ts b/packages/evolution/test/skill-synthesizer.test.ts index 9102d750b..a0f26a8cb 100644 --- a/packages/evolution/test/skill-synthesizer.test.ts +++ b/packages/evolution/test/skill-synthesizer.test.ts @@ -380,7 +380,9 @@ function fixture(state: SkillCandidateGroup["state"] = "evidence-qualified") { }, priorityScore: null, draft: null, + validation: null, synthesisIdempotencyKey: "skill-synthesis-1", + validationIdempotencyKey: "skill-validation-1", evaluationIdempotencyKey: "skill-evaluation-1", }; return { @@ -447,6 +449,7 @@ describe("StructuredSkillSynthesizer", () => { generatedScriptsBinariesDependenciesAndExecutableAssetsPermitted: false, useSuccessfulOutcomesForPrimaryProcedure: true, useFailuresOnlyForPitfallsMitigationsRecoveryAndVerification: true, + retainCandidateApplicabilityBoundariesNegativeTriggersAndVerificationContractVerbatim: true, }, sourceEvidence: [ { evidenceId: "support-1", relationship: "supporting", terminationStatus: "completed" }, diff --git a/packages/harness-state/src/index.ts b/packages/harness-state/src/index.ts index 3c24b946e..6368bb3af 100644 --- a/packages/harness-state/src/index.ts +++ b/packages/harness-state/src/index.ts @@ -369,13 +369,13 @@ function isRecord(value: unknown): value is Readonly> { return typeof value === "object" && value !== null && !Array.isArray(value); } -interface ParsedSkillFrontmatter { +export interface HarnessSkillContentMetadata { readonly name: string; readonly description: string; readonly disableModelInvocation: boolean; } -function parseSkillFrontmatter(content: string, artifactPath: string): ParsedSkillFrontmatter { +function parseSkillFrontmatter(content: string, artifactPath: string): HarnessSkillContentMetadata { const match = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/.exec(content); if (match === null) throw new Error(`Harness skill "${artifactPath}" must start with YAML frontmatter`); let parsed: unknown; @@ -427,6 +427,16 @@ function validateArtifactPath(path: string): void { } } +export function validateHarnessSkillContent(path: string, content: string): HarnessSkillContentMetadata { + if (!Check(SkillContentSchema, content)) throw new Error(`Harness skill "${path}" has invalid content`); + validateArtifactPath(path); + const frontmatter = parseSkillFrontmatter(content, path); + if (path !== `skills/${frontmatter.name}/SKILL.md`) { + throw new Error(`Harness skill "${path}" name does not match its canonical path`); + } + return frontmatter; +} + async function inspectBundleRoot(bundlePath: string): Promise { const stats = await lstat(bundlePath); if (!stats.isDirectory() || stats.isSymbolicLink()) { @@ -582,10 +592,7 @@ export async function loadHarnessState(bundlePath: string): Promise(); for (const inputSkill of [...(input.skills ?? [])].sort((left, right) => compareText(left.path, right.path))) { - validateArtifactPath(inputSkill.path); + const frontmatter = validateHarnessSkillContent(inputSkill.path, inputSkill.content); const contentBytes = UTF8_ENCODER.encode(inputSkill.content); - const frontmatter = parseSkillFrontmatter(inputSkill.content, inputSkill.path); - if (inputSkill.path !== `skills/${frontmatter.name}/SKILL.md`) { - throw new Error(`Harness skill "${inputSkill.path}" name does not match its canonical path`); - } if (skillNames.has(frontmatter.name)) { throw new Error(`Harness skill name collision for "${frontmatter.name}"`); } diff --git a/packages/harness-state/test/harness-state.test.ts b/packages/harness-state/test/harness-state.test.ts index 065e4e536..810087240 100644 --- a/packages/harness-state/test/harness-state.test.ts +++ b/packages/harness-state/test/harness-state.test.ts @@ -31,6 +31,7 @@ import { loadHarnessState, saveHarnessState, serializeHarnessVersion, + validateHarnessSkillContent, } from "../src/index.ts"; const temporaryRoots: string[] = []; @@ -209,6 +210,20 @@ describe("canonical harness versions", () => { ).not.toBe(version); }); + test("validates an in-memory skill with the same canonical rules as a bundle", () => { + const content = skillContent("focused-review"); + + expect(validateHarnessSkillContent("skills/focused-review/SKILL.md", content)).toEqual({ + name: "focused-review", + description: "Use focused-review for focused work.", + disableModelInvocation: false, + }); + expect(() => validateHarnessSkillContent("skills/other/SKILL.md", content)).toThrow(/canonical path/i); + expect(() => validateHarnessSkillContent("skills/focused-review/SKILL.md", `\uFEFF${content}`)).toThrow( + /invalid content/i, + ); + }); + test("exports strict schemas", () => { const artifact = createArtifact(""); const manifest = createManifest([artifact]);