diff --git a/packages/evolution/COUNTER.md b/packages/evolution/COUNTER.md index 45614cfe0..50c1d23d9 100644 --- a/packages/evolution/COUNTER.md +++ b/packages/evolution/COUNTER.md @@ -50,3 +50,13 @@ The observation and report contracts carry the applicable policy version without This reservation does not add observation persistence, aggregation, attribution, scoring, pruning, trace selection metadata, or memory mutation. A future implementation may provide the existing `PreReflectionStage` port, persist raw observations in a dedicated append-only store, and emit a versioned `CounterReport`. That implementation must preserve the frozen job input, checkpoint artifact format, and explicit `counterReport` input already consumed by Reflector. + +## Skill lifecycle observations + +Skill lifecycle outcomes use a separate append-only store under evolution state. They are never written into `SKILL.md`, `EVOLVE.md`, or a harness revision, so recording a later outcome cannot rewrite the active or historical harness. + +Each observation binds one exact skill ID, version, content digest, harness revision, workspace identity, and top-level task lineage. Observable invocation events remain distinct (`available`, `listed`, `selected`, `explicitly-referenced`, `read`, and `force-loaded`); none is represented as inferred skill use. Co-occurrence can record only an `unknown` outcome and cannot carry a lifecycle issue classification. + +Final observations are unique per workspace, exact skill version, and top-level task lineage. Direct user attribution and deterministic test, tool, replay, or environment outcomes receive more report weight than model self-assessment or inference. Candidate observations remain auditable but do not contribute to final outcome weights or recommendations. + +Lifecycle reports consider recommendations only for the exact evolved skill versions active in the supplied workspace snapshot. Outcomes for the same immutable skill version remain applicable when that version reappears in a later harness revision, while the observation retains its original harness attribution. Different skill versions and installed skills remain available for audit but cannot target the evolved-skill writer. Reports can recommend narrowing triggers, revising a procedure, merging active evolved duplicates, deactivating a harmful skill, or retiring obsolete guidance. A recommendation is not a mutation: it requires a new Skill Curator candidate and the existing deterministic validation, retrieval evaluation, and behavioral evaluation gates before the skill writer can act. diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 46c7b7f6d..49f2814be 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -11,6 +11,8 @@ export const COUNTER_OBSERVATION_SCHEMA_VERSION = 1 as const; export const COUNTER_REPORT_SCHEMA_VERSION = 1 as const; export const SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION = 1 as const; export const SKILL_EVALUATION_SCHEMA_VERSION = 1 as const; +export const SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION = 1 as const; +export const SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION = 1 as const; export type JsonPrimitive = boolean | number | string | null; export type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[]; @@ -109,6 +111,106 @@ export interface CounterReport { readonly trends: readonly CounterReportTrend[]; } +export interface SkillLifecycleWorkspaceIdentity { + readonly repositoryId: string; + readonly directoryId?: string; +} + +export interface SkillLifecycleSkillVersion { + readonly skillId: string; + readonly version: string; + readonly contentDigest: string; + readonly sourceKind: EvolutionSkillSourceKind; + readonly harnessRevision: HarnessVersion; + readonly workspace: SkillLifecycleWorkspaceIdentity; +} + +export type SkillLifecycleInvocationEvent = + | "available" + | "explicitly-referenced" + | "force-loaded" + | "listed" + | "read" + | "selected"; + +export type SkillLifecycleIssueKind = + | "duplicate-guidance" + | "obsolete-guidance" + | "procedure-defect" + | "trigger-mismatch"; + +export interface SkillLifecycleObservation { + readonly schemaVersion: typeof SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION; + readonly observationId: string; + readonly scope: EvolutionScope; + readonly skill: SkillLifecycleSkillVersion; + readonly relatedSkills: readonly SkillLifecycleSkillVersion[]; + readonly invocationEvents: readonly SkillLifecycleInvocationEvent[]; + readonly topLevelTaskLineageId: string; + readonly traceIds: readonly string[]; + readonly evidenceIds: readonly string[]; + readonly signal: CounterSignal; + readonly signalSource: CounterSignalSource; + readonly attributionMethod: CounterAttributionMethod; + readonly issueKind: SkillLifecycleIssueKind | null; + readonly status: CounterObservationStatus; + readonly policyVersion: string; + readonly timestamp: string; + readonly idempotencyKey: string; +} + +export interface SkillLifecycleSignalWeights { + readonly helpful: number; + readonly harmful: number; + readonly neutral: number; + readonly unknown: number; +} + +export interface SkillLifecycleOutcomeSummary { + readonly skill: SkillLifecycleSkillVersion; + readonly finalObservationIds: readonly string[]; + readonly candidateObservationIds: readonly string[]; + readonly topLevelTaskLineageIds: readonly string[]; + readonly traceIds: readonly string[]; + readonly evidenceIds: readonly string[]; + readonly weightedSignals: SkillLifecycleSignalWeights; +} + +export type SkillLifecycleRecommendationAction = + | "deactivate-harmful" + | "merge-duplicates" + | "narrow-triggers" + | "retire-obsolete" + | "revise-procedure"; + +export interface SkillLifecycleRecommendation { + readonly recommendationId: string; + readonly action: SkillLifecycleRecommendationAction; + readonly candidateOperation: Extract; + readonly skillVersions: readonly SkillLifecycleSkillVersion[]; + readonly observationIds: readonly string[]; + readonly topLevelTaskLineageIds: readonly string[]; + readonly traceIds: readonly string[]; + readonly evidenceIds: readonly string[]; + readonly evidenceWeight: number; + readonly reasonCodes: readonly string[]; + readonly handoff: { + readonly kind: "skill-curator-candidate"; + readonly requiredGates: readonly ["deterministic-validation", "retrieval-evaluation", "behavioral-evaluation"]; + }; +} + +export interface SkillLifecycleReport { + readonly artifactKind: "skill-lifecycle-report"; + readonly schemaVersion: typeof SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION; + readonly metadata: EvolutionArtifactMetadata; + readonly policyVersion: string; + readonly activeSkillVersions: readonly SkillLifecycleSkillVersion[]; + readonly summaries: readonly SkillLifecycleOutcomeSummary[]; + readonly recommendations: readonly SkillLifecycleRecommendation[]; + readonly staleObservationIds: readonly string[]; +} + export interface ImmutableHarnessSnapshot { readonly harnessVersion: HarnessVersion; readonly baseVersion: HarnessVersion | null; diff --git a/packages/evolution/src/filesystem-store.ts b/packages/evolution/src/filesystem-store.ts index f009907bf..7984ea205 100644 --- a/packages/evolution/src/filesystem-store.ts +++ b/packages/evolution/src/filesystem-store.ts @@ -12,8 +12,14 @@ import type { EvolutionScope, SkillCandidateRegistryEvent, SkillCandidateRegistrySnapshot, + SkillLifecycleObservation, + SkillLifecycleSkillVersion, +} from "./contracts.ts"; +import { + EVOLUTION_SCHEMA_VERSION, + SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, + SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION, } from "./contracts.ts"; -import { EVOLUTION_SCHEMA_VERSION, SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION } from "./contracts.ts"; import type { EvolutionArtifact, EvolutionArtifactStore, @@ -26,6 +32,8 @@ import type { EvolutionLedger, SkillCandidateRegistryAppendResult, SkillCandidateRegistryStore, + SkillLifecycleObservationAppendResult, + SkillLifecycleObservationStore, } from "./ports.ts"; import { computeEvolutionContentDigest, serializeEvolutionValue } from "./provenance.ts"; @@ -76,6 +84,36 @@ const SKILL_CANDIDATE_WORK_STAGES = new Set([ "evaluating", "completed", ]); +const SKILL_LIFECYCLE_SIGNALS = new Set(["helpful", "harmful", "neutral", "unknown"]); +const SKILL_LIFECYCLE_SIGNAL_SOURCES = new Set([ + "direct-user", + "environment", + "model-self-assessment", + "replay", + "test", + "tool", +]); +const SKILL_LIFECYCLE_ATTRIBUTION_METHODS = new Set([ + "co-occurrence", + "deterministic", + "direct", + "inferred", + "self-assessed", +]); +const SKILL_LIFECYCLE_INVOCATION_EVENTS = new Set([ + "available", + "explicitly-referenced", + "force-loaded", + "listed", + "read", + "selected", +]); +const SKILL_LIFECYCLE_ISSUE_KINDS = new Set([ + "duplicate-guidance", + "obsolete-guidance", + "procedure-defect", + "trigger-mismatch", +]); const JOB_OUTCOMES = new Set([ "active", "completed", @@ -400,6 +438,67 @@ function parseSkillCandidateRegistryEvent(contents: string, path: string): Skill return parsed as unknown as SkillCandidateRegistryEvent; } +function isSkillLifecycleVersion(value: unknown): value is SkillLifecycleSkillVersion { + return ( + isRecord(value) && + typeof value.skillId === "string" && + value.skillId.length > 0 && + typeof value.version === "string" && + value.version.length > 0 && + typeof value.contentDigest === "string" && + DIGEST_PATTERN.test(value.contentDigest) && + (value.sourceKind === "evolved" || value.sourceKind === "installed") && + typeof value.harnessRevision === "string" && + DIGEST_PATTERN.test(value.harnessRevision) && + isRecord(value.workspace) && + typeof value.workspace.repositoryId === "string" && + value.workspace.repositoryId.length > 0 && + (value.workspace.directoryId === undefined || typeof value.workspace.directoryId === "string") + ); +} + +function parseSkillLifecycleObservation(contents: string, path: string): SkillLifecycleObservation { + const parsed = parseJson(contents, path); + if ( + !isRecord(parsed) || + parsed.schemaVersion !== SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION || + typeof parsed.observationId !== "string" || + !SAFE_ID_PATTERN.test(parsed.observationId) || + !isRecord(parsed.scope) || + typeof parsed.scope.repositoryId !== "string" || + typeof parsed.scope.scopeId !== "string" || + !isSkillLifecycleVersion(parsed.skill) || + parsed.skill.workspace.repositoryId !== parsed.scope.repositoryId || + !Array.isArray(parsed.relatedSkills) || + parsed.relatedSkills.some((skill) => !isSkillLifecycleVersion(skill)) || + !isStringArray(parsed.invocationEvents) || + parsed.invocationEvents.length === 0 || + parsed.invocationEvents.some((event) => !SKILL_LIFECYCLE_INVOCATION_EVENTS.has(event)) || + typeof parsed.topLevelTaskLineageId !== "string" || + parsed.topLevelTaskLineageId.length === 0 || + !isStringArray(parsed.traceIds) || + !isStringArray(parsed.evidenceIds) || + typeof parsed.signal !== "string" || + !SKILL_LIFECYCLE_SIGNALS.has(parsed.signal) || + typeof parsed.signalSource !== "string" || + !SKILL_LIFECYCLE_SIGNAL_SOURCES.has(parsed.signalSource) || + typeof parsed.attributionMethod !== "string" || + !SKILL_LIFECYCLE_ATTRIBUTION_METHODS.has(parsed.attributionMethod) || + (parsed.issueKind !== null && + (typeof parsed.issueKind !== "string" || !SKILL_LIFECYCLE_ISSUE_KINDS.has(parsed.issueKind))) || + (parsed.status !== "candidate" && parsed.status !== "final") || + typeof parsed.policyVersion !== "string" || + parsed.policyVersion.length === 0 || + typeof parsed.timestamp !== "string" || + !Number.isFinite(Date.parse(parsed.timestamp)) || + typeof parsed.idempotencyKey !== "string" || + parsed.idempotencyKey.length === 0 + ) { + throw new EvolutionStorageCorruptionError(path, "skill lifecycle observation is invalid"); + } + return parsed as unknown as SkillLifecycleObservation; +} + function parseCancellation(contents: string, path: string, expectedJobId: string): EvolutionCancellationRequest { const parsed = parseJson(contents, path); if ( @@ -469,7 +568,8 @@ export class FileEvolutionPersistence EvolutionJobStore, EvolutionLeaseStore, EvolutionLedger, - SkillCandidateRegistryStore + SkillCandidateRegistryStore, + SkillLifecycleObservationStore { readonly #rootPath: string; readonly #artifactPath: string; @@ -479,6 +579,7 @@ export class FileEvolutionPersistence readonly #leasePath: string; readonly #ledgerPath: string; readonly #skillCandidateRegistryPath: string; + readonly #skillLifecycleObservationPath: string; readonly #storeLockPath: string; readonly #storeLockStaleMs: number; readonly #storeLockUpdateMs: number; @@ -492,6 +593,7 @@ export class FileEvolutionPersistence this.#leasePath = join(this.#rootPath, "leases"); this.#ledgerPath = join(this.#rootPath, "ledger"); this.#skillCandidateRegistryPath = join(this.#rootPath, "skill-candidates"); + this.#skillLifecycleObservationPath = join(this.#rootPath, "skill-observations"); this.#storeLockPath = join(this.#rootPath, ".store"); this.#storeLockStaleMs = options.storeLockStaleMs ?? STORE_LOCK_STALE_MS; this.#storeLockUpdateMs = options.storeLockUpdateMs ?? STORE_LOCK_UPDATE_MS; @@ -511,6 +613,7 @@ export class FileEvolutionPersistence this.#leasePath, this.#ledgerPath, this.#skillCandidateRegistryPath, + this.#skillLifecycleObservationPath, ]) { await mkdir(path, { recursive: true, mode: 0o700 }); const stats = await lstat(path); @@ -632,6 +735,45 @@ export class FileEvolutionPersistence return events; } + #skillLifecycleScopePath(scope: EvolutionScope): string { + return join(this.#skillLifecycleObservationPath, this.#scopeFileName(scope)); + } + + async #readSkillLifecycleObservations(scope: EvolutionScope): Promise { + const scopePath = this.#skillLifecycleScopePath(scope); + let entries: Dirent[]; + try { + entries = await readdir(scopePath, { withFileTypes: true }); + } catch (error) { + if (hasErrorCode(error, "ENOENT")) return []; + throw error; + } + const observations: SkillLifecycleObservation[] = []; + for (const entry of entries) { + if (!entry.isDirectory() || entry.isSymbolicLink() || entry.name.startsWith(".")) continue; + const path = join(scopePath, entry.name, "observation.json"); + const contents = await this.#readOptional(path); + if (contents === null) { + throw new EvolutionStorageCorruptionError(path, "skill lifecycle observation is missing"); + } + const observation = parseSkillLifecycleObservation(contents, path); + if (observation.observationId !== entry.name) { + throw new EvolutionStorageCorruptionError(path, "skill lifecycle observation ID does not match its path"); + } + if (!scopeMatches(observation.scope, scope)) { + throw new EvolutionStorageCorruptionError( + path, + "skill lifecycle observation scope does not match its path", + ); + } + observations.push(observation); + } + return observations.sort( + (left, right) => + left.timestamp.localeCompare(right.timestamp) || left.observationId.localeCompare(right.observationId), + ); + } + #leaseFile(kind: EvolutionLeaseToken["kind"], key: string): string { const digest = artifactDirectoryName(computeEvolutionContentDigest({ kind, key })); return join(this.#leasePath, `${kind}-${digest}.json`); @@ -992,6 +1134,70 @@ export class FileEvolutionPersistence }); } + async listSkillLifecycleObservations(scope: EvolutionScope): Promise { + await this.#ensureDirectories(); + return this.#readSkillLifecycleObservations(scope); + } + + async appendSkillLifecycleObservation( + observation: SkillLifecycleObservation, + ): Promise { + assertSafeId(observation.observationId, "skill lifecycle observation ID"); + const content = serializeEvolutionValue(observation); + parseSkillLifecycleObservation(content, observation.observationId); + return this.#withStoreLock(async () => { + const existing = await this.#readSkillLifecycleObservations(observation.scope); + const prior = existing.find( + (item) => + item.observationId === observation.observationId || item.idempotencyKey === observation.idempotencyKey, + ); + if (prior !== undefined) { + if (serializeEvolutionValue(prior) !== content) { + throw new Error("Skill lifecycle observation identity was reused with different content"); + } + return "already-recorded"; + } + if ( + observation.status === "final" && + existing.some( + (item) => + item.status === "final" && + item.skill.skillId === observation.skill.skillId && + item.skill.version === observation.skill.version && + item.skill.workspace.repositoryId === observation.skill.workspace.repositoryId && + item.skill.workspace.directoryId === observation.skill.workspace.directoryId && + item.topLevelTaskLineageId === observation.topLevelTaskLineageId, + ) + ) { + throw new Error( + "A final skill lifecycle observation already exists for this skill version and task lineage", + ); + } + const scopePath = this.#skillLifecycleScopePath(observation.scope); + await mkdir(scopePath, { recursive: true, mode: 0o700 }); + const stats = await lstat(scopePath); + if (!stats.isDirectory() || stats.isSymbolicLink()) { + throw new EvolutionStorageCorruptionError(scopePath, "expected a direct directory"); + } + const destination = join(scopePath, observation.observationId); + let temporaryDirectory: string | null = await mkdtemp(join(scopePath, ".observation.tmp-")); + try { + await writeFile(join(temporaryDirectory, "observation.json"), `${content}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + await rename(temporaryDirectory, destination); + temporaryDirectory = null; + } finally { + if (temporaryDirectory !== null) { + await rm(temporaryDirectory, { recursive: true, force: true }); + } + } + return "committed"; + }); + } + async append(event: EvolutionLedgerEvent): Promise { assertSafeId(event.jobId, "job ID"); assertSafeId(event.eventId, "ledger event ID"); diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 78d9309c7..abdce2160 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -122,6 +122,8 @@ export { type ScoredProposalDisposition, SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, SKILL_EVALUATION_SCHEMA_VERSION, + SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION, + SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION, type SkillabilityClassification, type SkillBehaviorEvaluationOutcome, type SkillBehaviorEvaluationResult, @@ -165,6 +167,16 @@ export { type SkillEvaluationRuntimeConfiguration, type SkillEvaluationVerifier, type SkillEvaluationVerifierKind, + type SkillLifecycleInvocationEvent, + type SkillLifecycleIssueKind, + type SkillLifecycleObservation, + type SkillLifecycleOutcomeSummary, + type SkillLifecycleRecommendation, + type SkillLifecycleRecommendationAction, + type SkillLifecycleReport, + type SkillLifecycleSignalWeights, + type SkillLifecycleSkillVersion, + type SkillLifecycleWorkspaceIdentity, type SkillMemoryRelationshipClassification, type SkillMutation, type SkillMutationArtifact, @@ -299,6 +311,8 @@ export type { SkillEvaluationSelectionInput, SkillEvaluator, SkillEvaluatorInput, + SkillLifecycleObservationAppendResult, + SkillLifecycleObservationStore, SkillSynthesizer, SkillSynthesizerInput, SuccessfulMaterialization, @@ -378,6 +392,12 @@ export { type DeterministicSkillEvaluatorDependencies, SKILL_EVALUATOR_VERSION, } from "./skill-evaluator.ts"; +export { + DeterministicSkillLifecycleObserver, + type DeterministicSkillLifecycleObserverDependencies, + type SkillLifecycleObservationInput, + type SkillLifecycleReportInput, +} from "./skill-lifecycle.ts"; export { StructuredSkillSynthesizer, type StructuredSkillSynthesizerDependencies, diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 06970afc2..c99205763 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -46,6 +46,8 @@ import type { SkillEvaluationPoolSkill, SkillEvaluationReport, SkillEvaluationRuntimeConfiguration, + SkillLifecycleObservation, + SkillLifecycleReport, SkillMutationBatch, SkillRetrievalEvaluationResult, ValidationReport, @@ -71,6 +73,7 @@ export type EvolutionArtifact = | SkillDraft | SkillDraftValidationReport | SkillEvaluationReport + | SkillLifecycleReport | SkillMutationBatch | DeferredSkillCandidateFinding | ValidationReport; @@ -303,6 +306,15 @@ export interface SkillCandidateRegistryStore { ): Promise; } +export type SkillLifecycleObservationAppendResult = "already-recorded" | "committed"; + +export interface SkillLifecycleObservationStore { + appendSkillLifecycleObservation( + observation: SkillLifecycleObservation, + ): Promise; + listSkillLifecycleObservations(scope: EvolutionScope): Promise; +} + export interface SkillCandidateRegistryConsolidationInput { readonly sourceEvolutionJobId: string; readonly input: EvolutionInputSnapshot; diff --git a/packages/evolution/src/skill-lifecycle.ts b/packages/evolution/src/skill-lifecycle.ts new file mode 100644 index 000000000..66648c3c8 --- /dev/null +++ b/packages/evolution/src/skill-lifecycle.ts @@ -0,0 +1,343 @@ +import type { + CounterAttributionMethod, + CounterSignal, + CounterSignalSource, + EvolutionArtifactProducer, + EvolutionScope, + SkillLifecycleIssueKind, + SkillLifecycleObservation, + SkillLifecycleOutcomeSummary, + SkillLifecycleRecommendation, + SkillLifecycleRecommendationAction, + SkillLifecycleReport, + SkillLifecycleSkillVersion, +} from "./contracts.ts"; +import { SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION, SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION } from "./contracts.ts"; +import type { SkillLifecycleObservationStore } from "./ports.ts"; +import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; + +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; +const HANDOFF = { + kind: "skill-curator-candidate" as const, + requiredGates: ["deterministic-validation", "retrieval-evaluation", "behavioral-evaluation"] as const, +}; + +export interface SkillLifecycleObservationInput { + readonly scope: EvolutionScope; + readonly skill: SkillLifecycleSkillVersion; + readonly relatedSkills?: readonly SkillLifecycleSkillVersion[]; + readonly invocationEvents: SkillLifecycleObservation["invocationEvents"]; + readonly topLevelTaskLineageId: string; + readonly traceIds: readonly string[]; + readonly evidenceIds: readonly string[]; + readonly signal: CounterSignal; + readonly signalSource: CounterSignalSource; + readonly attributionMethod: CounterAttributionMethod; + readonly issueKind: SkillLifecycleIssueKind | null; + readonly status: SkillLifecycleObservation["status"]; + readonly timestamp: string; + readonly idempotencyKey: string; +} + +export interface SkillLifecycleReportInput { + readonly scope: EvolutionScope; + readonly activeSkillVersions: readonly SkillLifecycleSkillVersion[]; + readonly createdAt: string; + readonly idempotencyKey: string; +} + +export interface DeterministicSkillLifecycleObserverDependencies { + readonly version: string; + readonly policyVersion: string; + readonly observations: SkillLifecycleObservationStore; +} + +function uniqueSorted(values: readonly string[]): string[] { + return [...new Set(values)].sort((left, right) => left.localeCompare(right)); +} + +function sameWorkspace( + left: SkillLifecycleSkillVersion["workspace"], + right: SkillLifecycleSkillVersion["workspace"], +): boolean { + return left.repositoryId === right.repositoryId && left.directoryId === right.directoryId; +} + +function skillVersionKey(skill: SkillLifecycleSkillVersion): string { + return [ + skill.workspace.repositoryId, + skill.workspace.directoryId ?? "", + skill.sourceKind, + skill.skillId, + skill.version, + skill.contentDigest, + ].join("\u0000"); +} + +function sortSkillVersions(skills: readonly SkillLifecycleSkillVersion[]): SkillLifecycleSkillVersion[] { + return [...skills].sort( + (left, right) => + skillVersionKey(left).localeCompare(skillVersionKey(right)) || + left.harnessRevision.localeCompare(right.harnessRevision), + ); +} + +function assertSkillVersion(skill: SkillLifecycleSkillVersion, scope: EvolutionScope): void { + if ( + skill.skillId.trim() === "" || + skill.version.trim() === "" || + !DIGEST_PATTERN.test(skill.contentDigest) || + !DIGEST_PATTERN.test(skill.harnessRevision) || + (skill.sourceKind !== "evolved" && skill.sourceKind !== "installed") || + skill.workspace.repositoryId !== scope.repositoryId || + skill.workspace.repositoryId.trim() === "" + ) { + throw new Error(`Skill lifecycle identity for "${skill.skillId}" is invalid`); + } +} + +function attributionWeight(source: CounterSignalSource, method: CounterAttributionMethod): number { + if (source === "direct-user" && method === "direct") return 4; + if ( + method === "deterministic" && + (source === "environment" || source === "replay" || source === "test" || source === "tool") + ) { + return 3; + } + if (method === "co-occurrence") return 0.25; + if (source === "model-self-assessment" || method === "self-assessed") return 1; + if (method === "inferred") return 0.5; + return 1; +} + +function actionFor(observation: SkillLifecycleObservation): SkillLifecycleRecommendationAction | null { + if (observation.issueKind === "trigger-mismatch") return "narrow-triggers"; + if (observation.issueKind === "procedure-defect") return "revise-procedure"; + if (observation.issueKind === "duplicate-guidance") return "merge-duplicates"; + if (observation.issueKind === "obsolete-guidance") return "retire-obsolete"; + return observation.signal === "harmful" ? "deactivate-harmful" : null; +} + +function operationFor(action: SkillLifecycleRecommendationAction): SkillLifecycleRecommendation["candidateOperation"] { + if (action === "merge-duplicates") return "MERGE"; + if (action === "deactivate-harmful" || action === "retire-obsolete") return "RETIRE"; + return "UPDATE"; +} + +function reasonFor(action: SkillLifecycleRecommendationAction): string { + if (action === "narrow-triggers") return "observed-trigger-mismatch"; + if (action === "revise-procedure") return "observed-procedure-defect"; + if (action === "merge-duplicates") return "observed-duplicate-guidance"; + if (action === "retire-obsolete") return "observed-obsolete-guidance"; + return "observed-harmful-outcome"; +} + +function emptyWeights(): Record { + return { helpful: 0, harmful: 0, neutral: 0, unknown: 0 }; +} + +export class DeterministicSkillLifecycleObserver { + readonly #observations: SkillLifecycleObservationStore; + readonly #policyVersion: string; + readonly #producer: EvolutionArtifactProducer; + + constructor(dependencies: DeterministicSkillLifecycleObserverDependencies) { + if (dependencies.version.trim() === "" || dependencies.policyVersion.trim() === "") { + throw new Error("Skill lifecycle observer and policy versions must be non-empty"); + } + this.#observations = dependencies.observations; + this.#policyVersion = dependencies.policyVersion; + this.#producer = { + name: "diffex-skill-lifecycle-observer", + version: dependencies.version, + promptVersion: null, + model: null, + }; + } + + async record(input: SkillLifecycleObservationInput): Promise { + assertSkillVersion(input.skill, input.scope); + if ( + input.topLevelTaskLineageId.trim() === "" || + input.idempotencyKey.trim() === "" || + !Number.isFinite(Date.parse(input.timestamp)) || + input.invocationEvents.length === 0 || + new Set(input.invocationEvents).size !== input.invocationEvents.length || + input.traceIds.some((traceId) => traceId.trim() === "") || + input.evidenceIds.some((evidenceId) => evidenceId.trim() === "") || + (input.attributionMethod === "co-occurrence" && (input.signal !== "unknown" || input.issueKind !== null)) + ) { + throw new Error("Skill lifecycle observation input is invalid"); + } + const relatedSkills = sortSkillVersions(input.relatedSkills ?? []); + for (const skill of relatedSkills) { + assertSkillVersion(skill, input.scope); + if (skillVersionKey(skill) === skillVersionKey(input.skill)) { + throw new Error("A skill lifecycle observation cannot relate a skill version to itself"); + } + if (!sameWorkspace(skill.workspace, input.skill.workspace)) { + throw new Error("Related skill lifecycle observations must share one workspace"); + } + } + if (input.issueKind === "duplicate-guidance" && relatedSkills.length === 0) { + throw new Error("Duplicate skill guidance requires at least one related skill version"); + } + const identity = { + scope: input.scope, + skill: input.skill, + topLevelTaskLineageId: input.topLevelTaskLineageId, + idempotencyKey: input.idempotencyKey, + }; + const observation: SkillLifecycleObservation = { + schemaVersion: SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION, + observationId: `skill-observation-${computeEvolutionContentDigest(identity).slice("sha256:".length, "sha256:".length + 32)}`, + scope: input.scope, + skill: input.skill, + relatedSkills, + invocationEvents: uniqueSorted(input.invocationEvents) as SkillLifecycleObservation["invocationEvents"], + topLevelTaskLineageId: input.topLevelTaskLineageId, + traceIds: uniqueSorted(input.traceIds), + evidenceIds: uniqueSorted(input.evidenceIds), + signal: input.signal, + signalSource: input.signalSource, + attributionMethod: input.attributionMethod, + issueKind: input.issueKind, + status: input.status, + policyVersion: this.#policyVersion, + timestamp: input.timestamp, + idempotencyKey: input.idempotencyKey, + }; + await this.#observations.appendSkillLifecycleObservation(observation); + return observation; + } + + async report(input: SkillLifecycleReportInput): Promise { + if (input.idempotencyKey.trim() === "" || !Number.isFinite(Date.parse(input.createdAt))) { + throw new Error("Skill lifecycle report input is invalid"); + } + const activeSkillVersions = sortSkillVersions(input.activeSkillVersions); + const activeByKey = new Map(); + for (const skill of activeSkillVersions) { + assertSkillVersion(skill, input.scope); + const key = skillVersionKey(skill); + if (activeByKey.has(key)) throw new Error(`Duplicate active skill lifecycle identity "${skill.skillId}"`); + activeByKey.set(key, skill); + } + const observations = await this.#observations.listSkillLifecycleObservations(input.scope); + const current = observations.filter((observation) => activeByKey.has(skillVersionKey(observation.skill))); + const staleObservationIds = uniqueSorted( + observations + .filter((observation) => !activeByKey.has(skillVersionKey(observation.skill))) + .map((observation) => observation.observationId), + ); + const summaries: SkillLifecycleOutcomeSummary[] = []; + for (const skill of activeSkillVersions) { + const matching = current.filter( + (observation) => skillVersionKey(observation.skill) === skillVersionKey(skill), + ); + const final = matching.filter((observation) => observation.status === "final"); + const weightedSignals = emptyWeights(); + for (const observation of final) { + weightedSignals[observation.signal] += attributionWeight( + observation.signalSource, + observation.attributionMethod, + ); + } + summaries.push({ + skill, + finalObservationIds: uniqueSorted(final.map((observation) => observation.observationId)), + candidateObservationIds: uniqueSorted( + matching + .filter((observation) => observation.status === "candidate") + .map((observation) => observation.observationId), + ), + topLevelTaskLineageIds: uniqueSorted(final.map((observation) => observation.topLevelTaskLineageId)), + traceIds: uniqueSorted(final.flatMap((observation) => observation.traceIds)), + evidenceIds: uniqueSorted(final.flatMap((observation) => observation.evidenceIds)), + weightedSignals, + }); + } + const recommendationGroups = new Map(); + for (const observation of current) { + if (observation.status !== "final" || observation.skill.sourceKind !== "evolved") continue; + const action = actionFor(observation); + if (action === null) continue; + const skillVersions = sortSkillVersions( + [observation.skill, ...observation.relatedSkills].flatMap((skill) => { + const active = activeByKey.get(skillVersionKey(skill)); + return active === undefined ? [] : [active]; + }), + ); + if ( + skillVersions.length !== observation.relatedSkills.length + 1 || + (action === "merge-duplicates" && skillVersions.some((skill) => skill.sourceKind !== "evolved")) + ) { + continue; + } + const key = `${action}\u0000${skillVersions.map(skillVersionKey).join("\u0001")}`; + const grouped = recommendationGroups.get(key) ?? []; + grouped.push(observation); + recommendationGroups.set(key, grouped); + } + const recommendations = [...recommendationGroups.entries()] + .map(([key, grouped]): SkillLifecycleRecommendation => { + const first = grouped[0]; + if (first === undefined) throw new Error(`Empty skill lifecycle recommendation group "${key}"`); + const action = actionFor(first); + if (action === null) throw new Error("Skill lifecycle recommendation lost its action"); + const skillVersions = sortSkillVersions( + [first.skill, ...first.relatedSkills].flatMap((skill) => { + const active = activeByKey.get(skillVersionKey(skill)); + return active === undefined ? [] : [active]; + }), + ); + const observationIds = uniqueSorted(grouped.map((observation) => observation.observationId)); + return { + recommendationId: `skill-lifecycle-recommendation-${computeEvolutionContentDigest({ action, skillVersions, observationIds }).slice("sha256:".length, "sha256:".length + 32)}`, + action, + candidateOperation: operationFor(action), + skillVersions, + observationIds, + topLevelTaskLineageIds: uniqueSorted(grouped.map((observation) => observation.topLevelTaskLineageId)), + traceIds: uniqueSorted(grouped.flatMap((observation) => observation.traceIds)), + evidenceIds: uniqueSorted(grouped.flatMap((observation) => observation.evidenceIds)), + evidenceWeight: grouped.reduce( + (total, observation) => + total + attributionWeight(observation.signalSource, observation.attributionMethod), + 0, + ), + reasonCodes: [reasonFor(action)], + handoff: HANDOFF, + }; + }) + .sort( + (left, right) => + right.evidenceWeight - left.evidenceWeight || + left.recommendationId.localeCompare(right.recommendationId), + ); + const content = { + artifactKind: "skill-lifecycle-report" as const, + schemaVersion: SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION, + policyVersion: this.#policyVersion, + activeSkillVersions, + summaries, + recommendations, + staleObservationIds, + }; + const artifactId = `skill-lifecycle-report-${computeEvolutionContentDigest({ + idempotencyKey: input.idempotencyKey, + content, + }).slice("sha256:".length, "sha256:".length + 32)}`; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId, + producer: this.#producer, + sourceIds: uniqueSorted(observations.map((observation) => observation.observationId)), + scope: input.scope, + createdAt: input.createdAt, + content, + }), + ...content, + }; + } +} diff --git a/packages/evolution/test/counter.test.ts b/packages/evolution/test/counter.test.ts index 23b3ab140..4b48cad99 100644 --- a/packages/evolution/test/counter.test.ts +++ b/packages/evolution/test/counter.test.ts @@ -5,15 +5,19 @@ import { DisabledPreReflectionStage, EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION, type EvolutionMemoryItem, + SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION, + SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION, } from "../src/index.ts"; const HARNESS_VERSION = `sha256:${"1".repeat(64)}`; describe("reserved counter contracts", () => { - it("versions memory items, observations, and reports independently", () => { + it("versions memory and skill lifecycle records independently", () => { expect(EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION).toBe(1); expect(COUNTER_OBSERVATION_SCHEMA_VERSION).toBe(1); expect(COUNTER_REPORT_SCHEMA_VERSION).toBe(1); + expect(SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION).toBe(1); + expect(SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION).toBe(1); }); it("keeps stable memory identity outside rendered memory", () => { diff --git a/packages/evolution/test/filesystem-store.test.ts b/packages/evolution/test/filesystem-store.test.ts index 8f2476c95..1f2dec2cc 100644 --- a/packages/evolution/test/filesystem-store.test.ts +++ b/packages/evolution/test/filesystem-store.test.ts @@ -15,7 +15,9 @@ import { type InstructionSnapshot, planEvolutionRetention, SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION, + SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION, type SkillCandidateRegistryEvent, + type SkillLifecycleObservation, } from "../src/index.ts"; const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; @@ -164,6 +166,39 @@ function candidateEvaluationEvent(eventId: string, registrySequence: number): Sk }; } +function skillObservation( + observationId: string, + idempotencyKey: string, + topLevelTaskLineageId = "task-1", +): SkillLifecycleObservation { + return { + schemaVersion: SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION, + observationId, + scope, + skill: { + skillId: "skill:review", + version: "v1", + contentDigest: `sha256:${"b".repeat(64)}`, + sourceKind: "evolved", + harnessRevision: `sha256:${"c".repeat(64)}`, + workspace: { repositoryId: scope.repositoryId, directoryId: "workspace" }, + }, + relatedSkills: [], + invocationEvents: ["listed", "read"], + topLevelTaskLineageId, + traceIds: ["trace-1"], + evidenceIds: ["evidence-1"], + signal: "harmful", + signalSource: "test", + attributionMethod: "deterministic", + issueKind: "procedure-defect", + status: "final", + policyVersion: "skill-lifecycle-policy-v1", + timestamp: "2026-08-29T12:00:00.000Z", + idempotencyKey, + }; +} + async function scopeLease(store: FileEvolutionPersistence, ownerId: string): Promise { const lease = await store.acquire( "scope", @@ -245,6 +280,30 @@ describe("filesystem evolution persistence", () => { ).rejects.toThrow("contiguous sequence"); }); + it("persists skill lifecycle observations outside harness artifacts", async () => { + const { root, store } = await storeFixture(); + const peer = new FileEvolutionPersistence(root); + const observation = skillObservation("skill-observation-1", "observation-key-1"); + + expect(await store.appendSkillLifecycleObservation(observation)).toBe("committed"); + expect(await peer.appendSkillLifecycleObservation(observation)).toBe("already-recorded"); + expect(await peer.listSkillLifecycleObservations(scope)).toEqual([observation]); + await expect( + peer.appendSkillLifecycleObservation(skillObservation("skill-observation-2", "observation-key-2")), + ).rejects.toThrow("final skill lifecycle observation already exists"); + await expect(peer.appendSkillLifecycleObservation({ ...observation, signal: "helpful" })).rejects.toThrow( + "identity was reused with different content", + ); + + const scopeDirectory = computeEvolutionContentDigest(scope).replace("sha256:", "sha256-"); + await writeFile( + join(root, "skill-observations", scopeDirectory, observation.observationId, "observation.json"), + "{}\n", + "utf8", + ); + await expect(peer.listSkillLifecycleObservations(scope)).rejects.toBeInstanceOf(EvolutionStorageCorruptionError); + }); + it("detects corrupted candidate registry history", async () => { const { root, store } = await storeFixture(); const first = candidateWorkEvent("candidate-event-1", 1); diff --git a/packages/evolution/test/skill-lifecycle.test.ts b/packages/evolution/test/skill-lifecycle.test.ts new file mode 100644 index 000000000..8f19cad31 --- /dev/null +++ b/packages/evolution/test/skill-lifecycle.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it } from "vitest"; +import { + computeEvolutionContentDigest, + DeterministicSkillLifecycleObserver, + type EvolutionScope, + type SkillLifecycleObservation, + type SkillLifecycleObservationAppendResult, + type SkillLifecycleObservationInput, + type SkillLifecycleObservationStore, + type SkillLifecycleSkillVersion, +} from "../src/index.ts"; + +const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; +const digest = (value: string) => computeEvolutionContentDigest(value); + +class MemoryObservationStore implements SkillLifecycleObservationStore { + readonly observations: SkillLifecycleObservation[] = []; + + appendSkillLifecycleObservation( + observation: SkillLifecycleObservation, + ): Promise { + const existing = this.observations.find((item) => item.observationId === observation.observationId); + if (existing !== undefined) return Promise.resolve("already-recorded"); + this.observations.push(observation); + return Promise.resolve("committed"); + } + + listSkillLifecycleObservations(requestedScope: EvolutionScope): Promise { + return Promise.resolve( + this.observations.filter( + (observation) => + observation.scope.repositoryId === requestedScope.repositoryId && + observation.scope.scopeId === requestedScope.scopeId, + ), + ); + } +} + +function skill( + skillId: string, + version: string, + sourceKind: SkillLifecycleSkillVersion["sourceKind"] = "evolved", +): SkillLifecycleSkillVersion { + return { + skillId, + version, + contentDigest: digest(`${skillId}:${version}`), + sourceKind, + harnessRevision: digest(`harness:${version}`), + workspace: { repositoryId: scope.repositoryId, directoryId: "workspace" }, + }; +} + +function input( + target: SkillLifecycleSkillVersion, + overrides: Partial = {}, +): SkillLifecycleObservationInput { + return { + scope, + skill: target, + invocationEvents: ["listed", "read"], + topLevelTaskLineageId: "task-1", + traceIds: ["trace-1"], + evidenceIds: ["evidence-1"], + signal: "harmful", + signalSource: "direct-user", + attributionMethod: "direct", + issueKind: null, + status: "final", + timestamp: "2026-08-29T12:00:00.000Z", + idempotencyKey: "observation-1", + ...overrides, + }; +} + +function observer(store: SkillLifecycleObservationStore): DeterministicSkillLifecycleObserver { + return new DeterministicSkillLifecycleObserver({ + version: "1.0.0", + policyVersion: "skill-lifecycle-policy-v1", + observations: store, + }); +} + +describe("deterministic skill lifecycle observer", () => { + it("attributes outcomes to an exact workspace skill version", async () => { + const store = new MemoryObservationStore(); + const lifecycle = observer(store); + const target = skill("skill:review", "v1"); + + const recorded = await lifecycle.record(input(target)); + + expect(recorded).toMatchObject({ + skill: target, + invocationEvents: ["listed", "read"], + topLevelTaskLineageId: "task-1", + signal: "harmful", + status: "final", + }); + expect(recorded).not.toHaveProperty("content"); + expect(recorded).not.toHaveProperty("mutation"); + }); + + it("weights direct and deterministic outcomes above self-assessment", async () => { + const store = new MemoryObservationStore(); + const lifecycle = observer(store); + const direct = skill("skill:direct", "v1"); + const deterministic = skill("skill:test", "v1"); + const selfAssessed = skill("skill:self", "v1"); + await lifecycle.record(input(direct)); + await lifecycle.record( + input(deterministic, { + topLevelTaskLineageId: "task-2", + traceIds: ["trace-2"], + evidenceIds: ["evidence-2"], + signalSource: "test", + attributionMethod: "deterministic", + idempotencyKey: "observation-2", + }), + ); + await lifecycle.record( + input(selfAssessed, { + topLevelTaskLineageId: "task-3", + traceIds: ["trace-3"], + evidenceIds: ["evidence-3"], + signalSource: "model-self-assessment", + attributionMethod: "self-assessed", + idempotencyKey: "observation-3", + }), + ); + + const report = await lifecycle.report({ + scope, + activeSkillVersions: [selfAssessed, deterministic, direct], + createdAt: "2026-08-29T12:10:00.000Z", + idempotencyKey: "report-1", + }); + + expect(report.recommendations.map((recommendation) => recommendation.evidenceWeight)).toEqual([4, 3, 1]); + expect(report.summaries.map((summary) => summary.weightedSignals.harmful).sort()).toEqual([1, 3, 4]); + expect(report.recommendations[0]?.handoff).toEqual({ + kind: "skill-curator-candidate", + requiredGates: ["deterministic-validation", "retrieval-evaluation", "behavioral-evaluation"], + }); + }); + + it("proposes bounded candidate operations without changing active content", async () => { + const store = new MemoryObservationStore(); + const lifecycle = observer(store); + const trigger = skill("skill:trigger", "v1"); + const procedure = skill("skill:procedure", "v1"); + const duplicate = skill("skill:duplicate", "v1"); + const related = skill("skill:related", "v2"); + const obsolete = skill("skill:obsolete", "v1"); + const candidateOnly = skill("skill:candidate", "v1"); + const installed = skill("skill:installed", "v1", "installed"); + const priorHarness = { ...trigger, harnessRevision: digest("prior-harness") }; + const oldVersion = skill("skill:trigger", "v0"); + const cases: readonly [SkillLifecycleSkillVersion, Partial][] = [ + [trigger, { issueKind: "trigger-mismatch", idempotencyKey: "trigger" }], + [procedure, { issueKind: "procedure-defect", idempotencyKey: "procedure" }], + [duplicate, { issueKind: "duplicate-guidance", relatedSkills: [related], idempotencyKey: "duplicate" }], + [obsolete, { issueKind: "obsolete-guidance", idempotencyKey: "obsolete" }], + [candidateOnly, { status: "candidate", idempotencyKey: "candidate" }], + [installed, { idempotencyKey: "installed" }], + [ + priorHarness, + { + issueKind: "trigger-mismatch", + topLevelTaskLineageId: "task-from-prior-harness", + idempotencyKey: "prior-harness", + }, + ], + [oldVersion, { idempotencyKey: "old-version" }], + ]; + for (const [target, overrides] of cases) { + await lifecycle.record(input(target, overrides)); + } + + const report = await lifecycle.report({ + scope, + activeSkillVersions: [trigger, procedure, duplicate, related, obsolete, candidateOnly, installed], + createdAt: "2026-08-29T12:10:00.000Z", + idempotencyKey: "report-2", + }); + + expect( + report.recommendations.map(({ action, candidateOperation }) => [action, candidateOperation]).sort(), + ).toEqual( + [ + ["merge-duplicates", "MERGE"], + ["narrow-triggers", "UPDATE"], + ["retire-obsolete", "RETIRE"], + ["revise-procedure", "UPDATE"], + ].sort(), + ); + expect(report.staleObservationIds).toHaveLength(1); + expect( + report.recommendations.find((recommendation) => recommendation.action === "narrow-triggers"), + ).toMatchObject({ evidenceWeight: 8, skillVersions: [trigger] }); + expect(report.activeSkillVersions).toEqual([ + candidateOnly, + duplicate, + obsolete, + procedure, + related, + trigger, + installed, + ]); + expect(report).not.toHaveProperty("skillContent"); + expect(report).not.toHaveProperty("writerPayload"); + }); + + it("does not treat co-occurrence as a causal outcome", async () => { + const lifecycle = observer(new MemoryObservationStore()); + const target = skill("skill:available", "v1"); + + await expect( + lifecycle.record( + input(target, { + signal: "helpful", + signalSource: "environment", + attributionMethod: "co-occurrence", + }), + ), + ).rejects.toThrow("observation input is invalid"); + }); +});