From e88fb574081bdc9d95d94c46d3c85b03dd9d879f Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Wed, 19 Aug 2026 16:13:43 -0700 Subject: [PATCH 1/2] feat(agent): add durable evolution control - Preserves frozen evidence ranges across crashes and retries - Fences concurrent workers before cursor or harness mutation --- package-lock.json | 4 +- packages/evolution/package.json | 4 +- packages/evolution/src/contracts.ts | 208 +++-- packages/evolution/src/control-plane.ts | 869 +++++++++++++++++++++ packages/evolution/src/errors.ts | 20 + packages/evolution/src/filesystem-store.ts | 670 ++++++++++++++++ packages/evolution/src/index.ts | 84 +- packages/evolution/src/pipeline.ts | 823 ++++++++++--------- packages/evolution/src/ports.ts | 141 +++- packages/evolution/src/provenance.ts | 6 +- packages/evolution/src/recovery.ts | 80 ++ packages/evolution/src/targets.ts | 18 +- packages/evolution/src/token-estimator.ts | 52 ++ packages/evolution/src/worker.ts | 149 ++++ packages/evolution/test/pipeline.test.ts | 441 ----------- 15 files changed, 2682 insertions(+), 887 deletions(-) create mode 100644 packages/evolution/src/control-plane.ts create mode 100644 packages/evolution/src/errors.ts create mode 100644 packages/evolution/src/filesystem-store.ts create mode 100644 packages/evolution/src/recovery.ts create mode 100644 packages/evolution/src/token-estimator.ts create mode 100644 packages/evolution/src/worker.ts delete mode 100644 packages/evolution/test/pipeline.test.ts diff --git a/package-lock.json b/package-lock.json index e850c8e85..31f5b53b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5729,10 +5729,12 @@ "license": "MIT", "dependencies": { "@earendil-works/diffex-harness-state": "0.2.0", - "@earendil-works/diffex-harness-trace": "0.2.0" + "@earendil-works/diffex-harness-trace": "0.2.0", + "proper-lockfile": "4.1.2" }, "devDependencies": { "@types/node": "24.12.4", + "@types/proper-lockfile": "4.1.4", "shx": "0.4.0", "vitest": "4.1.9" }, diff --git a/packages/evolution/package.json b/packages/evolution/package.json index 278408651..5358a66d0 100644 --- a/packages/evolution/package.json +++ b/packages/evolution/package.json @@ -33,10 +33,12 @@ }, "dependencies": { "@earendil-works/diffex-harness-state": "0.2.0", - "@earendil-works/diffex-harness-trace": "0.2.0" + "@earendil-works/diffex-harness-trace": "0.2.0", + "proper-lockfile": "4.1.2" }, "devDependencies": { "@types/node": "24.12.4", + "@types/proper-lockfile": "4.1.4", "shx": "0.4.0", "vitest": "4.1.9" } diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 0729ae6a0..3781a797d 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -50,14 +50,45 @@ export type EvolutionEvidenceOrigin = "product-runtime" | "evolution-pipeline"; export interface EvolutionEvidenceSource { readonly sourceId: string; + readonly materializationId: string; + readonly ingestionCursor: number; readonly origin: EvolutionEvidenceOrigin; readonly contentDigest: string; readonly invocation: MaterializedInvocation; } +export type EvolutionTriggerKind = "automatic" | "manual"; + +export interface EvolutionRepositorySnapshot { + readonly repositoryId: string; + readonly rootPath: string; + readonly revision: string; +} + +export interface EvolutionTokenMeasurement { + readonly estimatorVersion: string; + readonly estimatedTokens: number; +} + +export interface EvolutionPromptVersions { + readonly reflector: string; + readonly curator: string; + readonly validator: string; + readonly writer: string; +} + export interface EvolutionInputSnapshot { readonly metadata: EvolutionArtifactMetadata; + readonly trigger: EvolutionTriggerKind; + readonly repository: EvolutionRepositorySnapshot; readonly harness: ImmutableHarnessSnapshot; + readonly expectedHarnessBaseVersion: HarnessVersion; + readonly ingestionHighWaterCursor: number; + readonly instructions: readonly InstructionDocumentSnapshot[]; + readonly modelConfiguration: EvolutionModelIdentity; + readonly pipelineVersion: string; + readonly promptVersions: EvolutionPromptVersions; + readonly tokenMeasurement: EvolutionTokenMeasurement; readonly evidenceSources: readonly EvolutionEvidenceSource[]; } @@ -205,95 +236,132 @@ export interface MutationBatch { export interface MutationWriteResult { readonly batchId: string; - readonly status: "applied" | "failed"; - readonly resultingHarnessVersion: HarnessVersion | null; + readonly status: "failed" | "written"; + readonly output: JsonObject | null; readonly message: string | null; } -export interface CreatedCheckpoint { - readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "created"; -} - -export interface CorpusCheckpoint { - readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "corpus-prepared"; - readonly corpusManifestId: string; - readonly instructionSnapshotId: string; +export interface MutationWriteReceipt { + readonly metadata: EvolutionArtifactMetadata; + readonly batchId: string; + readonly idempotencyKey: string; + readonly result: MutationWriteResult; } -export interface ReflectedCheckpoint { - readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "reflected"; - readonly reflectionNodeIds: readonly string[]; - readonly lessonIds: readonly string[]; +export interface HarnessActivationResult { + readonly batchId: string; + readonly status: "activated" | "failed"; + readonly resultingHarnessVersion: HarnessVersion | null; + readonly message: string | null; } -export interface CuratedCheckpoint { - readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "curated"; - readonly proposalIds: readonly string[]; -} +export interface HarnessActivationReceipt { + readonly metadata: EvolutionArtifactMetadata; + readonly batchId: string; + readonly idempotencyKey: string; + readonly result: HarnessActivationResult; +} + +export type EvolutionJobStage = + | "queued" + | "snapshotting" + | "counter-slot" + | "reflecting" + | "curating" + | "validating" + | "writing" + | "activating" + | "completed"; + +export type ActiveEvolutionStage = Exclude; + +export type EvolutionJobOutcome = + | "active" + | "completed" + | "interrupted" + | "retryable-failed" + | "terminal-failed" + | "superseded"; -export interface ValidatedCheckpoint { - readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "validated"; - readonly validationReportId: string; +export type EvolutionFailureReasonCode = + | "artifact-corrupt" + | "harness-base-conflict" + | "lease-lost" + | "model-call-failed" + | "pipeline-invariant" + | "shutdown" + | "snapshot-invalid" + | "stage-failed" + | "superseded-by-operator" + | "validation-rejected" + | "writer-failed"; + +export interface EvolutionArtifactReference { + readonly artifactId: string; + readonly contentDigest: string; } -export interface ScoredCheckpoint { +export interface EvolutionStageCheckpoint { readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "scored"; - readonly scoredProposalIds: readonly string[]; + readonly stage: ActiveEvolutionStage; + readonly idempotencyKey: string; + readonly outputArtifacts: readonly EvolutionArtifactReference[]; + readonly state: JsonObject; + readonly completedAt: string; } -export interface WrittenCheckpoint { - readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "written"; - readonly mutationBatchId: string | null; - readonly writeResult: MutationWriteResult | null; -} +export type EvolutionCompletionKind = "activated" | "no-op"; -export interface CompletedCheckpoint { - readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "completed"; +export interface EvolutionJobCompletion { + readonly kind: EvolutionCompletionKind; readonly resultingHarnessVersion: HarnessVersion | null; + readonly processedEvidenceCursor: number; } -export type ActiveEvolutionStage = - | "created" - | "corpus-prepared" - | "reflected" - | "curated" - | "validated" - | "scored" - | "written"; - -export interface FailedCheckpoint { - readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; - readonly stage: "failed"; - readonly failedStage: ActiveEvolutionStage; - readonly message: string; +export interface EvolutionJobResolution { + readonly kind: "administratively-resolved" | "superseded"; + readonly reasonCode: EvolutionFailureReasonCode; + readonly resolvedAt: string; } -export type EvolutionCheckpoint = - | CreatedCheckpoint - | CorpusCheckpoint - | ReflectedCheckpoint - | CuratedCheckpoint - | ValidatedCheckpoint - | ScoredCheckpoint - | WrittenCheckpoint - | CompletedCheckpoint - | FailedCheckpoint; - export interface EvolutionJob { readonly metadata: EvolutionArtifactMetadata; readonly jobId: string; - readonly inputSnapshotId: string; - readonly status: "completed" | "failed" | "running"; - readonly checkpoint: EvolutionCheckpoint; + readonly inputSnapshot: EvolutionArtifactReference; + readonly trigger: EvolutionTriggerKind; + readonly evidenceRange: { + readonly afterCursor: number; + readonly throughCursor: number; + }; + readonly stage: EvolutionJobStage; + readonly outcome: EvolutionJobOutcome; + readonly reasonCode: EvolutionFailureReasonCode | null; + readonly failureMessage: string | null; + readonly completion: EvolutionJobCompletion | null; + readonly resolution: EvolutionJobResolution | null; + readonly checkpoints: readonly EvolutionStageCheckpoint[]; + readonly attempt: number; + readonly revision: number; + readonly createdAt: string; readonly updatedAt: string; + readonly terminalAt: string | null; + readonly retryNotBefore: string | null; +} + +export interface EvolutionProcessedEvidenceCursor { + readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; + readonly scope: EvolutionScope; + readonly sequence: number; + readonly committedByJobId: string | null; + readonly updatedAt: string; +} + +export interface EvolutionLeaseToken { + readonly kind: "scope" | "writer"; + readonly key: string; + readonly ownerId: string; + readonly fence: number; + readonly expiresAt: string; } export type EvolutionProposalLedgerDisposition = @@ -317,7 +385,7 @@ export interface JobCreatedLedgerEvent extends EvolutionLedgerEventBase { export interface CheckpointSavedLedgerEvent extends EvolutionLedgerEventBase { readonly type: "checkpoint-saved"; - readonly checkpoint: EvolutionCheckpoint; + readonly checkpoint: EvolutionStageCheckpoint; } export interface ProposalDispositionLedgerEvent extends EvolutionLedgerEventBase { @@ -335,12 +403,16 @@ export interface MutationWrittenLedgerEvent extends EvolutionLedgerEventBase { export interface JobCompletedLedgerEvent extends EvolutionLedgerEventBase { readonly type: "job-completed"; + readonly completionKind: EvolutionCompletionKind; readonly resultingHarnessVersion: HarnessVersion | null; + readonly processedEvidenceCursor: number; } export interface JobFailedLedgerEvent extends EvolutionLedgerEventBase { readonly type: "job-failed"; - readonly failedStage: FailedCheckpoint["failedStage"]; + readonly failedStage: ActiveEvolutionStage; + readonly outcome: Extract; + readonly reasonCode: EvolutionFailureReasonCode; readonly message: string; } diff --git a/packages/evolution/src/control-plane.ts b/packages/evolution/src/control-plane.ts new file mode 100644 index 000000000..f3f566187 --- /dev/null +++ b/packages/evolution/src/control-plane.ts @@ -0,0 +1,869 @@ +import type { + ActiveEvolutionStage, + CheckpointSavedLedgerEvent, + EvolutionArtifactMetadata, + EvolutionArtifactProducer, + EvolutionFailureReasonCode, + EvolutionInputSnapshot, + EvolutionJob, + EvolutionJobCompletion, + EvolutionJobOutcome, + EvolutionJobResolution, + EvolutionJobStage, + EvolutionLeaseToken, + EvolutionScope, + EvolutionStageCheckpoint, + JobCompletedLedgerEvent, + JobCreatedLedgerEvent, + JobFailedLedgerEvent, + JsonObject, +} from "./contracts.ts"; +import { EVOLUTION_SCHEMA_VERSION } from "./contracts.ts"; +import { EvolutionExecutionError } from "./errors.ts"; +import type { + EvolutionArtifact, + EvolutionArtifactStore, + EvolutionClock, + EvolutionCursorStore, + EvolutionEventSink, + EvolutionEvidenceCatalog, + EvolutionIdGenerator, + EvolutionJobCreationRequest, + EvolutionJobRunner, + EvolutionJobStore, + EvolutionLeaseStore, + EvolutionLedger, + EvolutionSnapshotSource, + EvolutionStageDirective, + EvolutionStageExecutor, +} from "./ports.ts"; +import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; +import { + type AutomaticEvolutionEligibilityReason, + EVOLUTION_TOKEN_ESTIMATOR_VERSION, + estimateEvolutionTokens, + evaluateAutomaticEvolutionEligibility, +} from "./token-estimator.ts"; + +const DEFAULT_LEASE_TTL_MS = 30_000; +const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000; +const DEFAULT_RETRY_BASE_DELAY_MS = 5_000; +const DEFAULT_RETRY_MAX_DELAY_MS = 5 * 60_000; + +const NEXT_STAGE: Readonly> = { + queued: "snapshotting", + snapshotting: "counter-slot", + "counter-slot": "reflecting", + reflecting: "curating", + curating: "validating", + validating: "writing", + writing: "activating", + activating: "completed", +}; + +export type EvolutionJobCreationBlockReason = + | AutomaticEvolutionEligibilityReason + | "blocking-job" + | "no-new-evidence" + | "scope-busy"; + +export type EvolutionJobCreationResult = + | { readonly status: "created"; readonly job: EvolutionJob } + | { + readonly status: "not-created"; + readonly reason: EvolutionJobCreationBlockReason; + readonly blockingJobId?: string; + }; + +export interface EvolutionControlPlaneDependencies { + readonly version: string; + readonly clock: EvolutionClock; + readonly ids: EvolutionIdGenerator; + readonly artifacts: EvolutionArtifactStore; + readonly jobs: EvolutionJobStore; + readonly cursors: EvolutionCursorStore; + readonly leases: EvolutionLeaseStore; + readonly ledger: EvolutionLedger; + readonly events: EvolutionEventSink; + readonly evidence: EvolutionEvidenceCatalog; + readonly snapshots: EvolutionSnapshotSource; + readonly executor: EvolutionStageExecutor; +} + +export interface EvolutionControlPlaneOptions { + readonly leaseTtlMs?: number; + readonly heartbeatIntervalMs?: number; + readonly retryBaseDelayMs?: number; + readonly retryMaxDelayMs?: number; +} + +interface StageCompletionState extends JsonObject { + readonly controlDirective: EvolutionStageDirective; + readonly resultingHarnessVersion: string | null; +} + +class EvolutionCompletionPendingError extends Error { + constructor(jobId: string, options: ErrorOptions) { + super( + `Evolution job "${jobId}" reached a durable completion condition but its manifest is not finalized`, + options, + ); + this.name = "EvolutionCompletionPendingError"; + } +} + +function isInputSnapshot(artifact: EvolutionArtifact): artifact is EvolutionInputSnapshot { + return "ingestionHighWaterCursor" in artifact && "evidenceSources" in artifact && "harness" in artifact; +} + +function isOpenBlockingJob(job: EvolutionJob): boolean { + return job.resolution === null && job.outcome !== "completed" && job.outcome !== "superseded"; +} + +function rangesOverlap(left: EvolutionJob["evidenceRange"], right: EvolutionJob["evidenceRange"]): boolean { + return left.afterCursor < right.throughCursor && right.afterCursor < left.throughCursor; +} + +function stableId(kind: string, value: unknown): string { + return `${kind}-${computeEvolutionContentDigest(value).slice("sha256:".length, "sha256:".length + 32)}`; +} + +function abortError(signal: AbortSignal | undefined): unknown { + return signal?.reason ?? new Error("Evolution operation was interrupted"); +} + +function abortableDelay(delayMs: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(abortError(signal)); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timer); + reject(abortError(signal)); + }; + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +function combineSignals(signals: readonly (AbortSignal | undefined)[]): { + readonly signal: AbortSignal; + readonly dispose: () => void; +} { + const controller = new AbortController(); + const active = signals.filter((signal): signal is AbortSignal => signal !== undefined); + const listeners = new Map void>(); + for (const signal of active) { + if (signal.aborted) { + controller.abort(signal.reason); + break; + } + const listener = () => controller.abort(signal.reason); + listeners.set(signal, listener); + signal.addEventListener("abort", listener, { once: true }); + } + return { + signal: controller.signal, + dispose() { + for (const [signal, listener] of listeners) signal.removeEventListener("abort", listener); + }, + }; +} + +export class EvolutionControlPlane implements EvolutionJobRunner { + readonly #dependencies: EvolutionControlPlaneDependencies; + readonly #producer: EvolutionArtifactProducer; + readonly #leaseTtlMs: number; + readonly #heartbeatIntervalMs: number; + readonly #retryBaseDelayMs: number; + readonly #retryMaxDelayMs: number; + + constructor(dependencies: EvolutionControlPlaneDependencies, options: EvolutionControlPlaneOptions = {}) { + this.#dependencies = dependencies; + this.#producer = { + name: "diffex-evolution-control-plane", + version: dependencies.version, + promptVersion: null, + model: null, + }; + this.#leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS; + this.#heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS; + this.#retryBaseDelayMs = options.retryBaseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS; + this.#retryMaxDelayMs = options.retryMaxDelayMs ?? DEFAULT_RETRY_MAX_DELAY_MS; + if (this.#leaseTtlMs < 2_000) throw new Error("leaseTtlMs must be at least 2000"); + if (this.#heartbeatIntervalMs < 1 || this.#heartbeatIntervalMs >= this.#leaseTtlMs) { + throw new Error("heartbeatIntervalMs must be positive and less than leaseTtlMs"); + } + if (this.#retryBaseDelayMs < 0 || this.#retryMaxDelayMs < this.#retryBaseDelayMs) { + throw new Error("Evolution retry delays are invalid"); + } + } + + #metadata( + artifactId: string, + sourceIds: readonly string[], + scope: EvolutionScope, + createdAt: string, + content: unknown, + ): EvolutionArtifactMetadata { + return createEvolutionArtifactMetadata({ + artifactId, + producer: this.#producer, + sourceIds, + scope, + createdAt, + content, + }); + } + + #scopeKey(scope: EvolutionScope): string { + return computeEvolutionContentDigest(scope); + } + + async #withHeartbeat( + initialLease: EvolutionLeaseToken, + signal: AbortSignal | undefined, + operation: (lease: () => EvolutionLeaseToken, signal: AbortSignal) => Promise, + ): Promise { + let currentLease = initialLease; + const heartbeatController = new AbortController(); + const compromisedController = new AbortController(); + const combined = combineSignals([signal, compromisedController.signal]); + const heartbeat = (async () => { + try { + while (!heartbeatController.signal.aborted) { + await abortableDelay(this.#heartbeatIntervalMs, heartbeatController.signal); + const renewed = await this.#dependencies.leases.renew( + currentLease, + this.#dependencies.clock.now(), + this.#leaseTtlMs, + ); + if (renewed === null) { + compromisedController.abort( + new EvolutionExecutionError(`Evolution ${currentLease.kind} lease was lost`, { + reasonCode: "lease-lost", + retryable: true, + }), + ); + return; + } + currentLease = renewed; + } + } catch (error) { + if (!heartbeatController.signal.aborted) compromisedController.abort(error); + } + })(); + try { + return await operation(() => currentLease, combined.signal); + } finally { + heartbeatController.abort(); + combined.dispose(); + await heartbeat.catch(() => {}); + await this.#dependencies.leases.release(currentLease); + } + } + + async #acquireScopeLease(scope: EvolutionScope, ownerId: string): Promise { + return this.#dependencies.leases.acquire( + "scope", + this.#scopeKey(scope), + ownerId, + this.#dependencies.clock.now(), + this.#leaseTtlMs, + ); + } + + #jobContent(job: Omit): Omit { + return job; + } + + #withJobRevision( + job: EvolutionJob, + changes: Partial>, + ): EvolutionJob { + const updatedAt = this.#dependencies.clock.now(); + const { metadata: _metadata, ...current } = job; + const nextWithoutMetadata: Omit = { + ...current, + ...changes, + jobId: job.jobId, + createdAt: job.createdAt, + updatedAt, + revision: job.revision + 1, + }; + return { + ...nextWithoutMetadata, + metadata: this.#metadata( + job.jobId, + [job.inputSnapshot.artifactId], + job.metadata.scope, + job.createdAt, + this.#jobContent(nextWithoutMetadata), + ), + }; + } + + #ledgerMetadata(job: EvolutionJob, eventId: string, content: unknown): EvolutionArtifactMetadata { + return this.#metadata(eventId, [job.jobId], job.metadata.scope, job.createdAt, content); + } + + #jobCreatedEvent(job: EvolutionJob): JobCreatedLedgerEvent { + const eventId = stableId("ledger-event", { jobId: job.jobId, type: "job-created" }); + const content = { inputSnapshotId: job.inputSnapshot.artifactId }; + return { + metadata: this.#ledgerMetadata(job, eventId, content), + eventId, + jobId: job.jobId, + type: "job-created", + inputSnapshotId: job.inputSnapshot.artifactId, + }; + } + + #checkpointEvent(job: EvolutionJob, checkpoint: EvolutionStageCheckpoint): CheckpointSavedLedgerEvent { + const eventId = stableId("ledger-event", { + jobId: job.jobId, + type: "checkpoint-saved", + stage: checkpoint.stage, + idempotencyKey: checkpoint.idempotencyKey, + }); + return { + metadata: this.#ledgerMetadata(job, eventId, checkpoint), + eventId, + jobId: job.jobId, + type: "checkpoint-saved", + checkpoint, + }; + } + + #completedEvent(job: EvolutionJob): JobCompletedLedgerEvent { + if (job.completion === null) throw new Error(`Completed evolution job "${job.jobId}" has no completion`); + const content = job.completion; + const eventId = stableId("ledger-event", { jobId: job.jobId, type: "job-completed", content }); + return { + metadata: this.#ledgerMetadata(job, eventId, content), + eventId, + jobId: job.jobId, + type: "job-completed", + completionKind: job.completion.kind, + resultingHarnessVersion: job.completion.resultingHarnessVersion, + processedEvidenceCursor: job.completion.processedEvidenceCursor, + }; + } + + #failedEvent(job: EvolutionJob): JobFailedLedgerEvent { + if ( + job.stage === "completed" || + job.reasonCode === null || + job.failureMessage === null || + (job.outcome !== "interrupted" && job.outcome !== "retryable-failed" && job.outcome !== "terminal-failed") + ) { + throw new Error(`Evolution job "${job.jobId}" is not failed`); + } + const content = { + failedStage: job.stage, + outcome: job.outcome, + reasonCode: job.reasonCode, + message: job.failureMessage, + revision: job.revision, + }; + const eventId = stableId("ledger-event", { jobId: job.jobId, type: "job-failed", content }); + return { + metadata: this.#ledgerMetadata(job, eventId, content), + eventId, + jobId: job.jobId, + type: "job-failed", + failedStage: job.stage, + outcome: job.outcome, + reasonCode: job.reasonCode, + message: job.failureMessage, + }; + } + + async #syncLedger(job: EvolutionJob): Promise { + await this.#dependencies.ledger.append(this.#jobCreatedEvent(job)); + for (const checkpoint of job.checkpoints) { + await this.#dependencies.ledger.append(this.#checkpointEvent(job, checkpoint)); + } + if (job.outcome === "completed") await this.#dependencies.ledger.append(this.#completedEvent(job)); + if (job.outcome === "interrupted" || job.outcome === "retryable-failed" || job.outcome === "terminal-failed") { + await this.#dependencies.ledger.append(this.#failedEvent(job)); + } + } + + async createJob(request: EvolutionJobCreationRequest): Promise { + request.signal?.throwIfAborted(); + const ownerId = this.#dependencies.ids.nextId("lease-owner"); + const initialLease = await this.#acquireScopeLease(request.scope, ownerId); + if (initialLease === null) return { status: "not-created", reason: "scope-busy" }; + return this.#withHeartbeat(initialLease, request.signal, async (lease, signal) => { + const processedCursor = await this.#dependencies.cursors.load(request.scope); + const ingestionHighWaterCursor = await this.#dependencies.evidence.highWaterCursor(request.scope); + if (!Number.isSafeInteger(ingestionHighWaterCursor) || ingestionHighWaterCursor < processedCursor.sequence) { + throw new EvolutionExecutionError("Materialization high-water cursor is invalid", { + reasonCode: "snapshot-invalid", + retryable: false, + }); + } + const materializations = await this.#dependencies.evidence.listSuccessful( + request.scope, + processedCursor.sequence, + ingestionHighWaterCursor, + ); + const seenMaterializations = new Set(); + let previousCursor = processedCursor.sequence; + for (const materialization of materializations) { + if ( + seenMaterializations.has(materialization.materializationId) || + materialization.ingestionCursor <= previousCursor || + materialization.ingestionCursor > ingestionHighWaterCursor || + computeEvolutionContentDigest(materialization.invocation) !== materialization.contentDigest + ) { + throw new EvolutionExecutionError("Materialization catalog returned an invalid frozen range", { + reasonCode: "snapshot-invalid", + retryable: false, + }); + } + seenMaterializations.add(materialization.materializationId); + previousCursor = materialization.ingestionCursor; + } + + if (materializations.length === 0) return { status: "not-created", reason: "no-new-evidence" }; + const evidenceRange = { + afterCursor: processedCursor.sequence, + throughCursor: ingestionHighWaterCursor, + }; + const scopeJobs = await this.#dependencies.jobs.list(request.scope); + const blocking = scopeJobs.find( + (job) => + isOpenBlockingJob(job) && + (rangesOverlap(job.evidenceRange, evidenceRange) || + (request.trigger === "automatic" && job.trigger === "automatic")), + ); + if (blocking !== undefined) { + return { status: "not-created", reason: "blocking-job", blockingJobId: blocking.jobId }; + } + + const estimatedTokens = estimateEvolutionTokens(materializations.map((item) => item.invocation)); + if (request.trigger === "automatic") { + const previousTerminalAt = + scopeJobs + .filter((job) => job.trigger === "automatic" && job.terminalAt !== null) + .map((job) => job.terminalAt as string) + .sort() + .at(-1) ?? null; + const eligibility = evaluateAutomaticEvolutionEligibility({ + invocationCount: materializations.length, + estimatedTokens, + now: this.#dependencies.clock.now(), + previousAutomaticTerminalAt: previousTerminalAt, + }); + if (!eligibility.eligible) return { status: "not-created", reason: eligibility.reason }; + } + + signal.throwIfAborted(); + const frozen = await this.#dependencies.snapshots.capture(request.scope, signal); + if (frozen.repository.repositoryId !== request.scope.repositoryId) { + throw new EvolutionExecutionError("Snapshot repository identity does not match the evolution scope", { + reasonCode: "snapshot-invalid", + retryable: false, + }); + } + if (computeEvolutionContentDigest(frozen.harness.memory) !== frozen.harness.memoryContentDigest) { + throw new EvolutionExecutionError("Frozen evolution memory digest is invalid", { + reasonCode: "snapshot-invalid", + retryable: false, + }); + } + for (const document of frozen.instructions) { + if (computeEvolutionContentDigest(document.content) !== document.contentDigest) { + throw new EvolutionExecutionError(`Frozen instruction digest is invalid for "${document.path}"`, { + reasonCode: "snapshot-invalid", + retryable: false, + }); + } + } + + const createdAt = this.#dependencies.clock.now(); + const snapshotId = this.#dependencies.ids.nextId("input-snapshot"); + const snapshotContent = { + trigger: request.trigger, + repository: frozen.repository, + harness: frozen.harness, + expectedHarnessBaseVersion: frozen.harness.harnessVersion, + ingestionHighWaterCursor, + instructions: frozen.instructions, + modelConfiguration: frozen.modelConfiguration, + pipelineVersion: frozen.pipelineVersion, + promptVersions: frozen.promptVersions, + tokenMeasurement: { + estimatorVersion: EVOLUTION_TOKEN_ESTIMATOR_VERSION, + estimatedTokens, + }, + evidenceSources: materializations.map((materialization) => ({ + sourceId: materialization.materializationId, + materializationId: materialization.materializationId, + ingestionCursor: materialization.ingestionCursor, + origin: materialization.origin, + contentDigest: materialization.contentDigest, + invocation: materialization.invocation, + })), + }; + const snapshot: EvolutionInputSnapshot = { + metadata: this.#metadata( + snapshotId, + materializations.map((item) => item.materializationId), + request.scope, + createdAt, + snapshotContent, + ), + ...snapshotContent, + }; + const snapshotReference = await this.#dependencies.artifacts.put(snapshot); + const jobId = this.#dependencies.ids.nextId("job"); + const jobWithoutMetadata: Omit = { + jobId, + inputSnapshot: snapshotReference, + trigger: request.trigger, + evidenceRange, + stage: "queued", + outcome: "active", + reasonCode: null, + failureMessage: null, + completion: null, + resolution: null, + checkpoints: [], + attempt: 0, + revision: 0, + createdAt, + updatedAt: createdAt, + terminalAt: null, + retryNotBefore: null, + }; + const job: EvolutionJob = { + ...jobWithoutMetadata, + metadata: this.#metadata(jobId, [snapshotId], request.scope, createdAt, jobWithoutMetadata), + }; + await this.#dependencies.jobs.create(job, lease()); + await this.#syncLedger(job); + await this.#dependencies.events.emit({ jobId, emittedAt: createdAt, type: "started", stage: "queued" }); + return { status: "created", job }; + }); + } + + async #loadPriorArtifacts(job: EvolutionJob): Promise { + const artifacts: EvolutionArtifact[] = []; + for (const checkpoint of job.checkpoints) { + for (const reference of checkpoint.outputArtifacts) { + artifacts.push(await this.#dependencies.artifacts.get(reference)); + } + } + return artifacts; + } + + #checkpointDirective(checkpoint: EvolutionStageCheckpoint): { + readonly directive: EvolutionStageDirective; + readonly resultingHarnessVersion: string | null; + } { + const directive = checkpoint.state.controlDirective; + const resultingHarnessVersion = checkpoint.state.resultingHarnessVersion; + if ( + (directive !== "continue" && directive !== "complete-no-op" && directive !== "complete-activated") || + (typeof resultingHarnessVersion !== "string" && resultingHarnessVersion !== null) + ) { + throw new EvolutionExecutionError("Stage checkpoint completion state is corrupt", { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } + return { directive, resultingHarnessVersion }; + } + + async #completeJob( + job: EvolutionJob, + checkpoint: EvolutionStageCheckpoint, + lease: EvolutionLeaseToken, + ): Promise { + const { directive, resultingHarnessVersion } = this.#checkpointDirective(checkpoint); + if (directive === "continue") throw new Error("Cannot complete a continuing evolution stage"); + const cursorResult = await this.#dependencies.cursors.commit( + job.metadata.scope, + job.evidenceRange.afterCursor, + job.evidenceRange.throughCursor, + job.jobId, + this.#dependencies.clock.now(), + lease, + ); + if (cursorResult === "conflict") { + throw new EvolutionExecutionError("Processed-evidence cursor changed before job completion", { + reasonCode: "pipeline-invariant", + retryable: false, + }); + } + const completion: EvolutionJobCompletion = { + kind: directive === "complete-no-op" ? "no-op" : "activated", + resultingHarnessVersion: directive === "complete-no-op" ? null : resultingHarnessVersion, + processedEvidenceCursor: job.evidenceRange.throughCursor, + }; + const completed = this.#withJobRevision(job, { + stage: "completed", + outcome: "completed", + completion, + reasonCode: null, + failureMessage: null, + terminalAt: this.#dependencies.clock.now(), + retryNotBefore: null, + }); + try { + await this.#dependencies.jobs.save(completed, job.revision, lease); + } catch (cause) { + throw new EvolutionCompletionPendingError(job.jobId, { cause }); + } + await this.#syncLedger(completed); + await this.#dependencies.events.emit({ + jobId: completed.jobId, + emittedAt: completed.updatedAt, + type: "completed", + stage: "completed", + }); + return completed; + } + + async #executeStage( + job: EvolutionJob, + input: EvolutionInputSnapshot, + scopeLease: EvolutionLeaseToken, + writerLease: EvolutionLeaseToken | null, + signal: AbortSignal, + ): Promise { + if (job.stage === "completed") return job; + const idempotencyKey = stableId("stage", { jobId: job.jobId, stage: job.stage }); + const result = await this.#dependencies.executor.execute({ + job, + input, + priorArtifacts: await this.#loadPriorArtifacts(job), + idempotencyKey, + scopeLease, + writerLease, + signal, + }); + signal.throwIfAborted(); + const outputArtifacts = []; + for (const artifact of result.artifacts) outputArtifacts.push(await this.#dependencies.artifacts.put(artifact)); + for (const event of result.ledgerEvents ?? []) await this.#dependencies.ledger.append(event); + const checkpoint: EvolutionStageCheckpoint = { + schemaVersion: EVOLUTION_SCHEMA_VERSION, + stage: job.stage, + idempotencyKey, + outputArtifacts, + state: { + ...result.state, + controlDirective: result.directive, + resultingHarnessVersion: result.resultingHarnessVersion ?? null, + } satisfies StageCompletionState, + completedAt: this.#dependencies.clock.now(), + }; + const checkpointed = this.#withJobRevision(job, { + stage: result.directive === "continue" ? NEXT_STAGE[job.stage] : job.stage, + checkpoints: [...job.checkpoints, checkpoint], + }); + await this.#dependencies.jobs.save(checkpointed, job.revision, scopeLease); + await this.#dependencies.ledger.append(this.#checkpointEvent(checkpointed, checkpoint)); + await this.#dependencies.events.emit({ + jobId: checkpointed.jobId, + emittedAt: checkpoint.completedAt, + type: "checkpoint", + stage: checkpoint.stage, + checkpoint, + }); + if (result.directive !== "continue") return this.#completeJob(checkpointed, checkpoint, scopeLease); + return checkpointed; + } + + async #markFailure( + job: EvolutionJob, + error: unknown, + lease: EvolutionLeaseToken, + interrupted: boolean, + ): Promise { + const executionError = error instanceof EvolutionExecutionError ? error : null; + const outcome: Extract = interrupted + ? "interrupted" + : executionError?.retryable === false + ? "terminal-failed" + : "retryable-failed"; + const reasonCode: EvolutionFailureReasonCode = interrupted + ? "shutdown" + : (executionError?.reasonCode ?? "stage-failed"); + const now = this.#dependencies.clock.now(); + const retryDelay = Math.min(this.#retryBaseDelayMs * 2 ** Math.max(job.attempt, 0), this.#retryMaxDelayMs); + const failed = this.#withJobRevision(job, { + outcome, + reasonCode, + failureMessage: error instanceof Error ? error.message : String(error), + terminalAt: now, + retryNotBefore: outcome === "retryable-failed" ? new Date(Date.parse(now) + retryDelay).toISOString() : null, + }); + await this.#dependencies.jobs.save(failed, job.revision, lease); + await this.#syncLedger(failed); + await this.#dependencies.events.emit({ + jobId: failed.jobId, + emittedAt: failed.updatedAt, + type: "failed", + stage: failed.stage, + }); + return failed; + } + + async runJob(jobId: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const initial = await this.#dependencies.jobs.load(jobId); + if (initial === null) throw new Error(`Unknown evolution job "${jobId}"`); + await this.#syncLedger(initial); + if (initial.outcome === "completed") return initial; + if (initial.outcome === "superseded" || initial.outcome === "terminal-failed" || initial.resolution !== null) { + throw new Error(`Evolution job "${jobId}" cannot be resumed from outcome "${initial.outcome}"`); + } + if ( + initial.retryNotBefore !== null && + Date.parse(initial.retryNotBefore) > Date.parse(this.#dependencies.clock.now()) + ) { + throw new Error(`Evolution job "${jobId}" is not eligible for retry yet`); + } + + const ownerId = this.#dependencies.ids.nextId("lease-owner"); + const initialLease = await this.#acquireScopeLease(initial.metadata.scope, ownerId); + if (initialLease === null) { + throw new EvolutionExecutionError(`Evolution scope for job "${jobId}" is already leased`, { + reasonCode: "lease-lost", + retryable: true, + }); + } + return this.#withHeartbeat(initialLease, signal, async (scopeLease, leaseSignal) => { + const loadedJob = await this.#dependencies.jobs.load(jobId); + if (loadedJob === null) throw new Error(`Unknown evolution job "${jobId}"`); + let job: EvolutionJob = loadedJob; + if (job.outcome === "completed") return job; + if (job.outcome === "interrupted" || job.outcome === "retryable-failed") { + const resumed = this.#withJobRevision(job, { + outcome: "active", + reasonCode: null, + failureMessage: null, + terminalAt: null, + retryNotBefore: null, + attempt: job.attempt + 1, + }); + await this.#dependencies.jobs.save(resumed, job.revision, scopeLease()); + job = resumed; + } + try { + const inputArtifact = await this.#dependencies.artifacts.get(job.inputSnapshot); + if (!isInputSnapshot(inputArtifact)) { + throw new EvolutionExecutionError("Job input snapshot artifact has the wrong type", { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } + if (job.stage === "queued") { + const started = this.#withJobRevision(job, { stage: "snapshotting" }); + await this.#dependencies.jobs.save(started, job.revision, scopeLease()); + job = started; + } + const lastCheckpoint = job.checkpoints.at(-1); + if (lastCheckpoint?.stage === job.stage) { + const completionState = this.#checkpointDirective(lastCheckpoint); + if (completionState.directive !== "continue") { + return await this.#completeJob(job, lastCheckpoint, scopeLease()); + } + } + + while (job.stage !== "completed") { + leaseSignal.throwIfAborted(); + if (job.stage === "writing" || job.stage === "activating") { + const writerOwnerId = this.#dependencies.ids.nextId("writer-owner"); + const writerInitial = await this.#dependencies.leases.acquire( + "writer", + "global", + writerOwnerId, + this.#dependencies.clock.now(), + this.#leaseTtlMs, + ); + if (writerInitial === null) { + throw new EvolutionExecutionError("Evolution writer is already leased", { + reasonCode: "writer-failed", + retryable: true, + }); + } + job = await this.#withHeartbeat(writerInitial, leaseSignal, async (writerLease, writerSignal) => { + let current: EvolutionJob = job; + while (current.stage === "writing" || current.stage === "activating") { + current = await this.#executeStage( + current, + inputArtifact, + scopeLease(), + writerLease(), + writerSignal, + ); + } + return current; + }); + continue; + } + job = await this.#executeStage(job, inputArtifact, scopeLease(), null, leaseSignal); + } + return job; + } catch (error) { + try { + const latest = await this.#dependencies.jobs.load(job.jobId); + if (latest !== null) job = latest; + if (job.outcome === "completed" || error instanceof EvolutionCompletionPendingError) throw error; + await this.#markFailure(job, error, scopeLease(), signal?.aborted === true); + } catch (persistenceError) { + if (persistenceError === error) throw error; + throw new AggregateError( + [error, persistenceError], + `Failed evolution job "${job.jobId}" and its recovery write`, + ); + } + throw error; + } + }); + } + + async #resolveJob( + jobId: string, + kind: EvolutionJobResolution["kind"], + reasonCode: EvolutionFailureReasonCode, + ): Promise { + const initial = await this.#dependencies.jobs.load(jobId); + if (initial === null) throw new Error(`Unknown evolution job "${jobId}"`); + if (initial.outcome === "completed") throw new Error(`Completed evolution job "${jobId}" cannot be resolved`); + const ownerId = this.#dependencies.ids.nextId("lease-owner"); + const initialLease = await this.#acquireScopeLease(initial.metadata.scope, ownerId); + if (initialLease === null) throw new Error(`Evolution scope for job "${jobId}" is already leased`); + return this.#withHeartbeat(initialLease, undefined, async (lease) => { + const current = await this.#dependencies.jobs.load(jobId); + if (current === null) throw new Error(`Unknown evolution job "${jobId}"`); + const resolution: EvolutionJobResolution = { + kind, + reasonCode, + resolvedAt: this.#dependencies.clock.now(), + }; + const resolved = this.#withJobRevision(current, { + outcome: kind === "superseded" ? "superseded" : current.outcome, + resolution, + terminalAt: current.terminalAt ?? resolution.resolvedAt, + }); + await this.#dependencies.jobs.save(resolved, current.revision, lease()); + return resolved; + }); + } + + supersedeJob(jobId: string): Promise { + return this.#resolveJob(jobId, "superseded", "superseded-by-operator"); + } + + resolveJob(jobId: string): Promise { + return this.#resolveJob(jobId, "administratively-resolved", "superseded-by-operator"); + } +} diff --git a/packages/evolution/src/errors.ts b/packages/evolution/src/errors.ts new file mode 100644 index 000000000..4fc8108ea --- /dev/null +++ b/packages/evolution/src/errors.ts @@ -0,0 +1,20 @@ +import type { EvolutionFailureReasonCode } from "./contracts.ts"; + +export class EvolutionExecutionError extends Error { + readonly reasonCode: EvolutionFailureReasonCode; + readonly retryable: boolean; + + constructor( + message: string, + options: { + readonly reasonCode: EvolutionFailureReasonCode; + readonly retryable: boolean; + readonly cause?: unknown; + }, + ) { + super(message, options.cause === undefined ? undefined : { cause: options.cause }); + this.name = "EvolutionExecutionError"; + this.reasonCode = options.reasonCode; + this.retryable = options.retryable; + } +} diff --git a/packages/evolution/src/filesystem-store.ts b/packages/evolution/src/filesystem-store.ts new file mode 100644 index 000000000..35646c109 --- /dev/null +++ b/packages/evolution/src/filesystem-store.ts @@ -0,0 +1,670 @@ +import { randomUUID } from "node:crypto"; +import type { Dirent } from "node:fs"; +import { lstat, mkdir, mkdtemp, readdir, readFile, rename, rm, unlink, writeFile } from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; +import lockfile from "proper-lockfile"; +import type { + EvolutionArtifactReference, + EvolutionJob, + EvolutionLeaseToken, + EvolutionLedgerEvent, + EvolutionProcessedEvidenceCursor, + EvolutionScope, +} from "./contracts.ts"; +import { EVOLUTION_SCHEMA_VERSION } from "./contracts.ts"; +import type { + EvolutionArtifact, + EvolutionArtifactStore, + EvolutionCursorCommitResult, + EvolutionCursorStore, + EvolutionJobStore, + EvolutionLeaseStore, + EvolutionLedger, +} from "./ports.ts"; +import { computeEvolutionContentDigest, serializeEvolutionValue } from "./provenance.ts"; + +const ARTIFACT_FILE_NAME = "artifact.json"; +const STORE_LOCK_STALE_MS = 30_000; +const STORE_LOCK_UPDATE_MS = 10_000; +const SAFE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/; +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; +const JOB_STAGES = new Set([ + "queued", + "snapshotting", + "counter-slot", + "reflecting", + "curating", + "validating", + "writing", + "activating", + "completed", +]); +const ACTIVE_JOB_STAGES = new Set([...JOB_STAGES].filter((stage) => stage !== "completed")); +const JOB_OUTCOMES = new Set([ + "active", + "completed", + "interrupted", + "retryable-failed", + "terminal-failed", + "superseded", +]); + +interface StoredLease { + readonly schemaVersion: typeof EVOLUTION_SCHEMA_VERSION; + readonly kind: EvolutionLeaseToken["kind"]; + readonly key: string; + readonly ownerId: string; + readonly fence: number; + readonly expiresAt: string; + readonly releasedAt: string | null; +} + +export class EvolutionStorageCorruptionError extends Error { + readonly path: string; + + constructor(path: string, message: string, options?: ErrorOptions) { + super(`Corrupt evolution storage at "${path}": ${message}`, options); + this.name = "EvolutionStorageCorruptionError"; + this.path = path; + } +} + +function hasErrorCode(error: unknown, code: string): boolean { + return error instanceof Error && "code" in error && error.code === code; +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function assertSafeId(value: string, description: string): void { + if (!SAFE_ID_PATTERN.test(value)) throw new Error(`Invalid ${description} "${value}"`); +} + +function parseIsoTimestamp(value: string, description: string): number { + const timestamp = Date.parse(value); + if (!Number.isFinite(timestamp)) throw new Error(`${description} must be a valid ISO timestamp`); + return timestamp; +} + +function parseJson(contents: string, path: string): unknown { + try { + return JSON.parse(contents); + } catch (cause) { + throw new EvolutionStorageCorruptionError(path, "invalid JSON", { cause }); + } +} + +function assertDirectFile(path: string, stats: Awaited>): void { + if (!stats.isFile() || stats.isSymbolicLink()) { + throw new EvolutionStorageCorruptionError(path, "expected a direct regular file"); + } +} + +function parseArtifact(contents: string, path: string): EvolutionArtifact { + const parsed = parseJson(contents, path); + if (!isRecord(parsed) || !isRecord(parsed.metadata) || typeof parsed.metadata.artifactId !== "string") { + throw new EvolutionStorageCorruptionError(path, "artifact metadata is invalid"); + } + return parsed as unknown as EvolutionArtifact; +} + +function parseJob(contents: string, path: string): EvolutionJob { + const parsed = parseJson(contents, path); + if ( + !isRecord(parsed) || + typeof parsed.jobId !== "string" || + !SAFE_ID_PATTERN.test(parsed.jobId) || + !Number.isSafeInteger(parsed.revision) || + (parsed.revision as number) < 0 || + !Number.isSafeInteger(parsed.attempt) || + (parsed.attempt as number) < 0 || + !JOB_STAGES.has(String(parsed.stage)) || + !JOB_OUTCOMES.has(String(parsed.outcome)) || + (parsed.trigger !== "automatic" && parsed.trigger !== "manual") || + !Array.isArray(parsed.checkpoints) || + !isRecord(parsed.metadata) || + typeof parsed.metadata.artifactId !== "string" || + parsed.metadata.artifactId !== parsed.jobId || + typeof parsed.metadata.contentDigest !== "string" || + !DIGEST_PATTERN.test(parsed.metadata.contentDigest) || + !isRecord(parsed.metadata.scope) || + typeof parsed.metadata.scope.repositoryId !== "string" || + typeof parsed.metadata.scope.scopeId !== "string" || + !isRecord(parsed.inputSnapshot) || + typeof parsed.inputSnapshot.artifactId !== "string" || + !SAFE_ID_PATTERN.test(parsed.inputSnapshot.artifactId) || + typeof parsed.inputSnapshot.contentDigest !== "string" || + !DIGEST_PATTERN.test(parsed.inputSnapshot.contentDigest) || + !isRecord(parsed.evidenceRange) || + !Number.isSafeInteger(parsed.evidenceRange.afterCursor) || + !Number.isSafeInteger(parsed.evidenceRange.throughCursor) || + (parsed.evidenceRange.afterCursor as number) < 0 || + (parsed.evidenceRange.throughCursor as number) < (parsed.evidenceRange.afterCursor as number) || + typeof parsed.createdAt !== "string" || + typeof parsed.updatedAt !== "string" || + (typeof parsed.terminalAt !== "string" && parsed.terminalAt !== null) || + (typeof parsed.retryNotBefore !== "string" && parsed.retryNotBefore !== null) || + (typeof parsed.reasonCode !== "string" && parsed.reasonCode !== null) || + (typeof parsed.failureMessage !== "string" && parsed.failureMessage !== null) || + (parsed.completion !== null && !isRecord(parsed.completion)) || + (parsed.resolution !== null && !isRecord(parsed.resolution)) || + parsed.checkpoints.some( + (checkpoint) => + !isRecord(checkpoint) || + checkpoint.schemaVersion !== EVOLUTION_SCHEMA_VERSION || + !ACTIVE_JOB_STAGES.has(String(checkpoint.stage)) || + typeof checkpoint.idempotencyKey !== "string" || + !Array.isArray(checkpoint.outputArtifacts) || + checkpoint.outputArtifacts.some( + (reference) => + !isRecord(reference) || + typeof reference.artifactId !== "string" || + !SAFE_ID_PATTERN.test(reference.artifactId) || + typeof reference.contentDigest !== "string" || + !DIGEST_PATTERN.test(reference.contentDigest), + ) || + !isRecord(checkpoint.state) || + typeof checkpoint.completedAt !== "string", + ) + ) { + throw new EvolutionStorageCorruptionError(path, "job manifest is invalid"); + } + const { metadata, ...jobContent } = parsed; + if (metadata.contentDigest !== computeEvolutionContentDigest(jobContent)) { + throw new EvolutionStorageCorruptionError(path, "job manifest content digest is invalid"); + } + return parsed as unknown as EvolutionJob; +} + +function parseLedgerEvent(contents: string, path: string): EvolutionLedgerEvent { + const parsed = parseJson(contents, path); + if (!isRecord(parsed) || typeof parsed.eventId !== "string" || typeof parsed.jobId !== "string") { + throw new EvolutionStorageCorruptionError(path, "ledger event is invalid"); + } + return parsed as unknown as EvolutionLedgerEvent; +} + +function parseCursor(contents: string, path: string): EvolutionProcessedEvidenceCursor { + const parsed = parseJson(contents, path); + if ( + !isRecord(parsed) || + parsed.schemaVersion !== EVOLUTION_SCHEMA_VERSION || + !isRecord(parsed.scope) || + !Number.isSafeInteger(parsed.sequence) || + (parsed.sequence as number) < 0 || + (typeof parsed.committedByJobId !== "string" && parsed.committedByJobId !== null) || + typeof parsed.updatedAt !== "string" + ) { + throw new EvolutionStorageCorruptionError(path, "processed-evidence cursor is invalid"); + } + return parsed as unknown as EvolutionProcessedEvidenceCursor; +} + +function parseLease(contents: string, path: string): StoredLease { + const parsed = parseJson(contents, path); + if ( + !isRecord(parsed) || + parsed.schemaVersion !== EVOLUTION_SCHEMA_VERSION || + (parsed.kind !== "scope" && parsed.kind !== "writer") || + typeof parsed.key !== "string" || + typeof parsed.ownerId !== "string" || + !Number.isSafeInteger(parsed.fence) || + (parsed.fence as number) < 1 || + typeof parsed.expiresAt !== "string" || + (typeof parsed.releasedAt !== "string" && parsed.releasedAt !== null) + ) { + throw new EvolutionStorageCorruptionError(path, "lease record is invalid"); + } + return parsed as unknown as StoredLease; +} + +function scopeMatches(left: EvolutionScope, right: EvolutionScope): boolean { + return left.repositoryId === right.repositoryId && left.scopeId === right.scopeId; +} + +function artifactDirectoryName(digest: string): string { + if (!DIGEST_PATTERN.test(digest)) throw new Error(`Invalid artifact digest "${digest}"`); + return digest.replace("sha256:", "sha256-"); +} + +export interface FileEvolutionPersistenceOptions { + readonly storeLockStaleMs?: number; + readonly storeLockUpdateMs?: number; +} + +export class FileEvolutionPersistence + implements EvolutionArtifactStore, EvolutionCursorStore, EvolutionJobStore, EvolutionLeaseStore, EvolutionLedger +{ + readonly #rootPath: string; + readonly #artifactPath: string; + readonly #cursorPath: string; + readonly #jobPath: string; + readonly #leasePath: string; + readonly #ledgerPath: string; + readonly #storeLockPath: string; + readonly #storeLockStaleMs: number; + readonly #storeLockUpdateMs: number; + + constructor(rootPath: string, options: FileEvolutionPersistenceOptions = {}) { + this.#rootPath = resolve(rootPath); + this.#artifactPath = join(this.#rootPath, "artifacts"); + this.#cursorPath = join(this.#rootPath, "cursors"); + this.#jobPath = join(this.#rootPath, "jobs"); + this.#leasePath = join(this.#rootPath, "leases"); + this.#ledgerPath = join(this.#rootPath, "ledger"); + this.#storeLockPath = join(this.#rootPath, ".store"); + this.#storeLockStaleMs = options.storeLockStaleMs ?? STORE_LOCK_STALE_MS; + this.#storeLockUpdateMs = options.storeLockUpdateMs ?? STORE_LOCK_UPDATE_MS; + if (this.#storeLockStaleMs < 2_000) throw new Error("storeLockStaleMs must be at least 2000"); + if (this.#storeLockUpdateMs < 1_000 || this.#storeLockUpdateMs > this.#storeLockStaleMs / 2) { + throw new Error("storeLockUpdateMs must be between 1000 and half of storeLockStaleMs"); + } + } + + async #ensureDirectories(): Promise { + for (const path of [ + this.#rootPath, + this.#artifactPath, + this.#cursorPath, + this.#jobPath, + this.#leasePath, + this.#ledgerPath, + ]) { + await mkdir(path, { recursive: true, mode: 0o700 }); + const stats = await lstat(path); + if (!stats.isDirectory() || stats.isSymbolicLink()) { + throw new EvolutionStorageCorruptionError(path, "expected a direct directory"); + } + } + } + + async #withStoreLock(operation: () => Promise): Promise { + await this.#ensureDirectories(); + let compromised: Error | null = null; + const release = await lockfile.lock(this.#storeLockPath, { + lockfilePath: `${this.#storeLockPath}.lock`, + realpath: false, + retries: { retries: 200, factor: 1, minTimeout: 5, maxTimeout: 25, randomize: true }, + stale: this.#storeLockStaleMs, + update: this.#storeLockUpdateMs, + onCompromised(error) { + compromised = error; + }, + }); + let result!: T; + let operationFailed = false; + let operationError: unknown; + try { + result = await operation(); + } catch (error) { + operationFailed = true; + operationError = error; + } + let releaseError: unknown; + try { + await release(); + } catch (error) { + if (!hasErrorCode(error, "ERELEASED")) releaseError = error; + } + if (operationFailed) throw operationError; + if (compromised !== null) throw compromised; + if (releaseError !== undefined) throw releaseError; + return result; + } + + async #readOptional(path: string): Promise { + try { + const stats = await lstat(path); + assertDirectFile(path, stats); + return await readFile(path, "utf8"); + } catch (error) { + if (hasErrorCode(error, "ENOENT")) return null; + throw error; + } + } + + async #atomicReplace(path: string, value: unknown): Promise { + const temporaryPath = join(dirname(path), `.${basename(path)}.${randomUUID()}.tmp`); + try { + await writeFile(temporaryPath, `${serializeEvolutionValue(value)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + await rename(temporaryPath, path); + } catch (error) { + try { + await unlink(temporaryPath); + } catch (cleanupError) { + if (!hasErrorCode(cleanupError, "ENOENT")) throw cleanupError; + } + throw error; + } + } + + #scopeFileName(scope: EvolutionScope): string { + return artifactDirectoryName(computeEvolutionContentDigest(scope)); + } + + #leaseFile(kind: EvolutionLeaseToken["kind"], key: string): string { + const digest = artifactDirectoryName(computeEvolutionContentDigest({ kind, key })); + return join(this.#leasePath, `${kind}-${digest}.json`); + } + + async #loadLease(kind: EvolutionLeaseToken["kind"], key: string): Promise { + const path = this.#leaseFile(kind, key); + const contents = await this.#readOptional(path); + return contents === null ? null : parseLease(contents, path); + } + + async #assertLeaseHeld(lease: EvolutionLeaseToken): Promise { + const stored = await this.#loadLease(lease.kind, lease.key); + if ( + stored === null || + stored.ownerId !== lease.ownerId || + stored.fence !== lease.fence || + stored.releasedAt !== null || + parseIsoTimestamp(stored.expiresAt, "lease expiry") <= Date.now() + ) { + throw new Error(`Evolution ${lease.kind} lease was lost`); + } + } + + async put(artifact: EvolutionArtifact): Promise { + await this.#ensureDirectories(); + const content = serializeEvolutionValue(artifact); + const contentDigest = computeEvolutionContentDigest(artifact); + const destination = join(this.#artifactPath, artifactDirectoryName(contentDigest)); + const artifactPath = join(destination, ARTIFACT_FILE_NAME); + try { + const existing = await this.#readOptional(artifactPath); + if (existing !== null) { + if (existing.trimEnd() !== content) { + throw new EvolutionStorageCorruptionError(artifactPath, "immutable artifact content changed"); + } + return { artifactId: artifact.metadata.artifactId, contentDigest }; + } + } catch (error) { + if (!(error instanceof EvolutionStorageCorruptionError) || !hasErrorCode(error.cause, "ENOENT")) throw error; + } + + let temporaryDirectory: string | null = await mkdtemp(join(this.#artifactPath, ".artifact.tmp-")); + try { + await writeFile(join(temporaryDirectory, ARTIFACT_FILE_NAME), `${content}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + try { + await rename(temporaryDirectory, destination); + temporaryDirectory = null; + } catch (error) { + if (!hasErrorCode(error, "EEXIST") && !hasErrorCode(error, "ENOTEMPTY")) throw error; + const existing = await this.#readOptional(artifactPath); + if (existing === null || existing.trimEnd() !== content) { + throw new EvolutionStorageCorruptionError( + artifactPath, + "immutable artifact race produced different content", + ); + } + } + } finally { + if (temporaryDirectory !== null) await rm(temporaryDirectory, { recursive: true, force: true }); + } + return { artifactId: artifact.metadata.artifactId, contentDigest }; + } + + async get(reference: EvolutionArtifactReference): Promise { + assertSafeId(reference.artifactId, "artifact ID"); + const path = join(this.#artifactPath, artifactDirectoryName(reference.contentDigest), ARTIFACT_FILE_NAME); + const contents = await this.#readOptional(path); + if (contents === null) throw new EvolutionStorageCorruptionError(path, "referenced artifact is missing"); + const artifact = parseArtifact(contents, path); + const actualDigest = computeEvolutionContentDigest(artifact); + if (actualDigest !== reference.contentDigest) { + throw new EvolutionStorageCorruptionError(path, "content digest does not match its reference"); + } + if (artifact.metadata.artifactId !== reference.artifactId) { + throw new EvolutionStorageCorruptionError(path, "artifact ID does not match its reference"); + } + return artifact; + } + + async create(job: EvolutionJob, lease: EvolutionLeaseToken): Promise { + assertSafeId(job.jobId, "job ID"); + await this.#withStoreLock(async () => { + await this.#assertLeaseHeld(lease); + const path = join(this.#jobPath, `${job.jobId}.json`); + parseJob(serializeEvolutionValue(job), path); + try { + await writeFile(path, `${serializeEvolutionValue(job)}\n`, { encoding: "utf8", flag: "wx", mode: 0o600 }); + } catch (error) { + if (!hasErrorCode(error, "EEXIST")) throw error; + const existing = await this.#readOptional(path); + if (existing === null || existing.trimEnd() !== serializeEvolutionValue(job)) { + throw new Error(`Evolution job "${job.jobId}" already exists with different content`); + } + } + }); + } + + async save(job: EvolutionJob, expectedRevision: number, lease: EvolutionLeaseToken): Promise { + assertSafeId(job.jobId, "job ID"); + await this.#withStoreLock(async () => { + await this.#assertLeaseHeld(lease); + const path = join(this.#jobPath, `${job.jobId}.json`); + parseJob(serializeEvolutionValue(job), path); + const contents = await this.#readOptional(path); + if (contents === null) throw new Error(`Unknown evolution job "${job.jobId}"`); + const current = parseJob(contents, path); + if (current.revision !== expectedRevision) { + throw new Error( + `Evolution job "${job.jobId}" revision conflict: expected ${expectedRevision}, found ${current.revision}`, + ); + } + if (job.revision !== expectedRevision + 1) { + throw new Error(`Evolution job "${job.jobId}" must advance exactly one revision`); + } + await this.#atomicReplace(path, job); + }); + } + + async list(scope?: EvolutionScope): Promise { + await this.#ensureDirectories(); + const jobs: EvolutionJob[] = []; + for (const entry of await readdir(this.#jobPath, { withFileTypes: true })) { + if (!entry.isFile() || entry.isSymbolicLink() || !entry.name.endsWith(".json")) continue; + const path = join(this.#jobPath, entry.name); + const job = parseJob(await readFile(path, "utf8"), path); + if (scope === undefined || scopeMatches(job.metadata.scope, scope)) jobs.push(job); + } + return jobs.sort( + (left, right) => left.createdAt.localeCompare(right.createdAt) || left.jobId.localeCompare(right.jobId), + ); + } + + async load(scope: EvolutionScope): Promise; + async load(jobId: string): Promise; + async load(value: EvolutionScope | string): Promise { + if (typeof value === "string") { + assertSafeId(value, "job ID"); + const path = join(this.#jobPath, `${value}.json`); + const contents = await this.#readOptional(path); + return contents === null ? null : parseJob(contents, path); + } + const path = join(this.#cursorPath, `${this.#scopeFileName(value)}.json`); + const contents = await this.#readOptional(path); + if (contents === null) { + return { + schemaVersion: EVOLUTION_SCHEMA_VERSION, + scope: value, + sequence: 0, + committedByJobId: null, + updatedAt: new Date(0).toISOString(), + }; + } + const cursor = parseCursor(contents, path); + if (!scopeMatches(cursor.scope, value)) { + throw new EvolutionStorageCorruptionError(path, "cursor scope does not match its path"); + } + return cursor; + } + + async commit( + scope: EvolutionScope, + expectedSequence: number, + nextSequence: number, + jobId: string, + updatedAt: string, + lease: EvolutionLeaseToken, + ): Promise { + if (!Number.isSafeInteger(expectedSequence) || expectedSequence < 0) throw new Error("Invalid expected cursor"); + if (!Number.isSafeInteger(nextSequence) || nextSequence < expectedSequence) + throw new Error("Invalid next cursor"); + assertSafeId(jobId, "job ID"); + return this.#withStoreLock(async () => { + await this.#assertLeaseHeld(lease); + const current = await this.load(scope); + if (current.sequence === nextSequence && current.committedByJobId === jobId) return "already-committed"; + if (current.sequence !== expectedSequence) return "conflict"; + const next: EvolutionProcessedEvidenceCursor = { + schemaVersion: EVOLUTION_SCHEMA_VERSION, + scope, + sequence: nextSequence, + committedByJobId: jobId, + updatedAt, + }; + await this.#atomicReplace(join(this.#cursorPath, `${this.#scopeFileName(scope)}.json`), next); + return "committed"; + }); + } + + async acquire( + kind: EvolutionLeaseToken["kind"], + key: string, + ownerId: string, + now: string, + ttlMs: number, + ): Promise { + if (key === "" || ownerId === "") throw new Error("Evolution lease key and owner must be non-empty"); + if (!Number.isFinite(ttlMs) || ttlMs < 2_000) throw new Error("Evolution lease ttlMs must be at least 2000"); + const nowMs = parseIsoTimestamp(now, "lease acquisition time"); + return this.#withStoreLock(async () => { + const current = await this.#loadLease(kind, key); + if ( + current !== null && + current.releasedAt === null && + parseIsoTimestamp(current.expiresAt, "lease expiry") > nowMs + ) { + return null; + } + const stored: StoredLease = { + schemaVersion: EVOLUTION_SCHEMA_VERSION, + kind, + key, + ownerId, + fence: (current?.fence ?? 0) + 1, + expiresAt: new Date(nowMs + ttlMs).toISOString(), + releasedAt: null, + }; + await this.#atomicReplace(this.#leaseFile(kind, key), stored); + return stored; + }); + } + + async renew(lease: EvolutionLeaseToken, now: string, ttlMs: number): Promise { + if (!Number.isFinite(ttlMs) || ttlMs < 2_000) throw new Error("Evolution lease ttlMs must be at least 2000"); + const nowMs = parseIsoTimestamp(now, "lease renewal time"); + return this.#withStoreLock(async () => { + const current = await this.#loadLease(lease.kind, lease.key); + if ( + current === null || + current.ownerId !== lease.ownerId || + current.fence !== lease.fence || + current.releasedAt !== null || + parseIsoTimestamp(current.expiresAt, "lease expiry") <= nowMs + ) { + return null; + } + const renewed: StoredLease = { ...current, expiresAt: new Date(nowMs + ttlMs).toISOString() }; + await this.#atomicReplace(this.#leaseFile(lease.kind, lease.key), renewed); + return renewed; + }); + } + + async release(lease: EvolutionLeaseToken): Promise { + await this.#withStoreLock(async () => { + const current = await this.#loadLease(lease.kind, lease.key); + if (current === null || current.ownerId !== lease.ownerId || current.fence !== lease.fence) return; + await this.#atomicReplace(this.#leaseFile(lease.kind, lease.key), { + ...current, + expiresAt: new Date(0).toISOString(), + releasedAt: new Date().toISOString(), + } satisfies StoredLease); + }); + } + + async append(event: EvolutionLedgerEvent): Promise { + assertSafeId(event.jobId, "job ID"); + assertSafeId(event.eventId, "ledger event ID"); + await this.#ensureDirectories(); + const jobLedgerPath = join(this.#ledgerPath, event.jobId); + await mkdir(jobLedgerPath, { recursive: true, mode: 0o700 }); + const destination = join(jobLedgerPath, event.eventId); + const eventPath = join(destination, "event.json"); + const content = serializeEvolutionValue(event); + const existing = await this.#readOptional(eventPath); + if (existing !== null) { + if (existing.trimEnd() !== content) { + throw new EvolutionStorageCorruptionError(eventPath, "ledger event ID was reused with different content"); + } + return; + } + let temporaryDirectory: string | null = await mkdtemp(join(jobLedgerPath, ".event.tmp-")); + try { + await writeFile(join(temporaryDirectory, "event.json"), `${content}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + try { + await rename(temporaryDirectory, destination); + temporaryDirectory = null; + } catch (error) { + if (!hasErrorCode(error, "EEXIST") && !hasErrorCode(error, "ENOTEMPTY")) throw error; + const raced = await this.#readOptional(eventPath); + if (raced === null || raced.trimEnd() !== content) { + throw new EvolutionStorageCorruptionError(eventPath, "ledger append race produced different content"); + } + } + } finally { + if (temporaryDirectory !== null) await rm(temporaryDirectory, { recursive: true, force: true }); + } + } + + async read(jobId: string): Promise { + assertSafeId(jobId, "job ID"); + const jobLedgerPath = join(this.#ledgerPath, jobId); + let entries: Dirent[]; + try { + entries = await readdir(jobLedgerPath, { withFileTypes: true }); + } catch (error) { + if (hasErrorCode(error, "ENOENT")) return []; + throw error; + } + const events: EvolutionLedgerEvent[] = []; + for (const entry of entries) { + if (!entry.isDirectory() || entry.isSymbolicLink() || entry.name.startsWith(".")) continue; + const path = join(jobLedgerPath, entry.name, "event.json"); + const contents = await this.#readOptional(path); + if (contents === null) throw new EvolutionStorageCorruptionError(path, "ledger event is missing"); + events.push(parseLedgerEvent(contents, path)); + } + return events.sort( + (left, right) => + left.metadata.createdAt.localeCompare(right.metadata.createdAt) || + left.eventId.localeCompare(right.eventId), + ); + } +} diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index b41307272..7f7c616a2 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -1,32 +1,41 @@ export { type ActiveEvolutionStage, type CheckpointSavedLedgerEvent, - type CompletedCheckpoint, - type CorpusCheckpoint, - type CreatedCheckpoint, - type CuratedCheckpoint, - type DeferredEvolutionProposal, EVOLUTION_SCHEMA_VERSION, type EvidenceCorpusDocument, type EvidenceCorpusManifest, type EvolutionArtifactMetadata, type EvolutionArtifactProducer, - type EvolutionCheckpoint, + type EvolutionArtifactReference, + type EvolutionCompletionKind, type EvolutionEvidenceOrigin, type EvolutionEvidenceSource, + type EvolutionFailureReasonCode, type EvolutionInputSnapshot, type EvolutionJob, + type EvolutionJobCompletion, + type EvolutionJobOutcome, + type EvolutionJobResolution, + type EvolutionJobStage, + type EvolutionLeaseToken, type EvolutionLedgerEvent, type EvolutionMemoryProposal, type EvolutionMemoryTarget, type EvolutionModelIdentity, type EvolutionMutation, + type EvolutionProcessedEvidenceCursor, + type EvolutionPromptVersions, type EvolutionProposal, type EvolutionProposalLedgerDisposition, + type EvolutionRepositorySnapshot, type EvolutionScope, + type EvolutionStageCheckpoint, type EvolutionTarget, + type EvolutionTokenMeasurement, + type EvolutionTriggerKind, type ExcludedEvidenceSource, - type FailedCheckpoint, + type HarnessActivationReceipt, + type HarnessActivationResult, type ImmutableHarnessSnapshot, type InstructionDocumentSnapshot, type InstructionSnapshot, @@ -39,41 +48,64 @@ export { type Lesson, type LessonKind, type MutationBatch, + type MutationWriteReceipt, type MutationWriteResult, type MutationWrittenLedgerEvent, type OrchestrationSourceTarget, type ProposalDispositionLedgerEvent, type ProposalRolledBackLedgerEvent, type ProposalValidationResult, - type ReflectedCheckpoint, type ReflectionNode, type ReflectionNodeStatus, - type ScoredCheckpoint, type ScoredProposal, type ScoredProposalDisposition, type SkillTarget, - type ValidatedCheckpoint, type ValidationIssue, type ValidationReport, type ValidationSeverity, - type WrittenCheckpoint, } from "./contracts.ts"; -export { EvolutionPipeline, type EvolutionPipelineDependencies, type EvolutionRunResult } from "./pipeline.ts"; +export { + EvolutionControlPlane, + type EvolutionControlPlaneDependencies, + type EvolutionControlPlaneOptions, + type EvolutionJobCreationBlockReason, + type EvolutionJobCreationResult, +} from "./control-plane.ts"; +export { EvolutionExecutionError } from "./errors.ts"; +export { + EvolutionStorageCorruptionError, + FileEvolutionPersistence, + type FileEvolutionPersistenceOptions, +} from "./filesystem-store.ts"; +export { EvolutionPipeline, type EvolutionPipelineDependencies } from "./pipeline.ts"; export type { Curator, CuratorInput, EvolutionArtifact, EvolutionArtifactStore, EvolutionClock, + EvolutionCursorCommitResult, + EvolutionCursorStore, EvolutionEventSink, + EvolutionEvidenceCatalog, EvolutionIdGenerator, + EvolutionJobCreationRequest, + EvolutionJobRunner, EvolutionJobStore, + EvolutionLeaseStore, EvolutionLedger, EvolutionModelExecutor, EvolutionModelRequest, EvolutionModelResponse, EvolutionProgressEvent, - InstructionLoader, + EvolutionSnapshotContext, + EvolutionSnapshotSource, + EvolutionStageDirective, + EvolutionStageExecutionContext, + EvolutionStageExecutionResult, + EvolutionStageExecutor, + EvolutionWorkSource, + InteractivePriorityGate, ProposalScorer, ProposalScoringInput, ProposalValidationInput, @@ -81,15 +113,41 @@ export type { ReflectionResult, Reflector, ReflectorInput, + SuccessfulMaterialization, } from "./ports.ts"; export { type CreateEvolutionArtifactMetadataOptions, computeEvolutionContentDigest, createEvolutionArtifactMetadata, + serializeEvolutionValue, } from "./provenance.ts"; +export { + DEFAULT_EVOLUTION_RETENTION_POLICY, + type EvolutionRetentionPlan, + type EvolutionRetentionPolicy, + planEvolutionRetention, +} from "./recovery.ts"; export { EvolutionTargetAdapterRegistry, type EvolutionTargetAdapterRegistryOptions, type EvolutionTargetKind, type EvolutionTargetWriter, } from "./targets.ts"; +export { + AUTOMATIC_EVOLUTION_COOLDOWN_MS, + AUTOMATIC_EVOLUTION_MINIMUM_INVOCATIONS, + AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD, + type AutomaticEvolutionEligibility, + type AutomaticEvolutionEligibilityInput, + type AutomaticEvolutionEligibilityReason, + EVOLUTION_TOKEN_ESTIMATOR_VERSION, + estimateEvolutionTokens, + evaluateAutomaticEvolutionEligibility, +} from "./token-estimator.ts"; +export { + EvolutionBackgroundWorker, + type EvolutionBackgroundWorkerDependencies, + type EvolutionBackgroundWorkerOptions, + PollingEvolutionWorkSource, + type PollingEvolutionWorkSourceOptions, +} from "./worker.ts"; diff --git a/packages/evolution/src/pipeline.ts b/packages/evolution/src/pipeline.ts index f29101bcb..6d2880737 100644 --- a/packages/evolution/src/pipeline.ts +++ b/packages/evolution/src/pipeline.ts @@ -1,42 +1,31 @@ import type { - ActiveEvolutionStage, - CheckpointSavedLedgerEvent, EvidenceCorpusManifest, EvolutionArtifactMetadata, EvolutionArtifactProducer, - EvolutionCheckpoint, EvolutionInputSnapshot, - EvolutionJob, EvolutionLedgerEvent, EvolutionMemoryProposal, EvolutionProposal, EvolutionProposalLedgerDisposition, EvolutionScope, - FailedCheckpoint, + HarnessActivationReceipt, InstructionSnapshot, - JobCompletedLedgerEvent, - JobCreatedLedgerEvent, - JobFailedLedgerEvent, + Lesson, MutationBatch, - MutationWriteResult, - MutationWrittenLedgerEvent, + MutationWriteReceipt, ProposalDispositionLedgerEvent, ReflectionNode, ScoredProposal, ValidationReport, } from "./contracts.ts"; -import { EVOLUTION_SCHEMA_VERSION } from "./contracts.ts"; +import { EvolutionExecutionError } from "./errors.ts"; import type { Curator, EvolutionArtifact, - EvolutionArtifactStore, - EvolutionClock, - EvolutionEventSink, - EvolutionIdGenerator, - EvolutionJobStore, - EvolutionLedger, EvolutionModelExecutor, - InstructionLoader, + EvolutionStageExecutionContext, + EvolutionStageExecutionResult, + EvolutionStageExecutor, ProposalScorer, ProposalValidator, ReflectionResult, @@ -49,52 +38,76 @@ const PIPELINE_PRODUCER_NAME = "diffex-evolution"; export interface EvolutionPipelineDependencies { readonly version: string; - readonly clock: EvolutionClock; - readonly ids: EvolutionIdGenerator; readonly model: EvolutionModelExecutor; - readonly instructions: InstructionLoader; readonly reflector: Reflector; readonly curator: Curator; readonly validator: ProposalValidator; readonly scorer: ProposalScorer; - readonly artifacts: EvolutionArtifactStore; - readonly jobs: EvolutionJobStore; - readonly ledger: EvolutionLedger; - readonly events: EvolutionEventSink; readonly targets: EvolutionTargetAdapterRegistry; } -export interface EvolutionRunResult { - readonly job: EvolutionJob; - readonly corpus: EvidenceCorpusManifest; - readonly instructions: InstructionSnapshot; - readonly reflection: ReflectionResult; - readonly proposals: readonly EvolutionProposal[]; - readonly validation: ValidationReport; - readonly scoredProposals: readonly ScoredProposal[]; - readonly mutationBatch: MutationBatch | null; - readonly writeResult: MutationWriteResult | null; +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); } -function proposalDisposition(scored: ScoredProposal): EvolutionProposalLedgerDisposition | null { - switch (scored.disposition) { - case "deferred": - case "rejected": - case "superseded": - return scored.disposition; - case "selected": - return null; - } +function isCorpus(artifact: EvolutionArtifact): artifact is EvidenceCorpusManifest { + return "excludedSources" in artifact; +} + +function isInstructionSnapshot(artifact: EvolutionArtifact): artifact is InstructionSnapshot { + return "documents" in artifact && !("excludedSources" in artifact); +} + +function isReflectionNode(artifact: EvolutionArtifact): artifact is ReflectionNode { + return "nodeId" in artifact; +} + +function isLesson(artifact: EvolutionArtifact): artifact is Lesson { + return "lessonId" in artifact; +} + +function isProposal(artifact: EvolutionArtifact): artifact is EvolutionProposal { + return "proposalId" in artifact && "disposition" in artifact; +} + +function isScoredProposal(artifact: EvolutionArtifact): artifact is ScoredProposal { + return "proposal" in artifact && "score" in artifact; +} + +function isMutationBatch(artifact: EvolutionArtifact): artifact is MutationBatch { + return "mutations" in artifact && "batchId" in artifact; +} + +function isMutationWriteReceipt(artifact: EvolutionArtifact): artifact is MutationWriteReceipt { + return ( + "result" in artifact && + isRecord(artifact.result) && + (artifact.result.status === "written" || artifact.result.status === "failed") + ); } function assertUniqueIds(values: readonly string[], description: string): void { const seen = new Set(); for (const value of values) { - if (seen.has(value)) throw new Error(`Duplicate ${description} "${value}"`); + if (seen.has(value)) + throw new EvolutionExecutionError(`Duplicate ${description} "${value}"`, { + reasonCode: "pipeline-invariant", + retryable: false, + }); seen.add(value); } } +function requireOne(values: readonly T[], description: string): T { + if (values.length !== 1) { + throw new EvolutionExecutionError(`Expected exactly one ${description}, found ${values.length}`, { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } + return values[0] as T; +} + function assertScoredProposals( proposals: readonly EvolutionProposal[], scoredProposals: readonly ScoredProposal[], @@ -106,22 +119,80 @@ function assertScoredProposals( const proposalsById = new Map(proposals.map((proposal) => [proposal.proposalId, proposal])); for (const scored of scoredProposals) { const proposal = proposalsById.get(scored.proposal.proposalId); - if (proposal === undefined) { - throw new Error(`Scorer returned unknown proposal "${scored.proposal.proposalId}"`); - } - if (proposal !== scored.proposal) { - throw new Error(`Scorer replaced proposal "${scored.proposal.proposalId}"`); + if (proposal === undefined || proposal !== scored.proposal) { + throw new EvolutionExecutionError( + `Scorer replaced or returned unknown proposal "${scored.proposal.proposalId}"`, + { + reasonCode: "pipeline-invariant", + retryable: false, + }, + ); } if (scored.disposition === "selected" && proposal.disposition !== "actionable") { - throw new Error(`Deferred proposal "${proposal.proposalId}" cannot be selected`); + throw new EvolutionExecutionError(`Deferred proposal "${proposal.proposalId}" cannot be selected`, { + reasonCode: "pipeline-invariant", + retryable: false, + }); } } if (scoredProposals.length !== proposals.length) { - throw new Error("Scorer must return exactly one result for every proposal"); + throw new EvolutionExecutionError("Scorer must return exactly one result for every proposal", { + reasonCode: "pipeline-invariant", + retryable: false, + }); } } -export class EvolutionPipeline { +function assertValidationReport( + proposals: readonly EvolutionProposal[], + validation: ValidationReport, + scoredProposals: readonly ScoredProposal[], +): void { + assertUniqueIds( + validation.proposalResults.map((result) => result.proposalId), + "validation proposal ID", + ); + const proposalIds = new Set(proposals.map((proposal) => proposal.proposalId)); + if ( + validation.proposalResults.length !== proposals.length || + validation.proposalResults.some((result) => !proposalIds.has(result.proposalId)) + ) { + throw new EvolutionExecutionError("Validator must return exactly one result for every proposal", { + reasonCode: "pipeline-invariant", + retryable: false, + }); + } + const accepted = new Set( + validation.proposalResults + .filter((result) => result.disposition === "accepted") + .map((result) => result.proposalId), + ); + if ( + scoredProposals.some((scored) => scored.disposition === "selected" && !accepted.has(scored.proposal.proposalId)) + ) { + throw new EvolutionExecutionError("Scorer selected a proposal that deterministic validation did not accept", { + reasonCode: "pipeline-invariant", + retryable: false, + }); + } +} + +function stableId(kind: string, value: unknown): string { + return `${kind}-${computeEvolutionContentDigest(value).slice("sha256:".length, "sha256:".length + 32)}`; +} + +function stageFailure( + error: unknown, + message: string, + reasonCode: "model-call-failed" | "validation-rejected" | "writer-failed", + retryable: boolean, +): never { + if (error instanceof EvolutionExecutionError) throw error; + if (error instanceof Error && error.name === "AbortError") throw error; + throw new EvolutionExecutionError(message, { reasonCode, retryable, cause: error }); +} + +export class EvolutionPipeline implements EvolutionStageExecutor { readonly #dependencies: EvolutionPipelineDependencies; readonly #producer: EvolutionArtifactProducer; @@ -152,124 +223,161 @@ export class EvolutionPipeline { }); } - #ledgerMetadata(eventId: string, job: EvolutionJob, createdAt: string, content: unknown): EvolutionArtifactMetadata { - return this.#metadata(eventId, [job.jobId], job.metadata.scope, createdAt, content); - } - - async #appendLedger(event: EvolutionLedgerEvent): Promise { - await this.#dependencies.ledger.append(event); + #ledgerMetadata( + eventId: string, + context: EvolutionStageExecutionContext, + content: unknown, + ): EvolutionArtifactMetadata { + return this.#metadata( + eventId, + [context.job.jobId], + context.job.metadata.scope, + context.input.metadata.createdAt, + content, + ); } - async #saveCheckpoint(job: EvolutionJob, checkpoint: EvolutionCheckpoint): Promise { - const updatedAt = this.#dependencies.clock.now(); - const status = - checkpoint.stage === "completed" ? "completed" : checkpoint.stage === "failed" ? "failed" : "running"; - const content = { - jobId: job.jobId, - inputSnapshotId: job.inputSnapshotId, - status, - checkpoint, - updatedAt, - }; - const updated: EvolutionJob = { - ...job, - metadata: this.#metadata( - job.jobId, - [job.inputSnapshotId], - job.metadata.scope, - job.metadata.createdAt, - content, - ), - status, - checkpoint, - updatedAt, - }; - await this.#dependencies.jobs.save(updated); - const eventId = this.#dependencies.ids.nextId("ledger-event"); - const event: CheckpointSavedLedgerEvent = { - metadata: this.#ledgerMetadata(eventId, updated, updatedAt, checkpoint), + #proposalDispositionEvent( + context: EvolutionStageExecutionContext, + scored: ScoredProposal, + disposition: EvolutionProposalLedgerDisposition, + reasonCodes: readonly string[] = scored.reasonCodes, + ): ProposalDispositionLedgerEvent { + const content = { proposalId: scored.proposal.proposalId, disposition, reasonCodes }; + const eventId = stableId("ledger-event", { idempotencyKey: context.idempotencyKey, ...content }); + return { + metadata: this.#ledgerMetadata(eventId, context, content), eventId, - jobId: updated.jobId, - type: "checkpoint-saved", - checkpoint, + jobId: context.job.jobId, + type: "proposal-disposition", + proposalId: scored.proposal.proposalId, + disposition, + reasonCodes, }; - await this.#appendLedger(event); - await this.#dependencies.events.emit({ - jobId: updated.jobId, - emittedAt: updatedAt, - type: checkpoint.stage === "failed" ? "failed" : "checkpoint", - checkpoint, - }); - return updated; - } - - async #putArtifacts(artifacts: readonly EvolutionArtifact[]): Promise { - for (const artifact of artifacts) await this.#dependencies.artifacts.put(artifact); } - #createCorpus(input: EvolutionInputSnapshot): EvidenceCorpusManifest { + #createCorpus(input: EvolutionInputSnapshot, idempotencyKey: string): EvidenceCorpusManifest { assertUniqueIds( input.evidenceSources.map((source) => source.sourceId), "evidence source ID", ); - const documents = input.evidenceSources - .filter((source) => source.origin === "product-runtime") - .map((source) => { - const actualDigest = computeEvolutionContentDigest(source.invocation); - if (source.contentDigest !== actualDigest) { - throw new Error(`Evidence source "${source.sourceId}" content digest does not match its invocation`); - } - return { - documentId: this.#dependencies.ids.nextId("corpus-document"), - sourceId: source.sourceId, + const documents = input.evidenceSources.map((source) => { + if (source.origin !== "product-runtime") { + throw new EvolutionExecutionError("Frozen evolution snapshots may contain only product runtime evidence", { + reasonCode: "snapshot-invalid", + retryable: false, + }); + } + const actualDigest = computeEvolutionContentDigest(source.invocation); + if (source.contentDigest !== actualDigest) { + throw new EvolutionExecutionError( + `Evidence source "${source.sourceId}" content digest does not match its invocation`, + { reasonCode: "snapshot-invalid", retryable: false }, + ); + } + if (source.ingestionCursor > input.ingestionHighWaterCursor) { + throw new EvolutionExecutionError( + `Evidence source "${source.sourceId}" is after the frozen high-water cursor`, + { + reasonCode: "snapshot-invalid", + retryable: false, + }, + ); + } + return { + documentId: stableId("corpus-document", { + idempotencyKey, + materializationId: source.materializationId, contentDigest: source.contentDigest, - materializedInvocationVersion: source.invocation.version, - traceId: source.invocation.traceId, - integrity: source.invocation.integrity, - terminationStatus: source.invocation.terminationStatus, - evidenceIds: source.invocation.evidence.map((evidence) => evidence.evidenceId), - }; - }); - const excludedSources = input.evidenceSources - .filter((source) => source.origin === "evolution-pipeline") - .map((source) => ({ sourceId: source.sourceId, reason: "evolution-pipeline-origin" as const })); - const artifactId = this.#dependencies.ids.nextId("corpus-manifest"); - const createdAt = this.#dependencies.clock.now(); + }), + sourceId: source.sourceId, + contentDigest: source.contentDigest, + materializedInvocationVersion: source.invocation.version, + traceId: source.invocation.traceId, + integrity: source.invocation.integrity, + terminationStatus: source.invocation.terminationStatus, + evidenceIds: source.invocation.evidence.map((evidence) => evidence.evidenceId), + }; + }); + const artifactId = stableId("corpus-manifest", { idempotencyKey, documents }); return { metadata: this.#metadata( artifactId, [input.metadata.artifactId, ...documents.map((document) => document.sourceId)], input.metadata.scope, - createdAt, - { documents, excludedSources }, + input.metadata.createdAt, + { documents, excludedSources: [] }, ), documents, - excludedSources, + excludedSources: [], + }; + } + + #createInstructionSnapshot(input: EvolutionInputSnapshot, idempotencyKey: string): InstructionSnapshot { + for (const document of input.instructions) { + if (computeEvolutionContentDigest(document.content) !== document.contentDigest) { + throw new EvolutionExecutionError(`Instruction snapshot "${document.path}" has an invalid content digest`, { + reasonCode: "snapshot-invalid", + retryable: false, + }); + } + } + const artifactId = stableId("instruction-snapshot", { idempotencyKey, documents: input.instructions }); + return { + metadata: this.#metadata( + artifactId, + [input.metadata.artifactId], + input.metadata.scope, + input.metadata.createdAt, + input.instructions, + ), + documents: input.instructions, + }; + } + + #pipelineInputs(context: EvolutionStageExecutionContext): { + readonly corpus: EvidenceCorpusManifest; + readonly instructions: InstructionSnapshot; + readonly reflection: ReflectionResult; + readonly proposals: readonly EvolutionProposal[]; + } { + return { + corpus: requireOne(context.priorArtifacts.filter(isCorpus), "corpus manifest"), + instructions: requireOne(context.priorArtifacts.filter(isInstructionSnapshot), "instruction snapshot"), + reflection: { + nodes: context.priorArtifacts.filter(isReflectionNode), + lessons: context.priorArtifacts.filter(isLesson), + }, + proposals: context.priorArtifacts.filter(isProposal), }; } - #createMutationBatch(job: EvolutionJob, selected: readonly ScoredProposal[]): MutationBatch | null { - if (selected.length === 0) return null; + #createMutationBatch(context: EvolutionStageExecutionContext, selected: readonly ScoredProposal[]): MutationBatch { const proposals = selected.map((scored) => scored.proposal as EvolutionMemoryProposal); - const expectedHarnessVersion = proposals[0]?.target.expectedHarnessVersion; - if (expectedHarnessVersion === undefined) return null; - if (proposals.some((proposal) => proposal.target.expectedHarnessVersion !== expectedHarnessVersion)) { - throw new Error("Selected proposals do not share one expected harness version"); + if ( + proposals.length === 0 || + proposals.some( + (proposal) => proposal.target.expectedHarnessVersion !== context.input.expectedHarnessBaseVersion, + ) + ) { + throw new EvolutionExecutionError("Selected proposals do not share the frozen harness base version", { + reasonCode: "harness-base-conflict", + retryable: false, + }); } const mutations = proposals.map((proposal) => ({ - mutationId: this.#dependencies.ids.nextId("mutation"), + mutationId: stableId("mutation", { idempotencyKey: context.idempotencyKey, proposalId: proposal.proposalId }), proposalId: proposal.proposalId, target: proposal.target, payload: proposal.executablePayload, })); - const batchId = this.#dependencies.ids.nextId("mutation-batch"); - const createdAt = this.#dependencies.clock.now(); + const batchId = stableId("mutation-batch", { idempotencyKey: context.idempotencyKey, mutations }); return { metadata: this.#metadata( batchId, - [job.jobId, ...selected.map((scored) => scored.metadata.artifactId)], - job.metadata.scope, - createdAt, + [context.job.jobId, ...selected.map((scored) => scored.metadata.artifactId)], + context.job.metadata.scope, + context.input.metadata.createdAt, { batchId, targetKind: "evolution-memory", mutations }, ), batchId, @@ -278,222 +386,237 @@ export class EvolutionPipeline { }; } - async #recordProposalDisposition( - job: EvolutionJob, - scored: ScoredProposal, - disposition: EvolutionProposalLedgerDisposition, - reasonCodes: readonly string[] = scored.reasonCodes, - ): Promise { - const eventId = this.#dependencies.ids.nextId("ledger-event"); - const createdAt = this.#dependencies.clock.now(); - const content = { proposalId: scored.proposal.proposalId, disposition, reasonCodes }; - const event: ProposalDispositionLedgerEvent = { - metadata: this.#ledgerMetadata(eventId, job, createdAt, content), - eventId, - jobId: job.jobId, - type: "proposal-disposition", - proposalId: scored.proposal.proposalId, - disposition, - reasonCodes, - }; - await this.#appendLedger(event); - } - - async run(input: EvolutionInputSnapshot): Promise { - const jobId = this.#dependencies.ids.nextId("job"); - const createdAt = this.#dependencies.clock.now(); - const initialCheckpoint = { schemaVersion: EVOLUTION_SCHEMA_VERSION, stage: "created" } as const; - const initialContent = { - jobId, - inputSnapshotId: input.metadata.artifactId, - status: "running", - checkpoint: initialCheckpoint, - updatedAt: createdAt, - }; - let job: EvolutionJob = { - metadata: this.#metadata(jobId, [input.metadata.artifactId], input.metadata.scope, createdAt, initialContent), - jobId, - inputSnapshotId: input.metadata.artifactId, - status: "running", - checkpoint: initialCheckpoint, - updatedAt: createdAt, - }; - await this.#dependencies.artifacts.put(input); - await this.#dependencies.jobs.create(job); - const createdEventId = this.#dependencies.ids.nextId("ledger-event"); - const createdEvent: JobCreatedLedgerEvent = { - metadata: this.#ledgerMetadata(createdEventId, job, createdAt, initialContent), - eventId: createdEventId, - jobId, - type: "job-created", - inputSnapshotId: input.metadata.artifactId, - }; - await this.#appendLedger(createdEvent); - await this.#dependencies.events.emit({ - jobId, - emittedAt: createdAt, - type: "started", - checkpoint: initialCheckpoint, - }); - - let activeStage: ActiveEvolutionStage = "created"; - try { - const corpus = this.#createCorpus(input); - const instructions = await this.#dependencies.instructions.load(input); - await this.#putArtifacts([corpus, instructions]); - activeStage = "corpus-prepared"; - job = await this.#saveCheckpoint(job, { - schemaVersion: EVOLUTION_SCHEMA_VERSION, - stage: activeStage, - corpusManifestId: corpus.metadata.artifactId, - instructionSnapshotId: instructions.metadata.artifactId, - }); - - const reflection = await this.#dependencies.reflector.reflect({ - input, - corpus, - instructions, - model: this.#dependencies.model, - }); - await this.#putArtifacts([...reflection.nodes, ...reflection.lessons]); - activeStage = "reflected"; - job = await this.#saveCheckpoint(job, { - schemaVersion: EVOLUTION_SCHEMA_VERSION, - stage: activeStage, - reflectionNodeIds: reflection.nodes.map((node: ReflectionNode) => node.nodeId), - lessonIds: reflection.lessons.map((lesson) => lesson.lessonId), - }); - - const proposals = await this.#dependencies.curator.curate({ - input, - corpus, - instructions, - reflection, - model: this.#dependencies.model, - }); - assertUniqueIds( - proposals.map((proposal) => proposal.proposalId), - "proposal ID", - ); - await this.#putArtifacts(proposals); - activeStage = "curated"; - job = await this.#saveCheckpoint(job, { - schemaVersion: EVOLUTION_SCHEMA_VERSION, - stage: activeStage, - proposalIds: proposals.map((proposal) => proposal.proposalId), - }); - - const validationInput = { input, corpus, instructions, reflection, proposals }; - const validation = await this.#dependencies.validator.validate(validationInput); - await this.#dependencies.artifacts.put(validation); - activeStage = "validated"; - job = await this.#saveCheckpoint(job, { - schemaVersion: EVOLUTION_SCHEMA_VERSION, - stage: activeStage, - validationReportId: validation.metadata.artifactId, - }); - - const scoredProposals = await this.#dependencies.scorer.score({ ...validationInput, validation }); - assertScoredProposals(proposals, scoredProposals); - await this.#putArtifacts(scoredProposals); - activeStage = "scored"; - job = await this.#saveCheckpoint(job, { - schemaVersion: EVOLUTION_SCHEMA_VERSION, - stage: activeStage, - scoredProposalIds: scoredProposals.map((scored) => scored.metadata.artifactId), - }); - for (const scored of scoredProposals) { - const disposition = proposalDisposition(scored); - if (disposition !== null) await this.#recordProposalDisposition(job, scored, disposition); + async execute(context: EvolutionStageExecutionContext): Promise { + context.signal?.throwIfAborted(); + switch (context.job.stage) { + case "snapshotting": { + const corpus = this.#createCorpus(context.input, context.idempotencyKey); + const instructions = this.#createInstructionSnapshot(context.input, context.idempotencyKey); + return { + artifacts: [corpus, instructions], + state: { + corpusManifestId: corpus.metadata.artifactId, + instructionSnapshotId: instructions.metadata.artifactId, + }, + directive: "continue", + }; } - - const selected = scoredProposals.filter((scored) => scored.disposition === "selected"); - const mutationBatch = this.#createMutationBatch(job, selected); - let writeResult: MutationWriteResult | null = null; - if (mutationBatch !== null) { - await this.#dependencies.artifacts.put(mutationBatch); - const writer = this.#dependencies.targets.writerFor(mutationBatch.targetKind); - if (writer === undefined) { - throw new Error(`No enabled writer for evolution target "${mutationBatch.targetKind}"`); + case "counter-slot": + return { artifacts: [], state: { enabled: false }, directive: "continue" }; + case "reflecting": { + const { corpus, instructions } = this.#pipelineInputs(context); + let reflection: ReflectionResult; + try { + reflection = await this.#dependencies.reflector.reflect({ + input: context.input, + corpus, + instructions, + model: this.#dependencies.model, + idempotencyKey: context.idempotencyKey, + signal: context.signal, + }); + } catch (error) { + stageFailure(error, "Reflector model stage failed", "model-call-failed", true); } - writeResult = await writer.write(mutationBatch); - const writeEventId = this.#dependencies.ids.nextId("ledger-event"); - const writeEvent: MutationWrittenLedgerEvent = { - metadata: this.#ledgerMetadata(writeEventId, job, this.#dependencies.clock.now(), writeResult), - eventId: writeEventId, - jobId, - type: "mutation-written", - batchId: mutationBatch.batchId, - writeResult, + assertUniqueIds( + reflection.nodes.map((node) => node.nodeId), + "reflection node ID", + ); + assertUniqueIds( + reflection.lessons.map((lesson) => lesson.lessonId), + "lesson ID", + ); + return { + artifacts: [...reflection.nodes, ...reflection.lessons], + state: { + reflectionNodeIds: reflection.nodes.map((node) => node.nodeId), + lessonIds: reflection.lessons.map((lesson) => lesson.lessonId), + }, + directive: "continue", }; - await this.#appendLedger(writeEvent); - for (const scored of selected) { - await this.#recordProposalDisposition( - job, - scored, - writeResult.status === "applied" ? "applied" : "failed", - writeResult.status === "applied" ? scored.reasonCodes : ["writer-failed"], - ); + } + case "curating": { + const { corpus, instructions, reflection } = this.#pipelineInputs(context); + let proposals: readonly EvolutionProposal[]; + try { + proposals = await this.#dependencies.curator.curate({ + input: context.input, + corpus, + instructions, + reflection, + model: this.#dependencies.model, + idempotencyKey: context.idempotencyKey, + signal: context.signal, + }); + } catch (error) { + stageFailure(error, "Curator model stage failed", "model-call-failed", true); } + assertUniqueIds( + proposals.map((proposal) => proposal.proposalId), + "proposal ID", + ); + return { + artifacts: proposals, + state: { proposalIds: proposals.map((proposal) => proposal.proposalId) }, + directive: "continue", + }; } - activeStage = "written"; - job = await this.#saveCheckpoint(job, { - schemaVersion: EVOLUTION_SCHEMA_VERSION, - stage: activeStage, - mutationBatchId: mutationBatch?.batchId ?? null, - writeResult, - }); - if (writeResult?.status === "failed") { - throw new Error(writeResult.message ?? `Mutation batch "${writeResult.batchId}" failed`); + case "validating": { + const { corpus, instructions, reflection, proposals } = this.#pipelineInputs(context); + const validationInput = { + input: context.input, + corpus, + instructions, + reflection, + proposals, + signal: context.signal, + }; + let validation: ValidationReport; + let scoredProposals: readonly ScoredProposal[]; + try { + validation = await this.#dependencies.validator.validate(validationInput); + scoredProposals = await this.#dependencies.scorer.score({ ...validationInput, validation }); + } catch (error) { + stageFailure(error, "Deterministic proposal validation failed", "validation-rejected", false); + } + assertScoredProposals(proposals, scoredProposals); + assertValidationReport(proposals, validation, scoredProposals); + const acceptedProposalIds = validation.proposalResults + .filter((result) => result.disposition === "accepted") + .map((result) => result.proposalId); + const selectedProposalIds = scoredProposals + .filter((scored) => scored.disposition === "selected") + .map((scored) => scored.proposal.proposalId); + if (acceptedProposalIds.length > 0 && selectedProposalIds.length === 0) { + throw new EvolutionExecutionError("Accepted proposals produced no writable selection", { + reasonCode: "validation-rejected", + retryable: false, + }); + } + const ledgerEvents = scoredProposals.flatMap((scored): EvolutionLedgerEvent[] => { + const disposition: EvolutionProposalLedgerDisposition | null = + scored.disposition === "selected" ? null : scored.disposition; + return disposition === null ? [] : [this.#proposalDispositionEvent(context, scored, disposition)]; + }); + return { + artifacts: [validation, ...scoredProposals], + ledgerEvents, + state: { + validationReportId: validation.metadata.artifactId, + scoredProposalIds: scoredProposals.map((scored) => scored.metadata.artifactId), + acceptedProposalIds, + selectedProposalIds, + }, + directive: acceptedProposalIds.length === 0 ? "complete-no-op" : "continue", + }; } - - const resultingHarnessVersion = writeResult?.resultingHarnessVersion ?? null; - job = await this.#saveCheckpoint(job, { - schemaVersion: EVOLUTION_SCHEMA_VERSION, - stage: "completed", - resultingHarnessVersion, - }); - const completedEventId = this.#dependencies.ids.nextId("ledger-event"); - const completedEvent: JobCompletedLedgerEvent = { - metadata: this.#ledgerMetadata(completedEventId, job, this.#dependencies.clock.now(), { - resultingHarnessVersion, - }), - eventId: completedEventId, - jobId, - type: "job-completed", - resultingHarnessVersion, - }; - await this.#appendLedger(completedEvent); - return { - job, - corpus, - instructions, - reflection, - proposals, - validation, - scoredProposals, - mutationBatch, - writeResult, - }; - } catch (error) { - const failedCheckpoint: FailedCheckpoint = { - schemaVersion: EVOLUTION_SCHEMA_VERSION, - stage: "failed", - failedStage: activeStage, - message: error instanceof Error ? error.message : String(error), - }; - job = await this.#saveCheckpoint(job, failedCheckpoint); - const failedEventId = this.#dependencies.ids.nextId("ledger-event"); - const failedEvent: JobFailedLedgerEvent = { - metadata: this.#ledgerMetadata(failedEventId, job, this.#dependencies.clock.now(), failedCheckpoint), - eventId: failedEventId, - jobId, - type: "job-failed", - failedStage: activeStage, - message: failedCheckpoint.message, - }; - await this.#appendLedger(failedEvent); - throw error; + case "writing": { + const selected = context.priorArtifacts + .filter(isScoredProposal) + .filter((item) => item.disposition === "selected"); + const batch = this.#createMutationBatch(context, selected); + const writer = this.#dependencies.targets.writerFor(batch.targetKind); + if (writer === undefined || context.writerLease === null) { + throw new EvolutionExecutionError("Evolution memory writer or writer lease is unavailable", { + reasonCode: "writer-failed", + retryable: true, + }); + } + let result: Awaited>; + try { + result = await writer.write(batch, { + idempotencyKey: context.idempotencyKey, + lease: context.writerLease, + signal: context.signal, + }); + } catch (error) { + stageFailure(error, `Mutation batch "${batch.batchId}" write failed`, "writer-failed", true); + } + if (result.status === "failed") { + throw new EvolutionExecutionError(result.message ?? `Mutation batch "${batch.batchId}" failed`, { + reasonCode: "writer-failed", + retryable: true, + }); + } + const receiptId = stableId("write-receipt", { idempotencyKey: context.idempotencyKey, result }); + const receipt: MutationWriteReceipt = { + metadata: this.#metadata( + receiptId, + [batch.metadata.artifactId], + context.job.metadata.scope, + context.input.metadata.createdAt, + { idempotencyKey: context.idempotencyKey, result }, + ), + batchId: batch.batchId, + idempotencyKey: context.idempotencyKey, + result, + }; + return { + artifacts: [batch, receipt], + state: { mutationBatchId: batch.batchId, writeReceiptId: receiptId }, + directive: "continue", + }; + } + case "activating": { + const batch = requireOne(context.priorArtifacts.filter(isMutationBatch), "mutation batch"); + const receipt = requireOne(context.priorArtifacts.filter(isMutationWriteReceipt), "mutation write receipt"); + const writer = this.#dependencies.targets.writerFor(batch.targetKind); + if (writer === undefined || context.writerLease === null) { + throw new EvolutionExecutionError("Evolution memory activator or writer lease is unavailable", { + reasonCode: "writer-failed", + retryable: true, + }); + } + let result: Awaited>; + try { + result = await writer.activate(batch, receipt.result, { + idempotencyKey: context.idempotencyKey, + lease: context.writerLease, + signal: context.signal, + }); + } catch (error) { + stageFailure(error, `Mutation batch "${batch.batchId}" activation failed`, "writer-failed", true); + } + if (result.status === "failed" || result.resultingHarnessVersion === null) { + throw new EvolutionExecutionError( + result.message ?? `Mutation batch "${batch.batchId}" activation failed`, + { + reasonCode: "writer-failed", + retryable: true, + }, + ); + } + const receiptId = stableId("activation-receipt", { idempotencyKey: context.idempotencyKey, result }); + const activationReceipt: HarnessActivationReceipt = { + metadata: this.#metadata( + receiptId, + [receipt.metadata.artifactId], + context.job.metadata.scope, + context.input.metadata.createdAt, + { idempotencyKey: context.idempotencyKey, result }, + ), + batchId: batch.batchId, + idempotencyKey: context.idempotencyKey, + result, + }; + const appliedEvents = context.priorArtifacts + .filter(isScoredProposal) + .filter((scored) => scored.disposition === "selected") + .map((scored) => this.#proposalDispositionEvent(context, scored, "applied")); + return { + artifacts: [activationReceipt], + ledgerEvents: appliedEvents, + state: { activationReceiptId: receiptId }, + directive: "complete-activated", + resultingHarnessVersion: result.resultingHarnessVersion, + }; + } + case "queued": + case "completed": + throw new EvolutionExecutionError(`Evolution stage "${context.job.stage}" is not executable`, { + reasonCode: "pipeline-invariant", + retryable: false, + }); } } } diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 2664a848c..6a230f104 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -1,14 +1,28 @@ +import type { MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; import type { EvidenceCorpusManifest, - EvolutionCheckpoint, + EvolutionArtifactReference, EvolutionInputSnapshot, EvolutionJob, + EvolutionJobStage, + EvolutionLeaseToken, EvolutionLedgerEvent, + EvolutionModelIdentity, + EvolutionProcessedEvidenceCursor, + EvolutionPromptVersions, EvolutionProposal, + EvolutionRepositorySnapshot, + EvolutionScope, + EvolutionStageCheckpoint, + EvolutionTriggerKind, + HarnessActivationReceipt, + ImmutableHarnessSnapshot, + InstructionDocumentSnapshot, InstructionSnapshot, JsonObject, Lesson, MutationBatch, + MutationWriteReceipt, ReflectionNode, ScoredProposal, ValidationReport, @@ -18,9 +32,11 @@ export type EvolutionArtifact = | EvidenceCorpusManifest | EvolutionInputSnapshot | EvolutionProposal + | HarnessActivationReceipt | InstructionSnapshot | Lesson | MutationBatch + | MutationWriteReceipt | ReflectionNode | ScoredProposal | ValidationReport; @@ -36,7 +52,9 @@ export interface EvolutionIdGenerator { export interface EvolutionModelRequest { readonly stage: "curator" | "reflector"; readonly promptVersion: string; + readonly idempotencyKey: string; readonly input: JsonObject; + readonly signal?: AbortSignal; } export interface EvolutionModelResponse { @@ -51,15 +69,13 @@ export interface EvolutionModelExecutor { execute(request: EvolutionModelRequest): Promise; } -export interface InstructionLoader { - load(input: EvolutionInputSnapshot): Promise; -} - export interface ReflectorInput { readonly input: EvolutionInputSnapshot; readonly corpus: EvidenceCorpusManifest; readonly instructions: InstructionSnapshot; readonly model: EvolutionModelExecutor; + readonly idempotencyKey: string; + readonly signal?: AbortSignal; } export interface ReflectionResult { @@ -77,6 +93,8 @@ export interface CuratorInput { readonly instructions: InstructionSnapshot; readonly reflection: ReflectionResult; readonly model: EvolutionModelExecutor; + readonly idempotencyKey: string; + readonly signal?: AbortSignal; } export interface Curator { @@ -89,6 +107,7 @@ export interface ProposalValidationInput { readonly instructions: InstructionSnapshot; readonly reflection: ReflectionResult; readonly proposals: readonly EvolutionProposal[]; + readonly signal?: AbortSignal; } export interface ProposalValidator { @@ -104,13 +123,41 @@ export interface ProposalScorer { } export interface EvolutionArtifactStore { - put(artifact: EvolutionArtifact): Promise; + put(artifact: EvolutionArtifact): Promise; + get(reference: EvolutionArtifactReference): Promise; } export interface EvolutionJobStore { - create(job: EvolutionJob): Promise; - save(job: EvolutionJob): Promise; + create(job: EvolutionJob, lease: EvolutionLeaseToken): Promise; + save(job: EvolutionJob, expectedRevision: number, lease: EvolutionLeaseToken): Promise; load(jobId: string): Promise; + list(scope?: EvolutionScope): Promise; +} + +export type EvolutionCursorCommitResult = "already-committed" | "committed" | "conflict"; + +export interface EvolutionCursorStore { + load(scope: EvolutionScope): Promise; + commit( + scope: EvolutionScope, + expectedSequence: number, + nextSequence: number, + jobId: string, + updatedAt: string, + lease: EvolutionLeaseToken, + ): Promise; +} + +export interface EvolutionLeaseStore { + acquire( + kind: EvolutionLeaseToken["kind"], + key: string, + ownerId: string, + now: string, + ttlMs: number, + ): Promise; + renew(lease: EvolutionLeaseToken, now: string, ttlMs: number): Promise; + release(lease: EvolutionLeaseToken): Promise; } export interface EvolutionLedger { @@ -121,10 +168,84 @@ export interface EvolutionLedger { export interface EvolutionProgressEvent { readonly jobId: string; readonly emittedAt: string; - readonly type: "checkpoint" | "failed" | "started"; - readonly checkpoint: EvolutionCheckpoint; + readonly type: "checkpoint" | "completed" | "failed" | "started"; + readonly stage: EvolutionJobStage; + readonly checkpoint?: EvolutionStageCheckpoint; } export interface EvolutionEventSink { emit(event: EvolutionProgressEvent): void | Promise; } + +export interface SuccessfulMaterialization { + readonly materializationId: string; + readonly ingestionCursor: number; + readonly origin: "product-runtime"; + readonly contentDigest: string; + readonly invocation: MaterializedInvocation; +} + +export interface EvolutionEvidenceCatalog { + highWaterCursor(scope: EvolutionScope): Promise; + listSuccessful( + scope: EvolutionScope, + afterCursor: number, + throughCursor: number, + ): Promise; +} + +export interface EvolutionSnapshotContext { + readonly repository: EvolutionRepositorySnapshot; + readonly harness: ImmutableHarnessSnapshot; + readonly instructions: readonly InstructionDocumentSnapshot[]; + readonly modelConfiguration: EvolutionModelIdentity; + readonly pipelineVersion: string; + readonly promptVersions: EvolutionPromptVersions; +} + +export interface EvolutionSnapshotSource { + capture(scope: EvolutionScope, signal?: AbortSignal): Promise; +} + +export interface EvolutionStageExecutionContext { + readonly job: EvolutionJob; + readonly input: EvolutionInputSnapshot; + readonly priorArtifacts: readonly EvolutionArtifact[]; + readonly idempotencyKey: string; + readonly scopeLease: EvolutionLeaseToken; + readonly writerLease: EvolutionLeaseToken | null; + readonly signal?: AbortSignal; +} + +export type EvolutionStageDirective = "complete-activated" | "complete-no-op" | "continue"; + +export interface EvolutionStageExecutionResult { + readonly artifacts: readonly EvolutionArtifact[]; + readonly ledgerEvents?: readonly EvolutionLedgerEvent[]; + readonly state: JsonObject; + readonly directive: EvolutionStageDirective; + readonly resultingHarnessVersion?: string; +} + +export interface EvolutionStageExecutor { + execute(context: EvolutionStageExecutionContext): Promise; +} + +export interface EvolutionJobCreationRequest { + readonly scope: EvolutionScope; + readonly trigger: EvolutionTriggerKind; + readonly signal?: AbortSignal; +} + +export interface EvolutionJobRunner { + runJob(jobId: string, signal?: AbortSignal): Promise; +} + +export interface EvolutionWorkSource { + nextRunnableJob(now: string, signal?: AbortSignal): Promise; + waitForWork(signal: AbortSignal): Promise; +} + +export interface InteractivePriorityGate { + waitUntilBackgroundAllowed(signal: AbortSignal): Promise; +} diff --git a/packages/evolution/src/provenance.ts b/packages/evolution/src/provenance.ts index 610c1e32f..a44bd4429 100644 --- a/packages/evolution/src/provenance.ts +++ b/packages/evolution/src/provenance.ts @@ -25,10 +25,14 @@ function canonicalize(value: unknown, path: string): JsonValue { } export function computeEvolutionContentDigest(value: unknown): string { - const canonical = JSON.stringify(canonicalize(value, "$")); + const canonical = serializeEvolutionValue(value); return `${CONTENT_DIGEST_PREFIX}${createHash("sha256").update(canonical).digest("hex")}`; } +export function serializeEvolutionValue(value: unknown): string { + return JSON.stringify(canonicalize(value, "$")); +} + export interface CreateEvolutionArtifactMetadataOptions { readonly artifactId: string; readonly producer: EvolutionArtifactProducer; diff --git a/packages/evolution/src/recovery.ts b/packages/evolution/src/recovery.ts new file mode 100644 index 000000000..246b9cc62 --- /dev/null +++ b/packages/evolution/src/recovery.ts @@ -0,0 +1,80 @@ +import type { EvolutionJob } from "./contracts.ts"; + +const DAY_MS = 24 * 60 * 60 * 1_000; + +export interface EvolutionRetentionPolicy { + readonly completedJobRetentionMs: number; + readonly resolvedJobRetentionMs: number; + readonly orphanArtifactRetentionMs: number; + readonly staleTemporaryFileRetentionMs: number; + readonly corruptArtifactBehavior: "retain-and-report"; + readonly interruptedJobBehavior: "resume-original-snapshot"; + readonly unresolvedFailureBehavior: "retain-and-block-overlap"; + readonly staleLeaseBehavior: "fenced-takeover"; +} + +export const DEFAULT_EVOLUTION_RETENTION_POLICY: EvolutionRetentionPolicy = Object.freeze({ + completedJobRetentionMs: 30 * DAY_MS, + resolvedJobRetentionMs: 30 * DAY_MS, + orphanArtifactRetentionMs: 7 * DAY_MS, + staleTemporaryFileRetentionMs: DAY_MS, + corruptArtifactBehavior: "retain-and-report", + interruptedJobBehavior: "resume-original-snapshot", + unresolvedFailureBehavior: "retain-and-block-overlap", + staleLeaseBehavior: "fenced-takeover", +}); + +export interface EvolutionRetentionPlan { + readonly removableJobIds: readonly string[]; + readonly retainedBlockingJobIds: readonly string[]; +} + +function assertDuration(value: number, name: string): void { + if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a non-negative duration`); +} + +function elapsedSince(timestamp: string, nowMs: number, description: string): number { + const value = Date.parse(timestamp); + if (!Number.isFinite(value)) throw new Error(`${description} must be a valid ISO timestamp`); + return nowMs - value; +} + +export function planEvolutionRetention( + jobs: readonly EvolutionJob[], + now: string, + policy: EvolutionRetentionPolicy = DEFAULT_EVOLUTION_RETENTION_POLICY, +): EvolutionRetentionPlan { + assertDuration(policy.completedJobRetentionMs, "completedJobRetentionMs"); + assertDuration(policy.resolvedJobRetentionMs, "resolvedJobRetentionMs"); + assertDuration(policy.orphanArtifactRetentionMs, "orphanArtifactRetentionMs"); + assertDuration(policy.staleTemporaryFileRetentionMs, "staleTemporaryFileRetentionMs"); + const nowMs = Date.parse(now); + if (!Number.isFinite(nowMs)) throw new Error("Retention planning time must be a valid ISO timestamp"); + + const removableJobIds: string[] = []; + const retainedBlockingJobIds: string[] = []; + for (const job of jobs) { + if (job.outcome === "completed" && job.terminalAt !== null) { + if ( + elapsedSince(job.terminalAt, nowMs, `Job "${job.jobId}" terminal time`) >= policy.completedJobRetentionMs + ) { + removableJobIds.push(job.jobId); + } + continue; + } + if (job.resolution !== null) { + if ( + elapsedSince(job.resolution.resolvedAt, nowMs, `Job "${job.jobId}" resolution time`) >= + policy.resolvedJobRetentionMs + ) { + removableJobIds.push(job.jobId); + } + continue; + } + if (job.outcome !== "superseded") retainedBlockingJobIds.push(job.jobId); + } + return { + removableJobIds: removableJobIds.sort(), + retainedBlockingJobIds: retainedBlockingJobIds.sort(), + }; +} diff --git a/packages/evolution/src/targets.ts b/packages/evolution/src/targets.ts index f03ba227e..db2fe2083 100644 --- a/packages/evolution/src/targets.ts +++ b/packages/evolution/src/targets.ts @@ -1,10 +1,24 @@ -import type { EvolutionTarget, MutationBatch, MutationWriteResult } from "./contracts.ts"; +import type { + EvolutionLeaseToken, + EvolutionTarget, + HarnessActivationResult, + MutationBatch, + MutationWriteResult, +} from "./contracts.ts"; export type EvolutionTargetKind = EvolutionTarget["kind"]; export interface EvolutionTargetWriter { readonly kind: EvolutionTargetKind; - write(batch: MutationBatch): Promise; + write( + batch: MutationBatch, + context: { readonly idempotencyKey: string; readonly lease: EvolutionLeaseToken; readonly signal?: AbortSignal }, + ): Promise; + activate( + batch: MutationBatch, + writeResult: MutationWriteResult, + context: { readonly idempotencyKey: string; readonly lease: EvolutionLeaseToken; readonly signal?: AbortSignal }, + ): Promise; } export interface EvolutionTargetAdapterRegistryOptions { diff --git a/packages/evolution/src/token-estimator.ts b/packages/evolution/src/token-estimator.ts new file mode 100644 index 000000000..8915bc40b --- /dev/null +++ b/packages/evolution/src/token-estimator.ts @@ -0,0 +1,52 @@ +import type { MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import { serializeEvolutionValue } from "./provenance.ts"; + +export const EVOLUTION_TOKEN_ESTIMATOR_VERSION = "canonical-json-utf8-v1" as const; +export const AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD = 1_000_000; +export const AUTOMATIC_EVOLUTION_MINIMUM_INVOCATIONS = 3; +export const AUTOMATIC_EVOLUTION_COOLDOWN_MS = 20 * 60 * 1_000; + +const CHARACTERS_PER_ESTIMATED_TOKEN = 4; + +export function estimateEvolutionTokens(invocations: readonly MaterializedInvocation[]): number { + const canonicalCorpus = serializeEvolutionValue(invocations); + return Math.ceil(new TextEncoder().encode(canonicalCorpus).byteLength / CHARACTERS_PER_ESTIMATED_TOKEN); +} + +export type AutomaticEvolutionEligibilityReason = + | "cooldown-active" + | "insufficient-invocations" + | "insufficient-tokens"; + +export type AutomaticEvolutionEligibility = + | { readonly eligible: true } + | { readonly eligible: false; readonly reason: AutomaticEvolutionEligibilityReason }; + +export interface AutomaticEvolutionEligibilityInput { + readonly invocationCount: number; + readonly estimatedTokens: number; + readonly now: string; + readonly previousAutomaticTerminalAt: string | null; +} + +export function evaluateAutomaticEvolutionEligibility( + input: AutomaticEvolutionEligibilityInput, +): AutomaticEvolutionEligibility { + if (input.invocationCount < AUTOMATIC_EVOLUTION_MINIMUM_INVOCATIONS) { + return { eligible: false, reason: "insufficient-invocations" }; + } + if (input.estimatedTokens <= AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD) { + return { eligible: false, reason: "insufficient-tokens" }; + } + if (input.previousAutomaticTerminalAt !== null) { + const terminalAt = Date.parse(input.previousAutomaticTerminalAt); + const now = Date.parse(input.now); + if (!Number.isFinite(terminalAt) || !Number.isFinite(now)) { + throw new Error("Automatic evolution cooldown timestamps must be valid ISO dates"); + } + if (now - terminalAt < AUTOMATIC_EVOLUTION_COOLDOWN_MS) { + return { eligible: false, reason: "cooldown-active" }; + } + } + return { eligible: true }; +} diff --git a/packages/evolution/src/worker.ts b/packages/evolution/src/worker.ts new file mode 100644 index 000000000..eccd3ea2b --- /dev/null +++ b/packages/evolution/src/worker.ts @@ -0,0 +1,149 @@ +import type { + EvolutionClock, + EvolutionJobRunner, + EvolutionJobStore, + EvolutionWorkSource, + InteractivePriorityGate, +} from "./ports.ts"; + +const DEFAULT_IDLE_POLL_MS = 1_000; +const DEFAULT_FAILURE_BACKOFF_MS = 5_000; + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new Error("Evolution background worker stopped"); +} + +function wait(delayMs: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(abortReason(signal)); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timer); + reject(abortReason(signal)); + }; + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +export interface PollingEvolutionWorkSourceOptions { + readonly pollIntervalMs?: number; +} + +export class PollingEvolutionWorkSource implements EvolutionWorkSource { + readonly #jobs: EvolutionJobStore; + readonly #clock: EvolutionClock; + readonly #pollIntervalMs: number; + + constructor(jobs: EvolutionJobStore, clock: EvolutionClock, options: PollingEvolutionWorkSourceOptions = {}) { + this.#jobs = jobs; + this.#clock = clock; + this.#pollIntervalMs = options.pollIntervalMs ?? DEFAULT_IDLE_POLL_MS; + if (!Number.isFinite(this.#pollIntervalMs) || this.#pollIntervalMs < 1) { + throw new Error("pollIntervalMs must be positive"); + } + } + + async nextRunnableJob(now: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const nowMs = Date.parse(now); + if (!Number.isFinite(nowMs)) throw new Error("Evolution worker time must be a valid ISO timestamp"); + const jobs = (await this.#jobs.list()) + .filter( + (job) => + job.resolution === null && + (job.outcome === "active" || job.outcome === "interrupted" || job.outcome === "retryable-failed") && + job.stage !== "completed" && + (job.retryNotBefore === null || Date.parse(job.retryNotBefore) <= nowMs), + ) + .sort( + (left, right) => + Number(left.trigger === "automatic") - Number(right.trigger === "automatic") || + left.createdAt.localeCompare(right.createdAt) || + left.jobId.localeCompare(right.jobId), + ); + signal?.throwIfAborted(); + return jobs[0]?.jobId ?? null; + } + + waitForWork(signal: AbortSignal): Promise { + return wait(this.#pollIntervalMs, signal); + } + + next(signal?: AbortSignal): Promise { + return this.nextRunnableJob(this.#clock.now(), signal); + } +} + +export interface EvolutionBackgroundWorkerDependencies { + readonly clock: EvolutionClock; + readonly source: EvolutionWorkSource; + readonly runner: EvolutionJobRunner; + readonly priority: InteractivePriorityGate; +} + +export interface EvolutionBackgroundWorkerOptions { + readonly failureBackoffMs?: number; +} + +export class EvolutionBackgroundWorker { + readonly #dependencies: EvolutionBackgroundWorkerDependencies; + readonly #failureBackoffMs: number; + #controller: AbortController | null = null; + #runPromise: Promise | null = null; + + constructor(dependencies: EvolutionBackgroundWorkerDependencies, options: EvolutionBackgroundWorkerOptions = {}) { + this.#dependencies = dependencies; + this.#failureBackoffMs = options.failureBackoffMs ?? DEFAULT_FAILURE_BACKOFF_MS; + if (!Number.isFinite(this.#failureBackoffMs) || this.#failureBackoffMs < 0) { + throw new Error("failureBackoffMs must be non-negative"); + } + } + + async runOnce(signal: AbortSignal): Promise<"idle" | "ran"> { + signal.throwIfAborted(); + const jobId = await this.#dependencies.source.nextRunnableJob(this.#dependencies.clock.now(), signal); + if (jobId === null) return "idle"; + await this.#dependencies.priority.waitUntilBackgroundAllowed(signal); + signal.throwIfAborted(); + await this.#dependencies.runner.runJob(jobId, signal); + return "ran"; + } + + start(): void { + if (this.#runPromise !== null) return; + const controller = new AbortController(); + this.#controller = controller; + this.#runPromise = this.#run(controller.signal).finally(() => { + if (this.#controller === controller) this.#controller = null; + this.#runPromise = null; + }); + void this.#runPromise.catch(() => {}); + } + + async #run(signal: AbortSignal): Promise { + while (!signal.aborted) { + try { + const result = await this.runOnce(signal); + if (result === "idle") await this.#dependencies.source.waitForWork(signal); + } catch { + if (signal.aborted) return; + await wait(this.#failureBackoffMs, signal); + } + } + } + + async stop(reason: unknown = new Error("Evolution background worker stopped")): Promise { + const runPromise = this.#runPromise; + if (runPromise === null) return; + const controller = this.#controller; + controller?.abort(reason); + try { + await runPromise; + } catch (error) { + if (!controller?.signal.aborted) throw error; + } + } +} diff --git a/packages/evolution/test/pipeline.test.ts b/packages/evolution/test/pipeline.test.ts deleted file mode 100644 index cebd9501e..000000000 --- a/packages/evolution/test/pipeline.test.ts +++ /dev/null @@ -1,441 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import type { MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; -import { describe, expect, it } from "vitest"; -import { - computeEvolutionContentDigest, - createEvolutionArtifactMetadata, - type EvolutionArtifact, - type EvolutionArtifactMetadata, - type EvolutionArtifactProducer, - type EvolutionEventSink, - type EvolutionInputSnapshot, - type EvolutionJob, - type EvolutionLedgerEvent, - type EvolutionMemoryProposal, - type EvolutionModelRequest, - EvolutionPipeline, - type EvolutionProgressEvent, - type EvolutionProposal, - type EvolutionScope, - EvolutionTargetAdapterRegistry, - type EvolutionTargetWriter, - type InstructionSnapshot, - type Lesson, - type MutationBatch, - type MutationWriteResult, - type ReflectionNode, - type ScoredProposal, - type ValidationReport, -} from "../src/index.ts"; - -const FIRST_HARNESS_VERSION = `sha256:${"1".repeat(64)}`; -const SECOND_HARNESS_VERSION = `sha256:${"2".repeat(64)}`; -const CREATED_AT = "2026-08-19T12:00:00.000Z"; -const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; -const pipelineProducer: EvolutionArtifactProducer = { - name: "diffex-evolution", - version: "1.0.0", - promptVersion: null, - model: null, -}; -const reflectorProducer: EvolutionArtifactProducer = { - name: "reflector", - version: "1.0.0", - promptVersion: "reflector-v1", - model: { provider: "faux", model: "fixture", configurationDigest: `sha256:${"3".repeat(64)}` }, -}; -const curatorProducer: EvolutionArtifactProducer = { - name: "curator", - version: "1.0.0", - promptVersion: "curator-v1", - model: { provider: "faux", model: "fixture", configurationDigest: `sha256:${"3".repeat(64)}` }, -}; - -function metadata( - artifactId: string, - producer: EvolutionArtifactProducer, - sourceIds: readonly string[], - content: unknown, -): EvolutionArtifactMetadata { - return createEvolutionArtifactMetadata({ artifactId, producer, sourceIds, scope, createdAt: CREATED_AT, content }); -} - -function invocation(traceId: string): MaterializedInvocation { - return { - version: 1, - tracePath: `/traces/${traceId}.jsonl`, - traceId, - producer: null, - integrity: "complete", - terminationStatus: "completed", - health: null, - ordering: { - criteria: ["trace_sequence", "session_cursor", "timestamp", "span_parentage", "causal_id"], - ambiguity: [], - }, - evidence: [], - diagnostics: [], - }; -} - -function inputSnapshot(): EvolutionInputSnapshot { - const productInvocation = invocation("product-trace"); - const evolutionInvocation = invocation("evolution-trace"); - const content = { - harnessVersion: FIRST_HARNESS_VERSION, - evidenceSourceIds: ["source-product", "source-evolution"], - }; - return { - metadata: metadata("input-1", pipelineProducer, ["source-product", "source-evolution"], content), - harness: { - harnessVersion: FIRST_HARNESS_VERSION, - baseVersion: null, - memory: "# Evolution memory\n", - memoryContentDigest: computeEvolutionContentDigest("# Evolution memory\n"), - }, - evidenceSources: [ - { - sourceId: "source-product", - origin: "product-runtime", - contentDigest: computeEvolutionContentDigest(productInvocation), - invocation: productInvocation, - }, - { - sourceId: "source-evolution", - origin: "evolution-pipeline", - contentDigest: computeEvolutionContentDigest(evolutionInvocation), - invocation: evolutionInvocation, - }, - ], - }; -} - -class SequenceIds { - #next = 1; - - nextId(kind: string): string { - const value = `${kind}-${this.#next}`; - this.#next += 1; - return value; - } -} - -class FixedClock { - now(): string { - return CREATED_AT; - } -} - -interface RecordingState { - readonly modelStages: EvolutionModelRequest["stage"][]; - readonly artifacts: EvolutionArtifact[]; - readonly createdJobs: EvolutionJob[]; - readonly savedJobs: EvolutionJob[]; - readonly ledger: EvolutionLedgerEvent[]; - readonly progress: EvolutionProgressEvent[]; - readonly writtenBatches: MutationBatch[]; -} - -function instructionSnapshot(input: EvolutionInputSnapshot): InstructionSnapshot { - const documents = [ - { - path: "/repo/AGENTS.md", - precedence: 1, - content: "Preserve source provenance.", - contentDigest: computeEvolutionContentDigest("Preserve source provenance."), - }, - ]; - return { - metadata: metadata("instructions-1", pipelineProducer, [input.metadata.artifactId], documents), - documents, - }; -} - -function reflectionArtifacts(): { readonly node: ReflectionNode; readonly lesson: Lesson } { - const lesson: Lesson = { - metadata: metadata("lesson-1", reflectorProducer, ["source-product"], { summary: "Keep narrow deltas" }), - lessonId: "lesson-1", - kind: "strategy", - summary: "Keep narrow deltas", - evidenceIds: [], - counterevidenceIds: [], - scope, - uncertainty: null, - }; - const node: ReflectionNode = { - metadata: metadata("reflection-1", reflectorProducer, ["source-product"], { lessonIds: [lesson.lessonId] }), - nodeId: "reflection-1", - parentNodeId: null, - childNodeIds: [], - evidenceIds: [], - lessonIds: [lesson.lessonId], - status: "complete", - }; - return { node, lesson }; -} - -function proposals(): readonly EvolutionProposal[] { - const actionable = (proposalId: string, payload: string): EvolutionMemoryProposal => ({ - metadata: metadata(proposalId, curatorProducer, ["lesson-1"], { payload }), - proposalId, - summary: payload, - lessonIds: ["lesson-1"], - evidenceIds: [], - disposition: "actionable", - target: { kind: "evolution-memory", expectedHarnessVersion: FIRST_HARNESS_VERSION }, - executablePayload: { content: payload }, - }); - return [ - actionable("proposal-applied", "Append one focused memory entry"), - { - metadata: metadata("proposal-deferred", curatorProducer, ["lesson-1"], { finding: "Improve a skill" }), - proposalId: "proposal-deferred", - summary: "Improve a skill", - lessonIds: ["lesson-1"], - evidenceIds: [], - disposition: "deferred", - target: { kind: "skill", skillName: "review" }, - finding: "A future skill adapter could apply this finding", - }, - actionable("proposal-rejected", "Unverified memory entry"), - ]; -} - -function validationReport(inputProposals: readonly EvolutionProposal[]): ValidationReport { - const proposalResults = inputProposals.map((proposal) => ({ - proposalId: proposal.proposalId, - disposition: - proposal.proposalId === "proposal-applied" - ? ("accepted" as const) - : proposal.disposition === "deferred" - ? ("deferred" as const) - : ("rejected" as const), - issues: [], - })); - return { - metadata: metadata( - "validation-1", - pipelineProducer, - inputProposals.map((proposal) => proposal.proposalId), - { - proposalResults, - }, - ), - proposalResults, - }; -} - -function scoredProposals(inputProposals: readonly EvolutionProposal[]): readonly ScoredProposal[] { - return inputProposals.map((proposal) => { - const disposition = - proposal.proposalId === "proposal-applied" - ? ("selected" as const) - : proposal.disposition === "deferred" - ? ("deferred" as const) - : ("rejected" as const); - return { - metadata: metadata(`score-${proposal.proposalId}`, pipelineProducer, [proposal.proposalId], { disposition }), - proposal, - score: disposition === "selected" ? 80 : null, - disposition, - reasonCodes: [disposition], - }; - }); -} - -function recordingPipeline(writeStatus: MutationWriteResult["status"] = "applied"): { - readonly pipeline: EvolutionPipeline; - readonly state: RecordingState; -} { - const state: RecordingState = { - modelStages: [], - artifacts: [], - createdJobs: [], - savedJobs: [], - ledger: [], - progress: [], - writtenBatches: [], - }; - const writer: EvolutionTargetWriter = { - kind: "evolution-memory", - async write(batch) { - state.writtenBatches.push(batch); - return { - batchId: batch.batchId, - status: writeStatus, - resultingHarnessVersion: writeStatus === "applied" ? SECOND_HARNESS_VERSION : null, - message: writeStatus === "failed" ? "fixture writer failure" : null, - }; - }, - }; - const events: EvolutionEventSink = { - emit(event) { - state.progress.push(event); - }, - }; - const pipeline = new EvolutionPipeline({ - version: "1.0.0", - clock: new FixedClock(), - ids: new SequenceIds(), - model: { - async execute(request) { - state.modelStages.push(request.stage); - return { output: { accepted: true }, model: { provider: "faux", model: "fixture" } }; - }, - }, - instructions: { - async load(input) { - return instructionSnapshot(input); - }, - }, - reflector: { - async reflect(input) { - await input.model.execute({ stage: "reflector", promptVersion: "reflector-v1", input: {} }); - const { node, lesson } = reflectionArtifacts(); - return { nodes: [node], lessons: [lesson] }; - }, - }, - curator: { - async curate(input) { - await input.model.execute({ stage: "curator", promptVersion: "curator-v1", input: {} }); - return proposals(); - }, - }, - validator: { - async validate(input) { - return validationReport(input.proposals); - }, - }, - scorer: { - async score(input) { - return scoredProposals(input.proposals); - }, - }, - artifacts: { - async put(artifact) { - state.artifacts.push(artifact); - }, - }, - jobs: { - async create(job) { - state.createdJobs.push(job); - }, - async save(job) { - state.savedJobs.push(job); - }, - async load(jobId) { - return [...state.createdJobs, ...state.savedJobs].filter((job) => job.jobId === jobId).at(-1) ?? null; - }, - }, - ledger: { - async append(event) { - state.ledger.push(event); - }, - async read(jobId) { - return state.ledger.filter((event) => event.jobId === jobId); - }, - }, - events, - targets: new EvolutionTargetAdapterRegistry({ writers: [writer] }), - }); - return { pipeline, state }; -} - -describe("evolution pipeline", () => { - it("orchestrates isolated stages and writes only enabled memory proposals", async () => { - const { pipeline, state } = recordingPipeline(); - const result = await pipeline.run(inputSnapshot()); - - expect(state.modelStages).toEqual(["reflector", "curator"]); - expect(state.savedJobs.map((job) => job.checkpoint.stage)).toEqual([ - "corpus-prepared", - "reflected", - "curated", - "validated", - "scored", - "written", - "completed", - ]); - expect(result.corpus.documents.map((document) => document.sourceId)).toEqual(["source-product"]); - expect(result.corpus.excludedSources).toEqual([ - { sourceId: "source-evolution", reason: "evolution-pipeline-origin" }, - ]); - expect(state.writtenBatches).toHaveLength(1); - expect(state.writtenBatches[0]?.mutations.map((mutation) => mutation.proposalId)).toEqual(["proposal-applied"]); - expect(result.job.status).toBe("completed"); - expect(result.job.checkpoint).toEqual({ - schemaVersion: 1, - stage: "completed", - resultingHarnessVersion: SECOND_HARNESS_VERSION, - }); - - const dispositions = state.ledger - .filter((event) => event.type === "proposal-disposition") - .map((event) => [event.proposalId, event.disposition]); - expect(dispositions).toEqual([ - ["proposal-deferred", "deferred"], - ["proposal-rejected", "rejected"], - ["proposal-applied", "applied"], - ]); - expect(state.artifacts.map((artifact) => artifact.metadata.artifactId)).toContain("input-1"); - expect(state.artifacts.map((artifact) => artifact.metadata.artifactId)).toContain("lesson-1"); - expect(state.artifacts.map((artifact) => artifact.metadata.artifactId)).toContain("proposal-applied"); - }); - - it("records writer failure without converting deferred findings into mutations", async () => { - const { pipeline, state } = recordingPipeline("failed"); - - await expect(pipeline.run(inputSnapshot())).rejects.toThrow("fixture writer failure"); - expect(state.savedJobs.at(-1)?.checkpoint).toMatchObject({ stage: "failed", failedStage: "written" }); - expect(state.writtenBatches[0]?.mutations.map((mutation) => mutation.proposalId)).toEqual(["proposal-applied"]); - expect( - state.ledger - .filter((event) => event.type === "proposal-disposition") - .find((event) => event.proposalId === "proposal-applied"), - ).toMatchObject({ disposition: "failed", reasonCodes: ["writer-failed"] }); - expect(state.ledger.at(-1)).toMatchObject({ type: "job-failed", failedStage: "written" }); - }); - - it("produces identical artifacts for identical inputs and injected determinism", async () => { - const first = await recordingPipeline().pipeline.run(inputSnapshot()); - const second = await recordingPipeline().pipeline.run(inputSnapshot()); - - expect(first.corpus).toEqual(second.corpus); - expect(first.mutationBatch).toEqual(second.mutationBatch); - expect(first.job).toEqual(second.job); - }); - - it("keeps non-memory writers disabled by default and rejects duplicates", () => { - const memoryWriter: EvolutionTargetWriter = { - kind: "evolution-memory", - async write(batch) { - return { batchId: batch.batchId, status: "applied", resultingHarnessVersion: null, message: null }; - }, - }; - const registry = new EvolutionTargetAdapterRegistry({ writers: [memoryWriter] }); - - expect(registry.isEnabled("evolution-memory")).toBe(true); - expect(registry.isEnabled("skill")).toBe(false); - expect(() => registry.register(memoryWriter)).toThrow("already registered"); - expect(() => - registry.register({ - kind: "skill", - async write(batch) { - return { batchId: batch.batchId, status: "applied", resultingHarnessVersion: null, message: null }; - }, - }), - ).toThrow('Evolution target "skill" is not enabled'); - }); - - it("does not import product-specific runtime packages", async () => { - const sourceDirectory = join(dirname(fileURLToPath(import.meta.url)), "../src"); - const sourceFiles = ["contracts.ts", "index.ts", "pipeline.ts", "ports.ts", "provenance.ts", "targets.ts"]; - const source = (await Promise.all(sourceFiles.map((file) => readFile(join(sourceDirectory, file), "utf8")))).join( - "\n", - ); - - expect(source).not.toMatch(/from ["'][^"']*(?:coding-agent|diffex-tui|agent-core|session|extension)[^"']*["']/); - }); -}); From 82e9b7bc8850160df8bb6e393d0f80a2e242ccfa Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Wed, 19 Aug 2026 16:14:06 -0700 Subject: [PATCH 2/2] feat(agent): cover evolution recovery - Proves restart safety at every executable job stage - Covers fencing, cursor commits, triggers, retention, and worker priority --- packages/evolution/test/control-plane.test.ts | 529 ++++++++++++++++++ .../evolution/test/filesystem-store.test.ts | 199 +++++++ packages/evolution/test/worker.test.ts | 67 +++ 3 files changed, 795 insertions(+) create mode 100644 packages/evolution/test/control-plane.test.ts create mode 100644 packages/evolution/test/filesystem-store.test.ts create mode 100644 packages/evolution/test/worker.test.ts diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts new file mode 100644 index 000000000..4b08e0551 --- /dev/null +++ b/packages/evolution/test/control-plane.test.ts @@ -0,0 +1,529 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import { afterEach, describe, expect, it } from "vitest"; +import { + AUTOMATIC_EVOLUTION_COOLDOWN_MS, + AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD, + computeEvolutionContentDigest, + createEvolutionArtifactMetadata, + EVOLUTION_TOKEN_ESTIMATOR_VERSION, + type EvolutionArtifactProducer, + EvolutionControlPlane, + type EvolutionEventSink, + type EvolutionJobStage, + type EvolutionMemoryProposal, + EvolutionPipeline, + type EvolutionProgressEvent, + type EvolutionScope, + type EvolutionSnapshotContext, + type EvolutionStageExecutionContext, + type EvolutionStageExecutionResult, + type EvolutionStageExecutor, + EvolutionTargetAdapterRegistry, + type EvolutionTargetWriter, + evaluateAutomaticEvolutionEligibility, + FileEvolutionPersistence, + type Lesson, + type ReflectionNode, + type ScoredProposal, + type SuccessfulMaterialization, + type ValidationReport, +} from "../src/index.ts"; + +const FIRST_HARNESS_VERSION = `sha256:${"1".repeat(64)}`; +const SECOND_HARNESS_VERSION = `sha256:${"2".repeat(64)}`; +const MODEL_CONFIGURATION_DIGEST = `sha256:${"3".repeat(64)}`; +const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +class SequenceIds { + #next = 1; + + nextId(kind: string): string { + const value = `${kind}-${this.#next}`; + this.#next += 1; + return value; + } +} + +class AdvancingClock { + #now = Date.now(); + + now(): string { + const value = new Date(this.#now).toISOString(); + this.#now += 1; + return value; + } + + advance(milliseconds: number): void { + this.#now += milliseconds; + } +} + +function invocation(traceId: string): MaterializedInvocation { + return { + version: 1, + tracePath: `/traces/${traceId}.jsonl`, + traceId, + producer: null, + integrity: "complete", + terminationStatus: "completed", + health: null, + ordering: { + criteria: ["trace_sequence", "session_cursor", "timestamp", "span_parentage", "causal_id"], + ambiguity: [], + }, + evidence: [], + diagnostics: [], + }; +} + +function materialization(sequence: number): SuccessfulMaterialization { + const materialized = invocation(`trace-${sequence}`); + return { + materializationId: `materialization-${sequence}`, + ingestionCursor: sequence, + origin: "product-runtime", + contentDigest: computeEvolutionContentDigest(materialized), + invocation: materialized, + }; +} + +function snapshotContext(): EvolutionSnapshotContext { + const memory = "# Evolution memory\n"; + const instruction = "Preserve exact provenance.\n"; + return { + repository: { repositoryId: "diffex", rootPath: "/repo", revision: "abc123" }, + harness: { + harnessVersion: FIRST_HARNESS_VERSION, + baseVersion: null, + memory, + memoryContentDigest: computeEvolutionContentDigest(memory), + }, + instructions: [ + { + path: "/repo/AGENTS.md", + precedence: 1, + content: instruction, + contentDigest: computeEvolutionContentDigest(instruction), + }, + ], + modelConfiguration: { + provider: "faux", + model: "fixture", + configurationDigest: MODEL_CONFIGURATION_DIGEST, + }, + pipelineVersion: "pipeline-v1", + promptVersions: { + reflector: "reflector-v1", + curator: "curator-v1", + validator: "validator-v1", + writer: "writer-v1", + }, + }; +} + +const EXECUTABLE_STAGES = [ + "snapshotting", + "counter-slot", + "reflecting", + "curating", + "validating", + "writing", + "activating", +] as const satisfies readonly EvolutionJobStage[]; + +class RecordingExecutor implements EvolutionStageExecutor { + readonly calls = new Map(); + readonly logicalEffects = new Set(); + readonly #crashStage: EvolutionJobStage | null; + readonly #noOp: boolean; + #crashed = false; + + constructor(options: { readonly crashStage?: EvolutionJobStage; readonly noOp?: boolean } = {}) { + this.#crashStage = options.crashStage ?? null; + this.#noOp = options.noOp ?? false; + } + + async execute(context: EvolutionStageExecutionContext): Promise { + const calls = this.calls.get(context.job.stage) ?? []; + calls.push(context.idempotencyKey); + this.calls.set(context.job.stage, calls); + this.logicalEffects.add(context.idempotencyKey); + if (context.job.stage === "writing" || context.job.stage === "activating") { + expect(context.writerLease).not.toBeNull(); + } + if (context.job.stage === this.#crashStage && !this.#crashed) { + this.#crashed = true; + throw new Error(`simulated crash during ${context.job.stage}`); + } + if (context.job.stage === "validating" && this.#noOp) { + return { artifacts: [], state: { acceptedProposalIds: [] }, directive: "complete-no-op" }; + } + if (context.job.stage === "activating") { + return { + artifacts: [], + state: { activation: "fixture" }, + directive: "complete-activated", + resultingHarnessVersion: SECOND_HARNESS_VERSION, + }; + } + return { artifacts: [], state: { stage: context.job.stage }, directive: "continue" }; + } +} + +interface ControlPlaneFixture { + readonly clock: AdvancingClock; + readonly controlPlane: EvolutionControlPlane; + readonly events: EvolutionProgressEvent[]; + readonly executor: TExecutor; + readonly persistence: FileEvolutionPersistence; +} + +async function controlPlaneFixture( + executor: TExecutor, + materializations: readonly SuccessfulMaterialization[] = [ + materialization(1), + materialization(2), + materialization(3), + ], +): Promise> { + const root = await mkdtemp(join(tmpdir(), "diffex-evolution-control-")); + temporaryDirectories.push(root); + const persistence = new FileEvolutionPersistence(root); + const clock = new AdvancingClock(); + const events: EvolutionProgressEvent[] = []; + const eventSink: EvolutionEventSink = { + emit(event) { + events.push(event); + }, + }; + const controlPlane = new EvolutionControlPlane( + { + version: "1.0.0", + clock, + ids: new SequenceIds(), + artifacts: persistence, + jobs: persistence, + cursors: persistence, + leases: persistence, + ledger: persistence, + events: eventSink, + evidence: { + async highWaterCursor() { + return materializations.at(-1)?.ingestionCursor ?? 0; + }, + async listSuccessful(_scope, afterCursor, throughCursor) { + return materializations.filter( + (item) => item.ingestionCursor > afterCursor && item.ingestionCursor <= throughCursor, + ); + }, + }, + snapshots: { + async capture() { + return snapshotContext(); + }, + }, + executor, + }, + { leaseTtlMs: 5_000, heartbeatIntervalMs: 1_000, retryBaseDelayMs: 0, retryMaxDelayMs: 0 }, + ); + return { clock, controlPlane, events, executor, persistence }; +} + +async function createManualJob(fixture: ControlPlaneFixture) { + const result = await fixture.controlPlane.createJob({ scope, trigger: "manual" }); + expect(result.status).toBe("created"); + if (result.status !== "created") throw new Error("Expected a created fixture job"); + return result.job; +} + +describe("durable evolution control plane", () => { + it.each(EXECUTABLE_STAGES)("resumes %s with stable idempotency and one completion", async (crashStage) => { + const fixture = await controlPlaneFixture(new RecordingExecutor({ crashStage })); + const created = await createManualJob(fixture); + + await expect(fixture.controlPlane.runJob(created.jobId)).rejects.toThrow(`simulated crash during ${crashStage}`); + const failed = await fixture.persistence.load(created.jobId); + expect(failed).toMatchObject({ outcome: "retryable-failed", stage: crashStage }); + + fixture.clock.advance(1); + const completed = await fixture.controlPlane.runJob(created.jobId); + expect(completed.outcome).toBe("completed"); + expect(completed.completion).toEqual({ + kind: "activated", + resultingHarnessVersion: SECOND_HARNESS_VERSION, + processedEvidenceCursor: 3, + }); + expect(completed.checkpoints.map((checkpoint) => checkpoint.stage)).toEqual(EXECUTABLE_STAGES); + expect(new Set(fixture.executor.calls.get(crashStage)).size).toBe(1); + expect(fixture.executor.calls.get(crashStage)).toHaveLength(2); + expect(fixture.executor.logicalEffects.size).toBe(EXECUTABLE_STAGES.length); + expect(await fixture.persistence.load(scope)).toMatchObject({ sequence: 3, committedByJobId: created.jobId }); + expect( + (await fixture.persistence.read(created.jobId)).filter((event) => event.type === "job-completed"), + ).toHaveLength(1); + }); + + it("advances the cursor for a validated no-op without writing or activating", async () => { + const fixture = await controlPlaneFixture(new RecordingExecutor({ noOp: true })); + const created = await createManualJob(fixture); + + const completed = await fixture.controlPlane.runJob(created.jobId); + + expect(completed.completion).toEqual({ + kind: "no-op", + resultingHarnessVersion: null, + processedEvidenceCursor: 3, + }); + expect(completed.checkpoints.map((checkpoint) => checkpoint.stage)).toEqual(EXECUTABLE_STAGES.slice(0, 5)); + expect(fixture.executor.calls.has("writing")).toBe(false); + expect(fixture.executor.calls.has("activating")).toBe(false); + }); + + it("blocks overlapping jobs until the frozen range is explicitly superseded", async () => { + const fixture = await controlPlaneFixture(new RecordingExecutor()); + const first = await createManualJob(fixture); + + await expect(fixture.controlPlane.createJob({ scope, trigger: "manual" })).resolves.toEqual({ + status: "not-created", + reason: "blocking-job", + blockingJobId: first.jobId, + }); + await fixture.controlPlane.supersedeJob(first.jobId); + const replacement = await fixture.controlPlane.createJob({ scope, trigger: "manual" }); + expect(replacement.status).toBe("created"); + }); + + it("reports no evidence instead of creating an empty manual job", async () => { + const fixture = await controlPlaneFixture(new RecordingExecutor(), []); + + await expect(fixture.controlPlane.createJob({ scope, trigger: "manual" })).resolves.toEqual({ + status: "not-created", + reason: "no-new-evidence", + }); + expect(await fixture.persistence.list(scope)).toEqual([]); + }); + + it("records the deterministic estimator and rejects a small automatic corpus", async () => { + const fixture = await controlPlaneFixture(new RecordingExecutor()); + await expect(fixture.controlPlane.createJob({ scope, trigger: "automatic" })).resolves.toEqual({ + status: "not-created", + reason: "insufficient-tokens", + }); + + const created = await createManualJob(fixture); + const snapshot = await fixture.persistence.get(created.inputSnapshot); + expect(snapshot).toMatchObject({ + tokenMeasurement: { + estimatorVersion: EVOLUTION_TOKEN_ESTIMATOR_VERSION, + estimatedTokens: expect.any(Number), + }, + }); + }); +}); + +describe("checkpointed evolution pipeline", () => { + it("executes isolated stages through writing and activation", async () => { + const artifactProducer: EvolutionArtifactProducer = { + name: "fixture-stage", + version: "1.0.0", + promptVersion: "fixture-v1", + model: { + provider: "faux", + model: "fixture", + configurationDigest: MODEL_CONFIGURATION_DIGEST, + }, + }; + const metadata = (artifactId: string, sourceIds: readonly string[], content: unknown) => + createEvolutionArtifactMetadata({ + artifactId, + producer: artifactProducer, + sourceIds, + scope, + createdAt: "2026-08-19T12:00:00.000Z", + content, + }); + const modelKeys: string[] = []; + const writeKeys = new Set(); + const activationKeys = new Set(); + const target: EvolutionTargetWriter = { + kind: "evolution-memory", + async write(batch, context) { + writeKeys.add(context.idempotencyKey); + return { + batchId: batch.batchId, + status: "written", + output: { candidateDigest: computeEvolutionContentDigest(batch) }, + message: null, + }; + }, + async activate(batch, _writeResult, context) { + activationKeys.add(context.idempotencyKey); + return { + batchId: batch.batchId, + status: "activated", + resultingHarnessVersion: SECOND_HARNESS_VERSION, + message: null, + }; + }, + }; + const pipeline = new EvolutionPipeline({ + version: "1.0.0", + model: { + async execute(request) { + modelKeys.push(request.idempotencyKey); + return { output: { accepted: true }, model: { provider: "faux", model: "fixture" } }; + }, + }, + reflector: { + async reflect(input) { + await input.model.execute({ + stage: "reflector", + promptVersion: "reflector-v1", + idempotencyKey: input.idempotencyKey, + input: {}, + signal: input.signal, + }); + const lesson: Lesson = { + metadata: metadata("lesson-1", ["materialization-1"], { summary: "Keep narrow changes" }), + lessonId: "lesson-1", + kind: "strategy", + summary: "Keep narrow changes", + evidenceIds: [], + counterevidenceIds: [], + scope, + uncertainty: null, + }; + const node: ReflectionNode = { + metadata: metadata("reflection-1", [lesson.lessonId], { lessonIds: [lesson.lessonId] }), + nodeId: "reflection-1", + parentNodeId: null, + childNodeIds: [], + evidenceIds: [], + lessonIds: [lesson.lessonId], + status: "complete", + }; + return { nodes: [node], lessons: [lesson] }; + }, + }, + curator: { + async curate(input) { + await input.model.execute({ + stage: "curator", + promptVersion: "curator-v1", + idempotencyKey: input.idempotencyKey, + input: {}, + signal: input.signal, + }); + const proposal: EvolutionMemoryProposal = { + metadata: metadata("proposal-1", ["lesson-1"], { content: "Append one focused rule" }), + proposalId: "proposal-1", + summary: "Append one focused rule", + lessonIds: ["lesson-1"], + evidenceIds: [], + disposition: "actionable", + target: { kind: "evolution-memory", expectedHarnessVersion: FIRST_HARNESS_VERSION }, + executablePayload: { content: "Append one focused rule" }, + }; + return [proposal]; + }, + }, + validator: { + async validate(input) { + const proposalResults = input.proposals.map((proposal) => ({ + proposalId: proposal.proposalId, + disposition: "accepted" as const, + issues: [], + })); + const report: ValidationReport = { + metadata: metadata( + "validation-1", + input.proposals.map((proposal) => proposal.proposalId), + { + proposalResults, + }, + ), + proposalResults, + }; + return report; + }, + }, + scorer: { + async score(input) { + return input.proposals.map( + (proposal): ScoredProposal => ({ + metadata: metadata(`score-${proposal.proposalId}`, [proposal.proposalId], { + disposition: "selected", + }), + proposal, + score: 100, + disposition: "selected", + reasonCodes: ["selected"], + }), + ); + }, + }, + targets: new EvolutionTargetAdapterRegistry({ writers: [target] }), + }); + const fixture = await controlPlaneFixture(pipeline); + const created = await createManualJob(fixture); + + const completed = await fixture.controlPlane.runJob(created.jobId); + + expect(completed.outcome).toBe("completed"); + expect(modelKeys).toHaveLength(2); + expect(new Set(modelKeys).size).toBe(2); + expect(writeKeys.size).toBe(1); + expect(activationKeys.size).toBe(1); + expect( + (await fixture.persistence.read(created.jobId)) + .filter((event) => event.type === "proposal-disposition") + .map((event) => [event.proposalId, event.disposition]), + ).toEqual([["proposal-1", "applied"]]); + }); +}); + +describe("automatic evolution policy", () => { + it("requires a strict token threshold and at least three invocations", () => { + expect( + evaluateAutomaticEvolutionEligibility({ + invocationCount: 3, + estimatedTokens: AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD, + now: "2026-08-19T12:00:00.000Z", + previousAutomaticTerminalAt: null, + }), + ).toEqual({ eligible: false, reason: "insufficient-tokens" }); + expect( + evaluateAutomaticEvolutionEligibility({ + invocationCount: 2, + estimatedTokens: AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD + 1, + now: "2026-08-19T12:00:00.000Z", + previousAutomaticTerminalAt: null, + }), + ).toEqual({ eligible: false, reason: "insufficient-invocations" }); + }); + + it("enforces the cooldown until the full interval has elapsed", () => { + const previous = "2026-08-19T12:00:00.000Z"; + const beforeCooldown = new Date(Date.parse(previous) + AUTOMATIC_EVOLUTION_COOLDOWN_MS - 1).toISOString(); + const atCooldown = new Date(Date.parse(previous) + AUTOMATIC_EVOLUTION_COOLDOWN_MS).toISOString(); + const input = { + invocationCount: 3, + estimatedTokens: AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD + 1, + previousAutomaticTerminalAt: previous, + }; + + expect(evaluateAutomaticEvolutionEligibility({ ...input, now: beforeCooldown })).toEqual({ + eligible: false, + reason: "cooldown-active", + }); + expect(evaluateAutomaticEvolutionEligibility({ ...input, now: atCooldown })).toEqual({ eligible: true }); + }); +}); diff --git a/packages/evolution/test/filesystem-store.test.ts b/packages/evolution/test/filesystem-store.test.ts new file mode 100644 index 000000000..202eb89ab --- /dev/null +++ b/packages/evolution/test/filesystem-store.test.ts @@ -0,0 +1,199 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + computeEvolutionContentDigest, + createEvolutionArtifactMetadata, + type EvolutionArtifactProducer, + type EvolutionJob, + type EvolutionLeaseToken, + type EvolutionScope, + EvolutionStorageCorruptionError, + FileEvolutionPersistence, + type InstructionSnapshot, + planEvolutionRetention, +} from "../src/index.ts"; + +const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; +const producer: EvolutionArtifactProducer = { + name: "fixture", + version: "1.0.0", + promptVersion: null, + model: null, +}; +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +async function storeFixture(): Promise<{ readonly root: string; readonly store: FileEvolutionPersistence }> { + const root = await mkdtemp(join(tmpdir(), "diffex-evolution-store-")); + temporaryDirectories.push(root); + return { root, store: new FileEvolutionPersistence(root) }; +} + +function instructionSnapshot(): InstructionSnapshot { + const documents = [ + { + path: "/repo/AGENTS.md", + precedence: 1, + content: "Keep durable state fenced.\n", + contentDigest: computeEvolutionContentDigest("Keep durable state fenced.\n"), + }, + ]; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: "instructions-1", + producer, + sourceIds: [], + scope, + createdAt: "2026-08-19T12:00:00.000Z", + content: documents, + }), + documents, + }; +} + +function job(reference: EvolutionJob["inputSnapshot"]): EvolutionJob { + const createdAt = new Date().toISOString(); + const content: Omit = { + jobId: "job-1", + inputSnapshot: reference, + trigger: "manual", + evidenceRange: { afterCursor: 0, throughCursor: 3 }, + stage: "queued", + outcome: "active", + reasonCode: null, + failureMessage: null, + completion: null, + resolution: null, + checkpoints: [], + attempt: 0, + revision: 0, + createdAt, + updatedAt: createdAt, + terminalAt: null, + retryNotBefore: null, + }; + return { + ...content, + metadata: createEvolutionArtifactMetadata({ + artifactId: content.jobId, + producer, + sourceIds: [reference.artifactId], + scope, + createdAt, + content, + }), + }; +} + +async function scopeLease(store: FileEvolutionPersistence, ownerId: string): Promise { + const lease = await store.acquire( + "scope", + computeEvolutionContentDigest(scope), + ownerId, + new Date().toISOString(), + 5_000, + ); + if (lease === null) throw new Error("Expected fixture scope lease"); + return lease; +} + +describe("filesystem evolution persistence", () => { + it("stores immutable artifacts by content digest and detects corruption", async () => { + const { root, store } = await storeFixture(); + const artifact = instructionSnapshot(); + + const first = await store.put(artifact); + const second = await store.put(artifact); + expect(second).toEqual(first); + expect(await store.get(first)).toEqual(artifact); + + const artifactPath = join(root, "artifacts", first.contentDigest.replace("sha256:", "sha256-"), "artifact.json"); + await writeFile(artifactPath, "{}\n", "utf8"); + await expect(store.get(first)).rejects.toBeInstanceOf(EvolutionStorageCorruptionError); + }); + + it("fences stale scope owners from job and cursor mutations", async () => { + const { store } = await storeFixture(); + const artifactReference = await store.put(instructionSnapshot()); + const firstLease = await scopeLease(store, "owner-1"); + await store.create(job(artifactReference), firstLease); + + const takeoverTime = new Date(Date.parse(firstLease.expiresAt) + 1).toISOString(); + const secondLease = await store.acquire( + "scope", + computeEvolutionContentDigest(scope), + "owner-2", + takeoverTime, + 5_000, + ); + if (secondLease === null) throw new Error("Expected stale lease takeover"); + expect(secondLease.fence).toBe(firstLease.fence + 1); + await expect(store.commit(scope, 0, 3, "job-1", new Date().toISOString(), firstLease)).rejects.toThrow( + "lease was lost", + ); + expect(await store.commit(scope, 0, 3, "job-1", new Date().toISOString(), secondLease)).toBe("committed"); + expect(await store.commit(scope, 0, 3, "job-1", new Date().toISOString(), secondLease)).toBe("already-committed"); + }); + + it("uses compare-and-swap revisions for durable job manifests", async () => { + const { store } = await storeFixture(); + const reference = await store.put(instructionSnapshot()); + const lease = await scopeLease(store, "owner-1"); + const initial = job(reference); + await store.create(initial, lease); + const { metadata: _metadata, ...initialContent } = initial; + const advancedContent: Omit = { + ...initialContent, + stage: "snapshotting", + revision: 1, + updatedAt: new Date().toISOString(), + }; + const advanced: EvolutionJob = { + ...advancedContent, + metadata: createEvolutionArtifactMetadata({ + artifactId: advancedContent.jobId, + producer, + sourceIds: [reference.artifactId], + scope, + createdAt: advancedContent.createdAt, + content: advancedContent, + }), + }; + + await store.save(advanced, 0, lease); + await expect(store.save(advanced, 0, lease)).rejects.toThrow("revision conflict"); + expect(await store.load(initial.jobId)).toMatchObject({ stage: "snapshotting", revision: 1 }); + }); + + it("retains unresolved failures while expiring completed audit state", () => { + const reference = { artifactId: "input-1", contentDigest: `sha256:${"a".repeat(64)}` }; + const initial = job(reference); + const terminalAt = "2026-01-01T00:00:00.000Z"; + const completed: EvolutionJob = { + ...initial, + jobId: "job-completed", + stage: "completed", + outcome: "completed", + terminalAt, + completion: { kind: "no-op", resultingHarnessVersion: null, processedEvidenceCursor: 3 }, + }; + const failed: EvolutionJob = { + ...initial, + jobId: "job-failed", + outcome: "terminal-failed", + reasonCode: "artifact-corrupt", + failureMessage: "corrupt artifact", + terminalAt, + }; + + expect(planEvolutionRetention([completed, failed], "2026-03-01T00:00:00.000Z")).toEqual({ + removableJobIds: ["job-completed"], + retainedBlockingJobIds: ["job-failed"], + }); + }); +}); diff --git a/packages/evolution/test/worker.test.ts b/packages/evolution/test/worker.test.ts new file mode 100644 index 000000000..ef72a43a4 --- /dev/null +++ b/packages/evolution/test/worker.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { EvolutionBackgroundWorker, type EvolutionWorkSource } from "../src/index.ts"; + +function waitForAbort(signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); +} + +describe("evolution background worker", () => { + it("admits durable work only after the interactive priority gate", async () => { + const order: string[] = []; + const worker = new EvolutionBackgroundWorker({ + clock: { now: () => "2026-08-19T12:00:00.000Z" }, + source: { + async nextRunnableJob() { + order.push("dequeue"); + return "job-1"; + }, + async waitForWork() {}, + }, + runner: { + async runJob() { + order.push("run"); + return {} as never; + }, + }, + priority: { + async waitUntilBackgroundAllowed() { + order.push("priority"); + }, + }, + }); + + expect(await worker.runOnce(new AbortController().signal)).toBe("ran"); + expect(order).toEqual(["dequeue", "priority", "run"]); + }); + + it("cancels an independently running idle worker during shutdown", async () => { + let waiting = false; + const source: EvolutionWorkSource = { + async nextRunnableJob() { + return null; + }, + async waitForWork(signal) { + waiting = true; + await waitForAbort(signal); + }, + }; + const worker = new EvolutionBackgroundWorker({ + clock: { now: () => "2026-08-19T12:00:00.000Z" }, + source, + runner: { + async runJob() { + throw new Error("No job should run"); + }, + }, + priority: { async waitUntilBackgroundAllowed() {} }, + }); + + worker.start(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(waiting).toBe(true); + await expect(worker.stop()).resolves.toBeUndefined(); + }); +});