diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index f8b1785a9..ac409dfec 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -539,9 +539,31 @@ export interface MutationBatch { readonly metadata: EvolutionArtifactMetadata; readonly batchId: string; readonly targetKind: "evolution-memory"; + readonly expectedBaseVersion: HarnessVersion; + readonly expectedBaseDigest: string; + readonly baseMemoryItems: readonly EvolutionMemoryItem[]; + readonly activationEventId: string; readonly mutations: readonly EvolutionMutation[]; } +export type EvolutionMemoryExactDiff = JsonObject & { + readonly start: number; + readonly removed: string; + readonly added: string; +}; + +export type EvolutionMemoryWriteOutput = JsonObject & { + readonly candidateId: string; + readonly candidateDigest: string; + readonly batchHash: string; + readonly expectedBaseVersion: HarnessVersion; + readonly expectedBaseDigest: string; + readonly resultingHarnessVersion: HarnessVersion; + readonly resultingMemoryDigest: string; + readonly operationIds: readonly string[]; + readonly diff: EvolutionMemoryExactDiff; +}; + export interface MutationWriteResult { readonly batchId: string; readonly status: "failed" | "written"; @@ -714,6 +736,19 @@ export interface MutationWrittenLedgerEvent extends EvolutionLedgerEventBase { readonly writeResult: MutationWriteResult; } +export interface HarnessActivatedLedgerEvent extends EvolutionLedgerEventBase { + readonly type: "harness-activated"; + readonly batchId: string; + readonly proposalIds: readonly string[]; + readonly evidenceIds: readonly string[]; + readonly operationIds: readonly string[]; + readonly exactDiff: EvolutionMemoryExactDiff; + readonly parentHarnessVersion: HarnessVersion; + readonly resultingHarnessVersion: HarnessVersion; + readonly activationResult: HarnessActivationResult; + readonly rollbackParentHarnessVersion: HarnessVersion; +} + export interface JobCompletedLedgerEvent extends EvolutionLedgerEventBase { readonly type: "job-completed"; readonly completionKind: EvolutionCompletionKind; @@ -779,6 +814,7 @@ export type EvolutionLedgerEvent = | CheckpointSavedLedgerEvent | ProposalDispositionLedgerEvent | MutationWrittenLedgerEvent + | HarnessActivatedLedgerEvent | JobCompletedLedgerEvent | JobFailedLedgerEvent | ProposalRolledBackLedgerEvent diff --git a/packages/evolution/src/evolution-memory-writer.ts b/packages/evolution/src/evolution-memory-writer.ts new file mode 100644 index 000000000..3544d6ca8 --- /dev/null +++ b/packages/evolution/src/evolution-memory-writer.ts @@ -0,0 +1,837 @@ +import { createHash, randomUUID } from "node:crypto"; +import { link, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; +import { + computeHarnessVersion, + type HarnessStateStore, + type HarnessVersion, + type HarnessVersionBundle, +} from "@earendil-works/diffex-harness-state"; +import type { + EvolutionMemoryAddDelta, + EvolutionMemoryExactDiff, + EvolutionMemoryItem, + EvolutionMemoryWriteOutput, + EvolutionMutation, + HarnessActivationResult, + ImmutableHarnessSnapshot, + MutationBatch, + MutationWriteResult, +} from "./contracts.ts"; +import { EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION } from "./contracts.ts"; +import { EvolutionExecutionError } from "./errors.ts"; +import { computeEvolutionContentDigest, serializeEvolutionValue } from "./provenance.ts"; +import type { EvolutionTargetWriter } from "./targets.ts"; + +const CATALOG_SCHEMA_VERSION = 1 as const; +const CANDIDATE_SCHEMA_VERSION = 1 as const; +const DEFAULT_MAX_MEMORY_BYTES = 64 * 1024; +const VERSION_PATTERN = /^sha256:[0-9a-f]{64}$/; +const SAFE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/; +const SECRET_PATTERNS = [ + /-----BEGIN (?:OPENSSH |RSA |EC |DSA )?PRIVATE KEY-----/, + /\bAKIA[0-9A-Z]{16}\b/, + /\bgh[pousr]_[A-Za-z0-9]{30,}\b/, + /\bxox[baprs]-[A-Za-z0-9-]{20,}\b/, + /\b(?:api[_ -]?key|password|secret|token)\s*[:=]\s*["']?[A-Za-z0-9_+./=-]{16,}/i, +] as const; + +interface MarkdownLine { + readonly start: number; + readonly end: number; + readonly contentEnd: number; + readonly text: string; +} + +interface MarkdownHeading { + readonly level: number; + readonly title: string; + readonly start: number; + readonly end: number; +} + +interface MarkdownMemoryBlock { + readonly start: number; + readonly end: number; + readonly content: string; + readonly section: string | null; +} + +interface ParsedEvolutionMemory { + readonly headings: readonly MarkdownHeading[]; + readonly blocks: readonly MarkdownMemoryBlock[]; +} + +interface StoredEvolutionMemoryCatalog { + readonly schemaVersion: typeof CATALOG_SCHEMA_VERSION; + readonly harnessVersion: HarnessVersion; + readonly baseVersion: HarnessVersion | null; + readonly memoryContentDigest: string; + readonly items: readonly EvolutionMemoryItem[]; + readonly activeOrder: readonly string[]; +} + +interface StoredEvolutionMemoryCandidate { + readonly schemaVersion: typeof CANDIDATE_SCHEMA_VERSION; + readonly writerVersion: string; + readonly candidateId: string; + readonly candidateDigest: string; + readonly batchId: string; + readonly batchHash: string; + readonly expectedBaseVersion: HarnessVersion; + readonly expectedBaseDigest: string; + readonly resultingHarnessVersion: HarnessVersion; + readonly resultingMemoryDigest: string; + readonly operationIds: readonly string[]; + readonly beforeMemory: string; + readonly afterMemory: string; + readonly diff: EvolutionMemoryExactDiff; + readonly catalog: StoredEvolutionMemoryCatalog; +} + +export interface EvolutionMemoryWriterOptions { + readonly version: string; + readonly harnessState: HarnessStateStore; + readonly statePath: string; + readonly maxMemoryBytes?: number; +} + +function hasErrorCode(error: unknown, code: string): boolean { + return error instanceof Error && "code" in error && error.code === code; +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizedText(value: string): string { + return value.normalize("NFKC").trim().replace(/\s+/g, " ").toLocaleLowerCase("en-US"); +} + +function markdownLines(memory: string): readonly MarkdownLine[] { + const lines: MarkdownLine[] = []; + let start = 0; + while (start < memory.length) { + const newline = memory.indexOf("\n", start); + const end = newline === -1 ? memory.length : newline + 1; + let contentEnd = newline === -1 ? memory.length : newline; + if (contentEnd > start && memory[contentEnd - 1] === "\r") contentEnd -= 1; + lines.push({ start, end, contentEnd, text: memory.slice(start, contentEnd) }); + start = end; + } + return lines; +} + +function headingFromLine(line: MarkdownLine): MarkdownHeading | null { + const match = /^(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$/.exec(line.text); + if (match === null) return null; + return { + level: match[1]?.length ?? 0, + title: match[2] ?? "", + start: line.start, + end: line.end, + }; +} + +function isTopLevelListItem(line: MarkdownLine): boolean { + return /^(?: {0,3})(?:[-+*]|\d+[.)])[ \t]+\S/.test(line.text); +} + +function isIndented(line: MarkdownLine): boolean { + return /^(?: {2,}|\t)/.test(line.text); +} + +function fenceMarker(line: MarkdownLine): string | null { + const match = /^(?: {0,3})(`{3,}|~{3,})/.exec(line.text); + return match?.[1] ?? null; +} + +function parseEvolutionMemory(memory: string): ParsedEvolutionMemory { + if (memory.startsWith("\uFEFF")) throw new Error("EVOLVE.md must not start with a UTF-8 BOM"); + if (memory.includes("\u0000")) throw new Error("EVOLVE.md must not contain NUL bytes"); + const lines = markdownLines(memory); + const headings: MarkdownHeading[] = []; + const blocks: MarkdownMemoryBlock[] = []; + let section: string | null = null; + let index = 0; + while (index < lines.length) { + const line = lines[index] as MarkdownLine; + if (line.text.trim() === "") { + index += 1; + continue; + } + const heading = headingFromLine(line); + if (heading !== null) { + headings.push(heading); + section = heading.title; + index += 1; + continue; + } + + const start = line.start; + let lastContentLine = line; + const openingFence = fenceMarker(line); + if (openingFence !== null) { + const fenceCharacter = openingFence[0] as string; + const minimumLength = openingFence.length; + let closed = false; + index += 1; + while (index < lines.length) { + const candidate = lines[index] as MarkdownLine; + lastContentLine = candidate; + index += 1; + const closing = fenceMarker(candidate); + if ( + closing !== null && + closing[0] === fenceCharacter && + closing.length >= minimumLength && + candidate.text.trim().replaceAll(fenceCharacter, "") === "" + ) { + closed = true; + break; + } + } + if (!closed) throw new Error("EVOLVE.md contains an unclosed fenced code block"); + } else if (isTopLevelListItem(line)) { + index += 1; + while (index < lines.length) { + const candidate = lines[index] as MarkdownLine; + if (headingFromLine(candidate) !== null || isTopLevelListItem(candidate)) break; + if (candidate.text.trim() === "") { + const next = lines[index + 1]; + if ( + next === undefined || + headingFromLine(next) !== null || + isTopLevelListItem(next) || + (next.text.trim() !== "" && !isIndented(next)) + ) { + break; + } + index += 1; + continue; + } + lastContentLine = candidate; + index += 1; + } + } else { + index += 1; + while (index < lines.length) { + const candidate = lines[index] as MarkdownLine; + if (candidate.text.trim() === "" || headingFromLine(candidate) !== null) break; + lastContentLine = candidate; + index += 1; + } + } + blocks.push({ + start, + end: lastContentLine.contentEnd, + content: memory.slice(start, lastContentLine.contentEnd), + section, + }); + } + return { headings, blocks }; +} + +function validateParsedMemory(memory: string, parsed: ParsedEvolutionMemory, maxMemoryBytes: number): void { + if (Buffer.byteLength(memory, "utf8") > maxMemoryBytes) { + throw new Error(`EVOLVE.md exceeds the ${maxMemoryBytes}-byte size limit`); + } + for (const pattern of SECRET_PATTERNS) { + if (pattern.test(memory)) throw new Error("EVOLVE.md candidate violates the sensitive-data policy"); + } + const normalizedItems = new Set(); + for (const block of parsed.blocks) { + const normalized = normalizedText(block.content); + if (normalized === "") throw new Error("EVOLVE.md contains an empty memory item"); + if (normalizedItems.has(normalized)) throw new Error("EVOLVE.md contains duplicate normalized memory items"); + normalizedItems.add(normalized); + } +} + +function exactDiff(before: string, after: string): EvolutionMemoryExactDiff { + let start = 0; + while (start < before.length && start < after.length && before[start] === after[start]) start += 1; + let beforeEnd = before.length; + let afterEnd = after.length; + while (beforeEnd > start && afterEnd > start && before[beforeEnd - 1] === after[afterEnd - 1]) { + beforeEnd -= 1; + afterEnd -= 1; + } + return { + start, + removed: before.slice(start, beforeEnd), + added: after.slice(start, afterEnd), + } as EvolutionMemoryExactDiff; +} + +function applyExactDiff(before: string, diff: EvolutionMemoryExactDiff): string { + if (before.slice(diff.start, diff.start + diff.removed.length) !== diff.removed) { + throw new Error("Stored EVOLVE.md diff does not match its base"); + } + return `${before.slice(0, diff.start)}${diff.added}${before.slice(diff.start + diff.removed.length)}`; +} + +function memoryArtifactContentHash(memory: string): HarnessVersion { + return `sha256:${createHash("sha256").update(memory, "utf8").digest("hex")}`; +} + +function resultingHarnessVersion(memory: string, base: HarnessVersion): HarnessVersion { + return computeHarnessVersion({ + schemaVersion: 1, + base, + artifacts: [{ kind: "evolution-memory", path: "EVOLVE.md", contentHash: memoryArtifactContentHash(memory) }], + }); +} + +function assertSafeId(value: string, description: string): void { + if (!SAFE_ID_PATTERN.test(value)) throw new Error(`Invalid ${description} "${value}"`); +} + +function assertUnique(values: readonly string[], description: string): void { + if (new Set(values).size !== values.length) throw new Error(`Duplicate ${description}`); +} + +function assertWriterContext(context: { readonly lease: { readonly kind: string; readonly key: string } }): void { + if (context.lease.kind !== "writer" || context.lease.key !== "global") { + throw new Error("Evolution memory mutations require the global writer lease"); + } +} + +function itemDigest(content: string): string { + return computeEvolutionContentDigest(content); +} + +function itemById(items: readonly EvolutionMemoryItem[], memoryItemId: string): EvolutionMemoryItem { + const item = items.find((candidate) => candidate.memoryItemId === memoryItemId); + if (item === undefined) throw new Error(`Unknown evolution memory item "${memoryItemId}"`); + return item; +} + +function activeItems(catalog: StoredEvolutionMemoryCatalog): readonly EvolutionMemoryItem[] { + return catalog.activeOrder.map((memoryItemId) => itemById(catalog.items, memoryItemId)); +} + +function sameMemoryItems(left: readonly EvolutionMemoryItem[], right: readonly EvolutionMemoryItem[]): boolean { + return serializeEvolutionValue(left) === serializeEvolutionValue(right); +} + +function validateCatalog(catalog: StoredEvolutionMemoryCatalog, bundle: HarnessVersionBundle): ParsedEvolutionMemory { + if ( + catalog.schemaVersion !== CATALOG_SCHEMA_VERSION || + catalog.harnessVersion !== bundle.manifest.version || + catalog.baseVersion !== bundle.manifest.base || + catalog.memoryContentDigest !== computeEvolutionContentDigest(bundle.state.memory) + ) { + throw new Error(`Evolution memory catalog for ${bundle.manifest.version} does not match its harness revision`); + } + assertUnique( + catalog.items.map((item) => item.memoryItemId), + "memory item ID in catalog", + ); + assertUnique(catalog.activeOrder, "active memory item ID in catalog"); + const parsed = parseEvolutionMemory(bundle.state.memory); + const currentItems = activeItems(catalog); + if ( + currentItems.length !== parsed.blocks.length || + currentItems.some( + (item, index) => + item.supersessionRevision !== null || + item.contentDigest !== itemDigest(parsed.blocks[index]?.content ?? ""), + ) + ) { + throw new Error(`Evolution memory catalog for ${bundle.manifest.version} has a stale item index`); + } + return parsed; +} + +function outputForCandidate(candidate: StoredEvolutionMemoryCandidate): EvolutionMemoryWriteOutput { + return { + candidateId: candidate.candidateId, + candidateDigest: candidate.candidateDigest, + batchHash: candidate.batchHash, + expectedBaseVersion: candidate.expectedBaseVersion, + expectedBaseDigest: candidate.expectedBaseDigest, + resultingHarnessVersion: candidate.resultingHarnessVersion, + resultingMemoryDigest: candidate.resultingMemoryDigest, + operationIds: candidate.operationIds, + diff: candidate.diff, + }; +} + +function contentWithoutMetadata(batch: MutationBatch): Omit { + const { metadata: _metadata, ...content } = batch; + return content; +} + +function insertionOffset(memory: string, parsed: ParsedEvolutionMemory, delta: EvolutionMemoryAddDelta): number { + if (delta.destination.anchor !== null) { + const matchingBlocks = parsed.blocks.filter( + (block) => + block.content === delta.destination.anchor || + normalizedText(block.content) === normalizedText(delta.destination.anchor ?? ""), + ); + if (matchingBlocks.length !== 1) throw new Error("EVOLVE.md addition anchor must match exactly one memory item"); + const block = matchingBlocks[0] as MarkdownMemoryBlock; + if (delta.destination.section !== null && block.section !== delta.destination.section) { + throw new Error(`EVOLVE.md addition anchor is outside section "${delta.destination.section}"`); + } + return block.end; + } + if (delta.destination.section === null) return parsed.blocks.at(-1)?.end ?? memory.trimEnd().length; + const headings = parsed.headings.filter((heading) => heading.title === delta.destination.section); + if (headings.length !== 1) + throw new Error(`EVOLVE.md section "${delta.destination.section}" must exist exactly once`); + const sectionBlocks = parsed.blocks.filter((block) => block.section === delta.destination.section); + return sectionBlocks.at(-1)?.end ?? (headings[0] as MarkdownHeading).end; +} + +function insertMemory(memory: string, offset: number, content: string): string { + if (content.trim() !== content || content.includes("\r")) { + throw new Error("EVOLVE.md additions must not contain leading, trailing, or carriage-return whitespace"); + } + const before = memory.slice(0, offset); + const after = memory.slice(offset); + const prefix = before.length === 0 ? "" : before.endsWith("\n\n") ? "" : before.endsWith("\n") ? "\n" : "\n\n"; + const suffix = + after.length === 0 + ? "\n" + : after.trim() === "" || after.startsWith("\n\n") + ? "" + : after.startsWith("\n") + ? "\n" + : "\n\n"; + return `${before}${prefix}${content}${suffix}${after}`; +} + +function removeMemoryBlock(memory: string, block: MarkdownMemoryBlock): string { + let start = block.start; + let end = block.end; + while (end < memory.length && (memory[end] === "\n" || memory[end] === "\r")) end += 1; + if (end === block.end) { + while (start > 0 && (memory[start - 1] === "\n" || memory[start - 1] === "\r")) start -= 1; + } + return `${memory.slice(0, start)}${memory.slice(end)}`; +} + +function parseWriteOutput(result: MutationWriteResult): EvolutionMemoryWriteOutput { + const output = result.output; + if ( + result.status !== "written" || + !isRecord(output) || + typeof output.candidateId !== "string" || + typeof output.candidateDigest !== "string" || + typeof output.batchHash !== "string" || + typeof output.expectedBaseVersion !== "string" || + typeof output.expectedBaseDigest !== "string" || + typeof output.resultingHarnessVersion !== "string" || + typeof output.resultingMemoryDigest !== "string" || + !Array.isArray(output.operationIds) || + !isRecord(output.diff) + ) { + throw new Error("Evolution memory write receipt is invalid"); + } + return output as EvolutionMemoryWriteOutput; +} + +export class EvolutionMemoryWriter implements EvolutionTargetWriter { + readonly kind = "evolution-memory" as const; + readonly #version: string; + readonly #harnessState: HarnessStateStore; + readonly #statePath: string; + readonly #catalogPath: string; + readonly #candidatePath: string; + readonly #maxMemoryBytes: number; + + constructor(options: EvolutionMemoryWriterOptions) { + if (options.version.trim() === "") throw new Error("Evolution memory writer version must be non-empty"); + if ( + !Number.isSafeInteger(options.maxMemoryBytes ?? DEFAULT_MAX_MEMORY_BYTES) || + (options.maxMemoryBytes ?? DEFAULT_MAX_MEMORY_BYTES) < 1 + ) { + throw new Error("maxMemoryBytes must be a positive integer"); + } + this.#version = options.version; + this.#harnessState = options.harnessState; + this.#statePath = resolve(options.statePath); + this.#catalogPath = join(this.#statePath, "catalogs"); + this.#candidatePath = join(this.#statePath, "candidates"); + this.#maxMemoryBytes = options.maxMemoryBytes ?? DEFAULT_MAX_MEMORY_BYTES; + } + + async #ensureDirectories(): Promise { + await mkdir(this.#catalogPath, { recursive: true, mode: 0o700 }); + await mkdir(this.#candidatePath, { recursive: true, mode: 0o700 }); + } + + #catalogFile(version: HarnessVersion): string { + if (!VERSION_PATTERN.test(version)) throw new Error(`Invalid harness version "${version}"`); + return join(this.#catalogPath, `${version.replace("sha256:", "sha256-")}.json`); + } + + #candidateFile(batchId: string): string { + assertSafeId(batchId, "mutation batch ID"); + return join(this.#candidatePath, `${batchId}.json`); + } + + async #readOptional(path: string): Promise { + try { + return await readFile(path, "utf8"); + } catch (error) { + if (hasErrorCode(error, "ENOENT")) return null; + throw error; + } + } + + async #writeImmutable(path: string, value: unknown): Promise { + await this.#ensureDirectories(); + const serialized = `${serializeEvolutionValue(value)}\n`; + const existing = await this.#readOptional(path); + if (existing !== null) { + if (existing !== serialized) throw new Error(`Immutable evolution metadata at "${path}" changed`); + return; + } + const temporaryDirectory: string | null = await mkdtemp(join(dirname(path), `.${basename(path)}.tmp-`)); + try { + const temporaryFile = join(temporaryDirectory, basename(path)); + await writeFile(temporaryFile, serialized, { encoding: "utf8", flag: "wx", mode: 0o600 }); + try { + await link(temporaryFile, path); + } catch (error) { + if (!hasErrorCode(error, "EEXIST")) throw error; + const raced = await this.#readOptional(path); + if (raced !== serialized) throw new Error(`Immutable evolution metadata race at "${path}" changed`); + } + } finally { + if (temporaryDirectory !== null) await rm(temporaryDirectory, { recursive: true, force: true }); + } + } + + async #loadCatalog(bundle: HarnessVersionBundle): Promise { + await this.#ensureDirectories(); + const path = this.#catalogFile(bundle.manifest.version); + const contents = await this.#readOptional(path); + if (contents !== null) { + const parsed: unknown = JSON.parse(contents); + if (!isRecord(parsed) || !Array.isArray(parsed.items) || !Array.isArray(parsed.activeOrder)) { + throw new Error(`Evolution memory catalog at "${path}" is invalid`); + } + const catalog = parsed as unknown as StoredEvolutionMemoryCatalog; + validateCatalog(catalog, bundle); + return catalog; + } + const memory = parseEvolutionMemory(bundle.state.memory); + validateParsedMemory(bundle.state.memory, memory, this.#maxMemoryBytes); + const items = memory.blocks.map( + (block): EvolutionMemoryItem => ({ + schemaVersion: EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION, + memoryItemId: `memory-item-${randomUUID()}`, + contentDigest: itemDigest(block.content), + creationRevision: bundle.manifest.version, + supersessionRevision: null, + ledger: { origin: "manual", creationEventId: null, supersessionEventId: null }, + }), + ); + const catalog: StoredEvolutionMemoryCatalog = { + schemaVersion: CATALOG_SCHEMA_VERSION, + harnessVersion: bundle.manifest.version, + baseVersion: bundle.manifest.base, + memoryContentDigest: computeEvolutionContentDigest(bundle.state.memory), + items, + activeOrder: items.map((item) => item.memoryItemId), + }; + try { + await this.#writeImmutable(path, catalog); + return catalog; + } catch (error) { + const raced = await this.#readOptional(path); + if (raced === null) throw error; + const parsed = JSON.parse(raced) as StoredEvolutionMemoryCatalog; + validateCatalog(parsed, bundle); + return parsed; + } + } + + async #loadCandidate(batchId: string): Promise { + const contents = await this.#readOptional(this.#candidateFile(batchId)); + if (contents === null) return null; + const parsed: unknown = JSON.parse(contents); + if ( + !isRecord(parsed) || + parsed.schemaVersion !== CANDIDATE_SCHEMA_VERSION || + typeof parsed.writerVersion !== "string" || + parsed.batchId !== batchId || + typeof parsed.candidateDigest !== "string" || + !isRecord(parsed.catalog) + ) { + throw new Error(`Evolution memory candidate for "${batchId}" is invalid`); + } + const candidate = parsed as unknown as StoredEvolutionMemoryCandidate; + const { candidateDigest: _candidateDigest, ...content } = candidate; + if (candidate.candidateDigest !== computeEvolutionContentDigest(content)) { + throw new Error(`Evolution memory candidate for "${batchId}" has an invalid digest`); + } + if (applyExactDiff(candidate.beforeMemory, candidate.diff) !== candidate.afterMemory) { + throw new Error(`Evolution memory candidate for "${batchId}" has an invalid exact diff`); + } + return candidate; + } + + async captureSnapshot(): Promise { + const bundle = await this.#harnessState.loadCurrent(); + const catalog = await this.#loadCatalog(bundle); + return { + harnessVersion: bundle.manifest.version, + baseVersion: bundle.manifest.base, + memory: bundle.state.memory, + memoryContentDigest: catalog.memoryContentDigest, + activeMemoryItems: activeItems(catalog), + }; + } + + async write( + batch: MutationBatch, + context: Parameters[1], + ): Promise { + context.signal?.throwIfAborted(); + assertWriterContext(context); + if (batch.targetKind !== this.kind || batch.mutations.length === 0) { + throw new Error("Evolution memory writer requires a non-empty evolution-memory batch"); + } + assertSafeId(batch.batchId, "mutation batch ID"); + assertSafeId(batch.activationEventId, "activation ledger event ID"); + assertUnique( + batch.mutations.map((mutation) => mutation.mutationId), + "mutation operation ID", + ); + assertUnique( + batch.mutations.map((mutation) => mutation.proposalId), + "mutation proposal ID", + ); + if (batch.metadata.contentDigest !== computeEvolutionContentDigest(contentWithoutMetadata(batch))) { + throw new Error(`Mutation batch "${batch.batchId}" metadata digest is invalid`); + } + const batchHash = computeEvolutionContentDigest(batch); + const priorCandidate = await this.#loadCandidate(batch.batchId); + if (priorCandidate !== null) { + if (priorCandidate.batchHash !== batchHash) { + throw new Error(`Mutation batch "${batch.batchId}" was already staged with different content`); + } + return { + batchId: batch.batchId, + status: "written", + output: outputForCandidate(priorCandidate), + message: null, + }; + } + + const current = await this.#harnessState.loadCurrent(); + if (current.manifest.version !== batch.expectedBaseVersion) { + throw new EvolutionExecutionError( + `Harness base changed from ${batch.expectedBaseVersion} to ${current.manifest.version} before writing`, + { reasonCode: "harness-base-conflict", retryable: false }, + ); + } + const currentDigest = computeEvolutionContentDigest(current.state.memory); + if (currentDigest !== batch.expectedBaseDigest) { + throw new EvolutionExecutionError("Harness memory changed before writing", { + reasonCode: "harness-base-conflict", + retryable: false, + }); + } + const baseCatalog = await this.#loadCatalog(current); + if (!sameMemoryItems(activeItems(baseCatalog), batch.baseMemoryItems)) { + throw new EvolutionExecutionError("Harness memory item catalog changed before writing", { + reasonCode: "harness-base-conflict", + retryable: false, + }); + } + + let memory = current.state.memory; + let items = [...baseCatalog.items]; + const activeOrder = [...baseCatalog.activeOrder]; + const addedIds = new Set(); + const removedIds = new Set(); + const remove = (memoryItemId: string, expectedContentDigest: string): void => { + const activeIndex = activeOrder.indexOf(memoryItemId); + if (activeIndex === -1) throw new Error(`Mutation removes inactive memory item "${memoryItemId}"`); + const parsed = parseEvolutionMemory(memory); + const block = parsed.blocks[activeIndex]; + const item = itemById(items, memoryItemId); + if ( + block === undefined || + item.contentDigest !== expectedContentDigest || + itemDigest(block.content) !== expectedContentDigest || + item.ledger.origin !== "evolution" + ) { + throw new Error(`Mutation cannot automatically remove memory item "${memoryItemId}"`); + } + memory = removeMemoryBlock(memory, block); + activeOrder.splice(activeIndex, 1); + removedIds.add(memoryItemId); + }; + const add = (mutation: EvolutionMutation, delta: EvolutionMemoryAddDelta): void => { + const addition = parseEvolutionMemory(delta.content); + if ( + addition.headings.length > 0 || + addition.blocks.length !== 1 || + addition.blocks[0]?.content !== delta.content + ) { + throw new Error("Each EVOLVE.md addition must contain exactly one non-heading memory item"); + } + const parsed = parseEvolutionMemory(memory); + const offset = insertionOffset(memory, parsed, delta); + const insertionIndex = parsed.blocks.filter((block) => block.end <= offset).length; + memory = insertMemory(memory, offset, delta.content); + const memoryItemId = `memory-item-${createHash("sha256").update(mutation.mutationId).digest("hex").slice(0, 32)}`; + if (items.some((item) => item.memoryItemId === memoryItemId)) { + throw new Error(`Mutation reuses memory item ID "${memoryItemId}"`); + } + items.push({ + schemaVersion: EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION, + memoryItemId, + contentDigest: itemDigest(delta.content), + creationRevision: batch.expectedBaseVersion, + supersessionRevision: null, + ledger: { + origin: "evolution", + creationEventId: batch.activationEventId, + supersessionEventId: null, + }, + }); + activeOrder.splice(insertionIndex, 0, memoryItemId); + addedIds.add(memoryItemId); + }; + + for (const mutation of batch.mutations) { + if ( + mutation.target.kind !== this.kind || + mutation.target.expectedHarnessVersion !== batch.expectedBaseVersion + ) { + throw new Error(`Mutation "${mutation.mutationId}" targets an unsupported or stale harness`); + } + if (mutation.payload.operation === "ADD") add(mutation, mutation.payload); + if (mutation.payload.operation === "REMOVE") { + remove(mutation.payload.memoryItemId, mutation.payload.expectedContentDigest); + } + if (mutation.payload.operation === "REPLACE") { + remove(mutation.payload.removal.memoryItemId, mutation.payload.removal.expectedContentDigest); + add(mutation, { operation: "ADD", ...mutation.payload.addition }); + } + } + const parsedCandidate = parseEvolutionMemory(memory); + validateParsedMemory(memory, parsedCandidate, this.#maxMemoryBytes); + if ( + parsedCandidate.blocks.length !== activeOrder.length || + parsedCandidate.blocks.some( + (block, index) => itemById(items, activeOrder[index] as string).contentDigest !== itemDigest(block.content), + ) + ) { + throw new Error("EVOLVE.md candidate item index is inconsistent after applying mutations"); + } + const nextVersion = resultingHarnessVersion(memory, batch.expectedBaseVersion); + items = items.map((item) => { + if (addedIds.has(item.memoryItemId)) return { ...item, creationRevision: nextVersion }; + if (removedIds.has(item.memoryItemId)) { + return { + ...item, + supersessionRevision: nextVersion, + ledger: { ...item.ledger, supersessionEventId: batch.activationEventId }, + }; + } + return item; + }); + const catalog: StoredEvolutionMemoryCatalog = { + schemaVersion: CATALOG_SCHEMA_VERSION, + harnessVersion: nextVersion, + baseVersion: batch.expectedBaseVersion, + memoryContentDigest: computeEvolutionContentDigest(memory), + items, + activeOrder, + }; + const candidateContent = { + schemaVersion: CANDIDATE_SCHEMA_VERSION, + writerVersion: this.#version, + candidateId: `candidate-${batch.batchId}`, + batchId: batch.batchId, + batchHash, + expectedBaseVersion: batch.expectedBaseVersion, + expectedBaseDigest: batch.expectedBaseDigest, + resultingHarnessVersion: nextVersion, + resultingMemoryDigest: catalog.memoryContentDigest, + operationIds: batch.mutations.map((mutation) => mutation.mutationId), + beforeMemory: current.state.memory, + afterMemory: memory, + diff: exactDiff(current.state.memory, memory), + catalog, + }; + const candidate: StoredEvolutionMemoryCandidate = { + ...candidateContent, + candidateDigest: computeEvolutionContentDigest(candidateContent), + }; + await this.#writeImmutable(this.#candidateFile(batch.batchId), candidate); + return { batchId: batch.batchId, status: "written", output: outputForCandidate(candidate), message: null }; + } + + async activate( + batch: MutationBatch, + writeResult: MutationWriteResult, + context: Parameters[2], + ): Promise { + context.signal?.throwIfAborted(); + assertWriterContext(context); + const output = parseWriteOutput(writeResult); + const candidate = await this.#loadCandidate(batch.batchId); + if ( + writeResult.batchId !== batch.batchId || + candidate === null || + candidate.candidateId !== output.candidateId || + candidate.candidateDigest !== output.candidateDigest || + candidate.batchHash !== computeEvolutionContentDigest(batch) || + candidate.batchHash !== output.batchHash || + serializeEvolutionValue(output) !== serializeEvolutionValue(outputForCandidate(candidate)) + ) { + throw new Error(`Mutation batch "${batch.batchId}" write receipt does not match its durable candidate`); + } + await this.#writeImmutable(this.#catalogFile(candidate.resultingHarnessVersion), candidate.catalog); + const current = await this.#harnessState.loadCurrent(); + if (current.manifest.version === candidate.resultingHarnessVersion) { + if ( + current.manifest.base !== candidate.expectedBaseVersion || + current.state.memory !== candidate.afterMemory + ) { + throw new Error(`Activated harness revision for batch "${batch.batchId}" has different content`); + } + return { + batchId: batch.batchId, + status: "activated", + resultingHarnessVersion: candidate.resultingHarnessVersion, + message: null, + }; + } + if (current.manifest.version !== candidate.expectedBaseVersion) { + throw new EvolutionExecutionError( + `Harness base changed from ${candidate.expectedBaseVersion} to ${current.manifest.version} before activation`, + { reasonCode: "harness-base-conflict", retryable: false }, + ); + } + const advanced = await this.#harnessState.advance( + { memory: candidate.afterMemory }, + candidate.expectedBaseVersion, + ); + if (advanced.status === "conflict") { + throw new EvolutionExecutionError( + `Harness base changed from ${advanced.expectedBase} to ${advanced.actualBase} before activation`, + { reasonCode: "harness-base-conflict", retryable: false }, + ); + } + if (advanced.bundle.manifest.version !== candidate.resultingHarnessVersion) { + throw new Error(`Harness activation for batch "${batch.batchId}" produced an unexpected revision`); + } + return { + batchId: batch.batchId, + status: "activated", + resultingHarnessVersion: advanced.bundle.manifest.version, + message: null, + }; + } +} diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index a52d9c85c..5ecd7fe44 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -46,6 +46,7 @@ export { type EvolutionLedgerEvent, type EvolutionMemoryAddDelta, type EvolutionMemoryDelta, + type EvolutionMemoryExactDiff, type EvolutionMemoryItem, type EvolutionMemoryItemLedgerMetadata, type EvolutionMemoryProposal, @@ -53,6 +54,7 @@ export { type EvolutionMemoryRemoveDelta, type EvolutionMemoryReplaceDelta, type EvolutionMemoryTarget, + type EvolutionMemoryWriteOutput, type EvolutionModelIdentity, type EvolutionMutation, type EvolutionProcessedEvidenceCursor, @@ -70,6 +72,7 @@ export { type EvolutionTriggerKind, type ExcludedEvidenceSource, type ExistingInstructionReference, + type HarnessActivatedLedgerEvent, type HarnessActivationReceipt, type HarnessActivationResult, type ImmutableHarnessSnapshot, @@ -123,6 +126,7 @@ export { type StructuredDeltaCuratorOptions, } from "./curator.ts"; export { EvolutionExecutionError } from "./errors.ts"; +export { EvolutionMemoryWriter, type EvolutionMemoryWriterOptions } from "./evolution-memory-writer.ts"; export { EvolutionStorageCorruptionError, FileEvolutionPersistence, diff --git a/packages/evolution/src/pipeline.ts b/packages/evolution/src/pipeline.ts index 5146d8a56..b7ed3abff 100644 --- a/packages/evolution/src/pipeline.ts +++ b/packages/evolution/src/pipeline.ts @@ -7,14 +7,17 @@ import type { EvolutionInputSnapshot, EvolutionLedgerEvent, EvolutionMemoryProposal, + EvolutionMemoryWriteOutput, EvolutionProposal, EvolutionProposalLedgerDisposition, EvolutionScope, + HarnessActivatedLedgerEvent, HarnessActivationReceipt, InstructionSnapshot, Lesson, MutationBatch, MutationWriteReceipt, + MutationWrittenLedgerEvent, ProposalDispositionLedgerEvent, ReflectionNode, ScoredProposal, @@ -100,6 +103,26 @@ function isMutationWriteReceipt(artifact: EvolutionArtifact): artifact is Mutati ); } +function isEvolutionMemoryWriteOutput(value: unknown): value is EvolutionMemoryWriteOutput { + return ( + isRecord(value) && + typeof value.candidateId === "string" && + typeof value.candidateDigest === "string" && + typeof value.batchHash === "string" && + typeof value.expectedBaseVersion === "string" && + typeof value.expectedBaseDigest === "string" && + typeof value.resultingHarnessVersion === "string" && + typeof value.resultingMemoryDigest === "string" && + Array.isArray(value.operationIds) && + value.operationIds.every((operationId) => typeof operationId === "string") && + isRecord(value.diff) && + Number.isSafeInteger(value.diff.start) && + (value.diff.start as number) >= 0 && + typeof value.diff.removed === "string" && + typeof value.diff.added === "string" + ); +} + function assertUniqueIds(values: readonly string[], description: string): void { const seen = new Set(); for (const value of values) { @@ -406,18 +429,88 @@ export class EvolutionPipeline implements EvolutionStageExecutor { target: proposal.target, payload: proposal.exactDelta, })); - const batchId = stableId("mutation-batch", { idempotencyKey: context.idempotencyKey, mutations }); + const activationEventId = stableId("ledger-event", { + jobId: context.job.jobId, + type: "harness-activated", + proposalIds: proposals.map((proposal) => proposal.proposalId), + }); + const content = { + batchId: stableId("mutation-batch", { idempotencyKey: context.idempotencyKey, mutations }), + targetKind: "evolution-memory" as const, + expectedBaseVersion: context.input.expectedHarnessBaseVersion, + expectedBaseDigest: context.input.harness.memoryContentDigest, + baseMemoryItems: context.input.harness.activeMemoryItems, + activationEventId, + mutations, + }; return { metadata: this.#metadata( - batchId, + content.batchId, [context.job.jobId, ...selected.map((scored) => scored.metadata.artifactId)], context.job.metadata.scope, context.input.metadata.createdAt, - { batchId, targetKind: "evolution-memory", mutations }, + content, ), - batchId, - targetKind: "evolution-memory", - mutations, + ...content, + }; + } + + #mutationWrittenEvent( + context: EvolutionStageExecutionContext, + batch: MutationBatch, + writeResult: MutationWriteReceipt["result"], + ): MutationWrittenLedgerEvent { + const content = { batchId: batch.batchId, writeResult }; + const eventId = stableId("ledger-event", { idempotencyKey: context.idempotencyKey, ...content }); + return { + metadata: this.#ledgerMetadata(eventId, context, content), + eventId, + jobId: context.job.jobId, + type: "mutation-written", + batchId: batch.batchId, + writeResult, + }; + } + + #harnessActivatedEvent( + context: EvolutionStageExecutionContext, + batch: MutationBatch, + output: EvolutionMemoryWriteOutput, + result: HarnessActivationReceipt["result"], + selected: readonly ScoredProposal[], + ): HarnessActivatedLedgerEvent { + if (result.resultingHarnessVersion === null) { + throw new EvolutionExecutionError("Activated harness result is missing its revision", { + reasonCode: "pipeline-invariant", + retryable: false, + }); + } + const proposalIds = selected.map((scored) => scored.proposal.proposalId); + const evidenceIds = [ + ...new Set( + selected.flatMap((scored) => [ + ...scored.proposal.supportingEvidenceIds, + ...scored.proposal.contradictingEvidenceIds, + ]), + ), + ].sort((left, right) => left.localeCompare(right)); + const content = { + batchId: batch.batchId, + proposalIds, + evidenceIds, + operationIds: output.operationIds, + exactDiff: output.diff, + parentHarnessVersion: batch.expectedBaseVersion, + resultingHarnessVersion: result.resultingHarnessVersion, + activationResult: result, + rollbackParentHarnessVersion: batch.expectedBaseVersion, + }; + return { + metadata: this.#ledgerMetadata(batch.activationEventId, context, content), + eventId: batch.activationEventId, + jobId: context.job.jobId, + type: "harness-activated", + ...content, }; } @@ -597,6 +690,12 @@ export class EvolutionPipeline implements EvolutionStageExecutor { retryable: true, }); } + if (result.batchId !== batch.batchId || !isEvolutionMemoryWriteOutput(result.output)) { + throw new EvolutionExecutionError(`Mutation batch "${batch.batchId}" returned an invalid write result`, { + reasonCode: "writer-failed", + retryable: false, + }); + } const receiptId = stableId("write-receipt", { idempotencyKey: context.idempotencyKey, result }); const receipt: MutationWriteReceipt = { metadata: this.#metadata( @@ -612,6 +711,7 @@ export class EvolutionPipeline implements EvolutionStageExecutor { }; return { artifacts: [batch, receipt], + ledgerEvents: [this.#mutationWrittenEvent(context, batch, result)], state: { mutationBatchId: batch.batchId, writeReceiptId: receiptId }, directive: "continue", }; @@ -626,6 +726,22 @@ export class EvolutionPipeline implements EvolutionStageExecutor { retryable: true, }); } + if ( + receipt.batchId !== batch.batchId || + receipt.result.batchId !== batch.batchId || + !isEvolutionMemoryWriteOutput(receipt.result.output) || + receipt.result.output.expectedBaseVersion !== batch.expectedBaseVersion || + receipt.result.output.expectedBaseDigest !== batch.expectedBaseDigest || + receipt.result.output.operationIds.length !== batch.mutations.length || + receipt.result.output.operationIds.some( + (operationId, index) => operationId !== batch.mutations[index]?.mutationId, + ) + ) { + throw new EvolutionExecutionError("Evolution memory write receipt does not match its mutation batch", { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } let result: Awaited>; try { result = await writer.activate(batch, receipt.result, { @@ -645,6 +761,15 @@ export class EvolutionPipeline implements EvolutionStageExecutor { }, ); } + if ( + result.batchId !== batch.batchId || + result.resultingHarnessVersion !== receipt.result.output.resultingHarnessVersion + ) { + throw new EvolutionExecutionError("Harness activation result does not match its staged candidate", { + reasonCode: "writer-failed", + retryable: false, + }); + } const receiptId = stableId("activation-receipt", { idempotencyKey: context.idempotencyKey, result }); const activationReceipt: HarnessActivationReceipt = { metadata: this.#metadata( @@ -662,9 +787,19 @@ export class EvolutionPipeline implements EvolutionStageExecutor { .filter(isScoredProposal) .filter((scored) => scored.disposition === "selected") .map((scored) => this.#proposalDispositionEvent(context, scored, "applied")); + const selected = context.priorArtifacts + .filter(isScoredProposal) + .filter((scored) => scored.disposition === "selected"); + const activationEvent = this.#harnessActivatedEvent( + context, + batch, + receipt.result.output, + result, + selected, + ); return { artifacts: [activationReceipt], - ledgerEvents: appliedEvents, + ledgerEvents: [...appliedEvents, activationEvent], state: { activationReceiptId: receiptId }, directive: "complete-activated", resultingHarnessVersion: result.resultingHarnessVersion, diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index fe19496f6..d8b599c5e 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -362,7 +362,17 @@ describe("checkpointed evolution pipeline", () => { return { batchId: batch.batchId, status: "written", - output: { candidateDigest: computeEvolutionContentDigest(batch) }, + output: { + candidateId: `candidate-${batch.batchId}`, + candidateDigest: computeEvolutionContentDigest(batch), + batchHash: computeEvolutionContentDigest(batch), + expectedBaseVersion: batch.expectedBaseVersion, + expectedBaseDigest: batch.expectedBaseDigest, + resultingHarnessVersion: SECOND_HARNESS_VERSION, + resultingMemoryDigest: computeEvolutionContentDigest("candidate"), + operationIds: batch.mutations.map((mutation) => mutation.mutationId), + diff: { start: 0, removed: "", added: "candidate" }, + }, message: null, }; }, @@ -562,11 +572,23 @@ describe("checkpointed evolution pipeline", () => { expect(new Set(modelKeys).size).toBe(2); expect(writeKeys.size).toBe(1); expect(activationKeys.size).toBe(1); + const ledger = await fixture.persistence.read(created.jobId); expect( - (await fixture.persistence.read(created.jobId)) + ledger .filter((event) => event.type === "proposal-disposition") .map((event) => [event.proposalId, event.disposition]), ).toEqual([["proposal-1", "applied"]]); + expect(ledger.filter((event) => event.type === "mutation-written")).toHaveLength(1); + expect(ledger.find((event) => event.type === "harness-activated")).toMatchObject({ + batchId: expect.stringMatching(/^mutation-batch-/), + proposalIds: ["proposal-1"], + evidenceIds: [], + operationIds: [expect.stringMatching(/^mutation-/)], + exactDiff: { start: 0, removed: "", added: "candidate" }, + parentHarnessVersion: FIRST_HARNESS_VERSION, + resultingHarnessVersion: SECOND_HARNESS_VERSION, + rollbackParentHarnessVersion: FIRST_HARNESS_VERSION, + }); }); }); diff --git a/packages/evolution/test/evolution-memory-writer.test.ts b/packages/evolution/test/evolution-memory-writer.test.ts new file mode 100644 index 000000000..362ad06e8 --- /dev/null +++ b/packages/evolution/test/evolution-memory-writer.test.ts @@ -0,0 +1,364 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { HarnessStateStore } from "../../harness-state/src/index.ts"; +import { + computeEvolutionContentDigest, + createEvolutionArtifactMetadata, + type EvolutionArtifactProducer, + type EvolutionLeaseToken, + EvolutionMemoryWriter, + type EvolutionMutation, + type EvolutionScope, + type MutationBatch, +} from "../src/index.ts"; + +const scope: EvolutionScope = { repositoryId: "diffex", scopeId: "repository" }; +const producer: EvolutionArtifactProducer = { + name: "fixture", + version: "1.0.0", + promptVersion: null, + model: null, +}; +const writerLease: EvolutionLeaseToken = { + kind: "writer", + key: "global", + ownerId: "writer-1", + fence: 1, + expiresAt: "2099-01-01T00:00:00.000Z", +}; +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "diffex-evolution-writer-")); + temporaryDirectories.push(root); + const harnessState = new HarnessStateStore(join(root, "agent")); + const empty = await harnessState.initialize(); + const initialMemory = + "# Evolution memory\n\n## Existing\n\n- Preserve this manual rule.\n\nTail bytes stay exact.\n"; + const initial = await harnessState.advance({ memory: initialMemory }, empty.manifest.version); + if (initial.status !== "advanced") throw new Error("Expected fixture harness advance"); + const writer = new EvolutionMemoryWriter({ + version: "1.0.0", + harnessState, + statePath: join(root, "writer-state"), + }); + const snapshot = await writer.captureSnapshot(); + return { harnessState, initial: initial.bundle, initialMemory, snapshot, writer }; +} + +function batch( + snapshot: Awaited>, + mutations: readonly EvolutionMutation[], + options: { readonly batchId?: string; readonly activationEventId?: string } = {}, +): MutationBatch { + const content = { + batchId: options.batchId ?? "batch-1", + targetKind: "evolution-memory" as const, + expectedBaseVersion: snapshot.harnessVersion, + expectedBaseDigest: snapshot.memoryContentDigest, + baseMemoryItems: snapshot.activeMemoryItems, + activationEventId: options.activationEventId ?? "activation-event-1", + mutations, + }; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: content.batchId, + producer, + sourceIds: mutations.map((mutation) => mutation.proposalId), + scope, + createdAt: "2026-08-19T12:00:00.000Z", + content, + }), + ...content, + }; +} + +describe("evolution memory writer", () => { + it("stages selected deltas and atomically activates one harness revision", async () => { + const { harnessState, initial, initialMemory, snapshot, writer } = await fixture(); + const mutations: readonly EvolutionMutation[] = [ + { + mutationId: "mutation-add", + proposalId: "proposal-add", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: "Existing", anchor: null }, + content: "- Apply selected deltas as one batch.", + intendedScope: ["repository"], + }, + }, + ]; + const selectedBatch = batch(snapshot, mutations); + + const writeResult = await writer.write(selectedBatch, { + idempotencyKey: "write-key", + lease: writerLease, + }); + + expect(writeResult).toMatchObject({ + batchId: selectedBatch.batchId, + status: "written", + output: { + expectedBaseVersion: initial.manifest.version, + operationIds: ["mutation-add"], + }, + message: null, + }); + expect((await harnessState.loadCurrent()).state.memory).toBe(initialMemory); + + const activation = await writer.activate(selectedBatch, writeResult, { + idempotencyKey: "activate-key", + lease: writerLease, + }); + + expect(activation.status).toBe("activated"); + expect(activation.resultingHarnessVersion).not.toBe(initial.manifest.version); + const current = await harnessState.loadCurrent(); + expect(current.manifest.base).toBe(initial.manifest.version); + expect(current.state.memory).toBe( + "# Evolution memory\n\n## Existing\n\n- Preserve this manual rule.\n\nTail bytes stay exact.\n\n- Apply selected deltas as one batch.\n", + ); + const activatedSnapshot = await writer.captureSnapshot(); + expect(activatedSnapshot.activeMemoryItems).toHaveLength(3); + expect(activatedSnapshot.activeMemoryItems.at(-1)).toMatchObject({ + contentDigest: computeEvolutionContentDigest("- Apply selected deltas as one batch."), + creationRevision: current.manifest.version, + ledger: { origin: "evolution", creationEventId: "activation-event-1", supersessionEventId: null }, + }); + expect(activatedSnapshot.activeMemoryItems.slice(0, 2)).toEqual( + snapshot.activeMemoryItems.map((item) => expect.objectContaining({ memoryItemId: item.memoryItemId })), + ); + }); + + it("rejects a partial batch without publishing any mutation", async () => { + const { harnessState, initial, initialMemory, snapshot, writer } = await fixture(); + const selectedBatch = batch(snapshot, [ + { + mutationId: "mutation-valid", + proposalId: "proposal-valid", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: "Existing", anchor: null }, + content: "- This change must not leak from a failed batch.", + intendedScope: ["repository"], + }, + }, + { + mutationId: "mutation-invalid", + proposalId: "proposal-invalid", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: "Missing section", anchor: null }, + content: "- This destination is stale.", + intendedScope: ["repository"], + }, + }, + ]); + + await expect(writer.write(selectedBatch, { idempotencyKey: "write-key", lease: writerLease })).rejects.toThrow( + 'section "Missing section" must exist exactly once', + ); + expect((await harnessState.loadCurrent()).state.memory).toBe(initialMemory); + expect((await harnessState.listVersions()).versions).toHaveLength(2); + expect((await writer.captureSnapshot()).harnessVersion).toBe(initial.manifest.version); + }); + + it("retains manually authored items for explicit review", async () => { + const { harnessState, initialMemory, snapshot, writer } = await fixture(); + const manualItem = snapshot.activeMemoryItems[0]; + if (manualItem === undefined) throw new Error("Expected a manual memory item"); + const selectedBatch = batch(snapshot, [ + { + mutationId: "mutation-remove-manual", + proposalId: "proposal-remove-manual", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "REMOVE", + memoryItemId: manualItem.memoryItemId, + expectedContentDigest: manualItem.contentDigest, + reason: "Automatic removal is not authorized", + }, + }, + ]); + + await expect(writer.write(selectedBatch, { idempotencyKey: "write-key", lease: writerLease })).rejects.toThrow( + `cannot automatically remove memory item "${manualItem.memoryItemId}"`, + ); + expect((await harnessState.loadCurrent()).state.memory).toBe(initialMemory); + }); + + it("resumes an interrupted write and activation without applying a delta twice", async () => { + const { harnessState, snapshot, writer } = await fixture(); + const addition = batch( + snapshot, + [ + { + mutationId: "mutation-managed", + proposalId: "proposal-managed", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: "Existing", anchor: null }, + content: "- Remove this evolution-managed rule later.", + intendedScope: ["repository"], + }, + }, + ], + { batchId: "batch-add", activationEventId: "activation-add" }, + ); + const additionWrite = await writer.write(addition, { idempotencyKey: "write-add", lease: writerLease }); + await writer.activate(addition, additionWrite, { idempotencyKey: "activate-add", lease: writerLease }); + const addedSnapshot = await writer.captureSnapshot(); + const managedItem = addedSnapshot.activeMemoryItems.find((item) => item.ledger.origin === "evolution"); + if (managedItem === undefined) throw new Error("Expected an evolution-managed memory item"); + const removal = batch( + addedSnapshot, + [ + { + mutationId: "mutation-remove-managed", + proposalId: "proposal-remove-managed", + target: { kind: "evolution-memory", expectedHarnessVersion: addedSnapshot.harnessVersion }, + payload: { + operation: "REMOVE", + memoryItemId: managedItem.memoryItemId, + expectedContentDigest: managedItem.contentDigest, + reason: "Validated counterevidence", + }, + }, + ], + { batchId: "batch-remove", activationEventId: "activation-remove" }, + ); + + const firstWrite = await writer.write(removal, { idempotencyKey: "write-remove", lease: writerLease }); + const resumedWrite = await writer.write(removal, { idempotencyKey: "write-remove", lease: writerLease }); + expect(resumedWrite).toEqual(firstWrite); + const firstActivation = await writer.activate(removal, firstWrite, { + idempotencyKey: "activate-remove", + lease: writerLease, + }); + const resumedActivation = await writer.activate(removal, resumedWrite, { + idempotencyKey: "activate-remove", + lease: writerLease, + }); + + expect(resumedActivation).toEqual(firstActivation); + expect((await harnessState.listVersions()).versions).toHaveLength(4); + expect((await harnessState.loadCurrent()).state.memory).not.toContain("Remove this evolution-managed rule"); + expect((await writer.captureSnapshot()).activeMemoryItems).toHaveLength(2); + }); + + it("rejects sensitive or malformed candidates before harness activation", async () => { + const { harnessState, initialMemory, snapshot, writer } = await fixture(); + const rejected = [ + { + id: "secret", + content: "- api_key = abcdefghijklmnopqrstuvwxyz123456", + message: "sensitive-data policy", + }, + { id: "fence", content: "```text\nunclosed", message: "unclosed fenced code block" }, + { + id: "duplicate", + content: "- Preserve this manual rule.", + message: "duplicate normalized memory items", + }, + ] as const; + for (const candidate of rejected) { + const selectedBatch = batch( + snapshot, + [ + { + mutationId: `mutation-${candidate.id}`, + proposalId: `proposal-${candidate.id}`, + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: null, anchor: null }, + content: candidate.content, + intendedScope: ["repository"], + }, + }, + ], + { batchId: `batch-${candidate.id}`, activationEventId: `activation-${candidate.id}` }, + ); + await expect( + writer.write(selectedBatch, { idempotencyKey: `write-${candidate.id}`, lease: writerLease }), + ).rejects.toThrow(candidate.message); + } + expect((await harnessState.loadCurrent()).state.memory).toBe(initialMemory); + }); + + it("leaves a newer selected harness active when compare-and-swap loses", async () => { + const { harnessState, snapshot, writer } = await fixture(); + const selectedBatch = batch(snapshot, [ + { + mutationId: "mutation-stale", + proposalId: "proposal-stale", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: null, anchor: null }, + content: "- This candidate loses the activation race.", + intendedScope: ["repository"], + }, + }, + ]); + const writeResult = await writer.write(selectedBatch, { + idempotencyKey: "write-stale", + lease: writerLease, + }); + const competing = await harnessState.advance( + { memory: `${snapshot.memory}\nCompeting selected memory.\n` }, + snapshot.harnessVersion, + ); + if (competing.status !== "advanced") throw new Error("Expected competing harness advance"); + + await expect( + writer.activate(selectedBatch, writeResult, { idempotencyKey: "activate-stale", lease: writerLease }), + ).rejects.toMatchObject({ reasonCode: "harness-base-conflict", retryable: false }); + expect(await harnessState.loadCurrent()).toEqual(competing.bundle); + expect((await harnessState.listVersions()).versions).toHaveLength(3); + }); + + it("rejects a tampered write receipt before activation", async () => { + const { harnessState, initialMemory, snapshot, writer } = await fixture(); + const selectedBatch = batch(snapshot, [ + { + mutationId: "mutation-receipt", + proposalId: "proposal-receipt", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: null, anchor: null }, + content: "- Activate only the exact staged receipt.", + intendedScope: ["repository"], + }, + }, + ]); + const writeResult = await writer.write(selectedBatch, { + idempotencyKey: "write-receipt", + lease: writerLease, + }); + if (writeResult.output === null) throw new Error("Expected a candidate output"); + const tampered = { + ...writeResult, + output: { ...writeResult.output, diff: { start: 0, removed: "", added: "false audit data" } }, + }; + + await expect( + writer.activate(selectedBatch, tampered, { + idempotencyKey: "activate-receipt", + lease: writerLease, + }), + ).rejects.toThrow("write receipt does not match its durable candidate"); + expect((await harnessState.loadCurrent()).state.memory).toBe(initialMemory); + }); +}); diff --git a/packages/evolution/vitest.config.ts b/packages/evolution/vitest.config.ts index 157f467a0..46038c0e4 100644 --- a/packages/evolution/vitest.config.ts +++ b/packages/evolution/vitest.config.ts @@ -1,9 +1,15 @@ +import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; +const harnessStateSource = fileURLToPath(new URL("../harness-state/src/index.ts", import.meta.url)); + export default defineConfig({ test: { globals: true, environment: "node", reporters: process.env.GITHUB_ACTIONS ? ["dot", "github-actions"] : ["dot"], }, + resolve: { + alias: [{ find: /^@earendil-works\/diffex-harness-state$/, replacement: harnessStateSource }], + }, });