diff --git a/packages/evolution/COUNTER.md b/packages/evolution/COUNTER.md new file mode 100644 index 000000000..45614cfe0 --- /dev/null +++ b/packages/evolution/COUNTER.md @@ -0,0 +1,52 @@ +# Counter Reservation + +## Pipeline boundary + +The eventual topology is `traces + current memory -> Counter`, followed by `traces + CounterReport -> Reflector -> Curator -> deterministic Validator -> Writer`. +The pipeline already owns a pre-reflection checkpoint and accepts an optional `PreReflectionStage` implementation at that boundary. +The current default is `DisabledPreReflectionStage`. + +The disabled stage returns no report. +Its checkpoint contains no artifacts and no Counter-specific state. +It emits no observations, signals, trends, scores, proposals, mutations, or ledger events. +It does not make model calls or change deterministic validation and selection. + +## Stable memory identity + +`EvolutionMemoryItem.memoryItemId` is a generated stable address and must not be derived from rendered text or its digest. +`contentDigest` verifies the item content associated with a revision, while `creationRevision` and `supersessionRevision` describe its harness lifetime. +Ledger event references connect the item to durable audit history without putting that history into EVOLVE.md. +Frozen evolution snapshots carry active memory-item records separately from the rendered memory string. + +Mutable Counter metadata must remain outside EVOLVE.md. +Observation totals, trends, policy versions, and other mutable statistics must not alter the rendered prompt or force a harness revision. + +## Observation invariants + +`CounterObservation` is an append-only raw record. +Implementations must never replace raw observations with aggregate totals. +Each record identifies the memory item, harness revision, top-level task lineage, traces, evidence, signal source, attribution method, policy version, timestamp, and idempotency key. + +Candidate observations may be appended while evidence is incomplete. +At most one final observation may exist for each `(memoryItemId, topLevelTaskLineageId)` pair. +The uniqueness key deliberately excludes retries, resumed processes, repeated citations, subagent activity, and policy version. +Those execution details may contribute evidence to the same task-level conclusion but must not inflate the final sample count. + +Future trace metadata should distinguish memory items that were available, selected, and used. +Attribution may use direct user feedback or deterministic test, tool, replay, and environment outcomes. +Model self-assessment and co-occurrence are weaker attribution methods than direct feedback and deterministic outcomes. + +## Report and policy boundary + +`CounterReport` contains evidence references and trends only. +It must not contain executable mutations, proposal scores, or decisions to keep, refine, rescope, deactivate, remove, or delete an item. +Those decisions belong to Curator proposals and deterministic policy gates. + +Removal thresholds, minimum sample sizes, recency treatment, and deactivation-before-deletion behavior belong to a separately versioned policy. +The observation and report contracts carry the applicable policy version without embedding policy knobs into their schemas. + +## Deferred implementation + +This reservation does not add observation persistence, aggregation, attribution, scoring, pruning, trace selection metadata, or memory mutation. +A future implementation may provide the existing `PreReflectionStage` port, persist raw observations in a dedicated append-only store, and emit a versioned `CounterReport`. +That implementation must preserve the frozen job input, checkpoint artifact format, and explicit `counterReport` input already consumed by Reflector. diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 3781a797d..500d9c70b 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -6,6 +6,9 @@ import type { } from "@earendil-works/diffex-harness-trace"; export const EVOLUTION_SCHEMA_VERSION = 1 as const; +export const EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION = 1 as const; +export const COUNTER_OBSERVATION_SCHEMA_VERSION = 1 as const; +export const COUNTER_REPORT_SCHEMA_VERSION = 1 as const; export type JsonPrimitive = boolean | number | string | null; export type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[]; @@ -39,11 +42,77 @@ export interface EvolutionArtifactMetadata { readonly createdAt: string; } +export interface EvolutionMemoryItemLedgerMetadata { + readonly origin: "evolution" | "manual"; + readonly creationEventId: string | null; + readonly supersessionEventId: string | null; +} + +export interface EvolutionMemoryItem { + readonly schemaVersion: typeof EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION; + readonly memoryItemId: string; + readonly contentDigest: string; + readonly creationRevision: HarnessVersion; + readonly supersessionRevision: HarnessVersion | null; + readonly ledger: EvolutionMemoryItemLedgerMetadata; +} + +export type CounterSignal = "helpful" | "harmful" | "neutral" | "unknown"; + +export type CounterSignalSource = "direct-user" | "environment" | "model-self-assessment" | "replay" | "test" | "tool"; + +export type CounterAttributionMethod = "co-occurrence" | "deterministic" | "direct" | "inferred" | "self-assessed"; + +export type CounterObservationStatus = "candidate" | "final"; + +export interface CounterObservation { + readonly schemaVersion: typeof COUNTER_OBSERVATION_SCHEMA_VERSION; + readonly observationId: string; + readonly memoryItemId: string; + readonly harnessRevision: HarnessVersion; + readonly topLevelTaskLineageId: string; + readonly traceIds: readonly string[]; + readonly evidenceIds: readonly string[]; + readonly signal: CounterSignal; + readonly signalSource: CounterSignalSource; + readonly attributionMethod: CounterAttributionMethod; + readonly status: CounterObservationStatus; + readonly policyVersion: string; + readonly timestamp: string; + readonly idempotencyKey: string; +} + +export interface CounterReportEvidence { + readonly memoryItemId: string; + readonly signal: CounterSignal; + readonly observationIds: readonly string[]; + readonly topLevelTaskLineageIds: readonly string[]; + readonly traceIds: readonly string[]; + readonly evidenceIds: readonly string[]; +} + +export interface CounterReportTrend { + readonly memoryItemId: string; + readonly signal: CounterSignal; + readonly observationIds: readonly string[]; + readonly topLevelTaskLineageIds: readonly string[]; +} + +export interface CounterReport { + readonly artifactKind: "counter-report"; + readonly schemaVersion: typeof COUNTER_REPORT_SCHEMA_VERSION; + readonly metadata: EvolutionArtifactMetadata; + readonly policyVersion: string; + readonly evidence: readonly CounterReportEvidence[]; + readonly trends: readonly CounterReportTrend[]; +} + export interface ImmutableHarnessSnapshot { readonly harnessVersion: HarnessVersion; readonly baseVersion: HarnessVersion | null; readonly memory: string; readonly memoryContentDigest: string; + readonly activeMemoryItems: readonly EvolutionMemoryItem[]; } export type EvolutionEvidenceOrigin = "product-runtime" | "evolution-pipeline"; diff --git a/packages/evolution/src/counter.ts b/packages/evolution/src/counter.ts new file mode 100644 index 000000000..6433cb420 --- /dev/null +++ b/packages/evolution/src/counter.ts @@ -0,0 +1,7 @@ +import type { PreReflectionStage } from "./ports.ts"; + +export class DisabledPreReflectionStage implements PreReflectionStage { + run(): Promise { + return Promise.resolve(null); + } +} diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 7f7c616a2..ad13e8147 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -1,6 +1,17 @@ export { type ActiveEvolutionStage, type CheckpointSavedLedgerEvent, + COUNTER_OBSERVATION_SCHEMA_VERSION, + COUNTER_REPORT_SCHEMA_VERSION, + type CounterAttributionMethod, + type CounterObservation, + type CounterObservationStatus, + type CounterReport, + type CounterReportEvidence, + type CounterReportTrend, + type CounterSignal, + type CounterSignalSource, + EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION, EVOLUTION_SCHEMA_VERSION, type EvidenceCorpusDocument, type EvidenceCorpusManifest, @@ -19,6 +30,8 @@ export { type EvolutionJobStage, type EvolutionLeaseToken, type EvolutionLedgerEvent, + type EvolutionMemoryItem, + type EvolutionMemoryItemLedgerMetadata, type EvolutionMemoryProposal, type EvolutionMemoryTarget, type EvolutionModelIdentity, @@ -71,6 +84,7 @@ export { type EvolutionJobCreationBlockReason, type EvolutionJobCreationResult, } from "./control-plane.ts"; +export { DisabledPreReflectionStage } from "./counter.ts"; export { EvolutionExecutionError } from "./errors.ts"; export { EvolutionStorageCorruptionError, @@ -106,6 +120,8 @@ export type { EvolutionStageExecutor, EvolutionWorkSource, InteractivePriorityGate, + PreReflectionStage, + PreReflectionStageInput, ProposalScorer, ProposalScoringInput, ProposalValidationInput, diff --git a/packages/evolution/src/pipeline.ts b/packages/evolution/src/pipeline.ts index 6d2880737..2b36a7d97 100644 --- a/packages/evolution/src/pipeline.ts +++ b/packages/evolution/src/pipeline.ts @@ -1,4 +1,5 @@ import type { + CounterReport, EvidenceCorpusManifest, EvolutionArtifactMetadata, EvolutionArtifactProducer, @@ -18,6 +19,7 @@ import type { ScoredProposal, ValidationReport, } from "./contracts.ts"; +import { DisabledPreReflectionStage } from "./counter.ts"; import { EvolutionExecutionError } from "./errors.ts"; import type { Curator, @@ -26,6 +28,7 @@ import type { EvolutionStageExecutionContext, EvolutionStageExecutionResult, EvolutionStageExecutor, + PreReflectionStage, ProposalScorer, ProposalValidator, ReflectionResult, @@ -44,6 +47,7 @@ export interface EvolutionPipelineDependencies { readonly validator: ProposalValidator; readonly scorer: ProposalScorer; readonly targets: EvolutionTargetAdapterRegistry; + readonly preReflectionStage?: PreReflectionStage; } function isRecord(value: unknown): value is Readonly> { @@ -54,6 +58,10 @@ function isCorpus(artifact: EvolutionArtifact): artifact is EvidenceCorpusManife return "excludedSources" in artifact; } +function isCounterReport(artifact: EvolutionArtifact): artifact is CounterReport { + return "artifactKind" in artifact && artifact.artifactKind === "counter-report"; +} + function isInstructionSnapshot(artifact: EvolutionArtifact): artifact is InstructionSnapshot { return "documents" in artifact && !("excludedSources" in artifact); } @@ -108,6 +116,16 @@ function requireOne(values: readonly T[], description: string): T { return values[0] as T; } +function optionalOne(values: readonly T[], description: string): T | null { + if (values.length > 1) { + throw new EvolutionExecutionError(`Expected at most one ${description}, found ${values.length}`, { + reasonCode: "artifact-corrupt", + retryable: false, + }); + } + return values[0] ?? null; +} + function assertScoredProposals( proposals: readonly EvolutionProposal[], scoredProposals: readonly ScoredProposal[], @@ -194,10 +212,12 @@ function stageFailure( export class EvolutionPipeline implements EvolutionStageExecutor { readonly #dependencies: EvolutionPipelineDependencies; + readonly #preReflectionStage: PreReflectionStage; readonly #producer: EvolutionArtifactProducer; constructor(dependencies: EvolutionPipelineDependencies) { this.#dependencies = dependencies; + this.#preReflectionStage = dependencies.preReflectionStage ?? new DisabledPreReflectionStage(); this.#producer = { name: PIPELINE_PRODUCER_NAME, version: dependencies.version, @@ -337,12 +357,14 @@ export class EvolutionPipeline implements EvolutionStageExecutor { #pipelineInputs(context: EvolutionStageExecutionContext): { readonly corpus: EvidenceCorpusManifest; + readonly counterReport: CounterReport | null; readonly instructions: InstructionSnapshot; readonly reflection: ReflectionResult; readonly proposals: readonly EvolutionProposal[]; } { return { corpus: requireOne(context.priorArtifacts.filter(isCorpus), "corpus manifest"), + counterReport: optionalOne(context.priorArtifacts.filter(isCounterReport), "Counter report"), instructions: requireOne(context.priorArtifacts.filter(isInstructionSnapshot), "instruction snapshot"), reflection: { nodes: context.priorArtifacts.filter(isReflectionNode), @@ -401,15 +423,30 @@ export class EvolutionPipeline implements EvolutionStageExecutor { directive: "continue", }; } - case "counter-slot": - return { artifacts: [], state: { enabled: false }, directive: "continue" }; + case "counter-slot": { + const corpus = requireOne(context.priorArtifacts.filter(isCorpus), "corpus manifest"); + const counterReport = await this.#preReflectionStage.run({ + input: context.input, + corpus, + activeMemoryItems: context.input.harness.activeMemoryItems, + idempotencyKey: context.idempotencyKey, + signal: context.signal, + }); + if (counterReport === null) return { artifacts: [], state: {}, directive: "continue" }; + return { + artifacts: [counterReport], + state: { counterReportId: counterReport.metadata.artifactId }, + directive: "continue", + }; + } case "reflecting": { - const { corpus, instructions } = this.#pipelineInputs(context); + const { corpus, counterReport, instructions } = this.#pipelineInputs(context); let reflection: ReflectionResult; try { reflection = await this.#dependencies.reflector.reflect({ input: context.input, corpus, + counterReport, instructions, model: this.#dependencies.model, idempotencyKey: context.idempotencyKey, diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 6a230f104..85c7f0197 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -1,5 +1,6 @@ import type { MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; import type { + CounterReport, EvidenceCorpusManifest, EvolutionArtifactReference, EvolutionInputSnapshot, @@ -7,6 +8,7 @@ import type { EvolutionJobStage, EvolutionLeaseToken, EvolutionLedgerEvent, + EvolutionMemoryItem, EvolutionModelIdentity, EvolutionProcessedEvidenceCursor, EvolutionPromptVersions, @@ -29,6 +31,7 @@ import type { } from "./contracts.ts"; export type EvolutionArtifact = + | CounterReport | EvidenceCorpusManifest | EvolutionInputSnapshot | EvolutionProposal @@ -69,9 +72,22 @@ export interface EvolutionModelExecutor { execute(request: EvolutionModelRequest): Promise; } +export interface PreReflectionStageInput { + readonly input: EvolutionInputSnapshot; + readonly corpus: EvidenceCorpusManifest; + readonly activeMemoryItems: readonly EvolutionMemoryItem[]; + readonly idempotencyKey: string; + readonly signal?: AbortSignal; +} + +export interface PreReflectionStage { + run(input: PreReflectionStageInput): Promise; +} + export interface ReflectorInput { readonly input: EvolutionInputSnapshot; readonly corpus: EvidenceCorpusManifest; + readonly counterReport: CounterReport | null; readonly instructions: InstructionSnapshot; readonly model: EvolutionModelExecutor; readonly idempotencyKey: string; diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index 4b08e0551..815e2cbde 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -105,6 +105,7 @@ function snapshotContext(): EvolutionSnapshotContext { baseVersion: null, memory, memoryContentDigest: computeEvolutionContentDigest(memory), + activeMemoryItems: [], }, instructions: [ { @@ -351,6 +352,7 @@ describe("checkpointed evolution pipeline", () => { content, }); const modelKeys: string[] = []; + const reflectorCounterReports: unknown[] = []; const writeKeys = new Set(); const activationKeys = new Set(); const target: EvolutionTargetWriter = { @@ -384,6 +386,7 @@ describe("checkpointed evolution pipeline", () => { }, reflector: { async reflect(input) { + reflectorCounterReports.push(input.counterReport); await input.model.execute({ stage: "reflector", promptVersion: "reflector-v1", @@ -478,6 +481,9 @@ describe("checkpointed evolution pipeline", () => { const completed = await fixture.controlPlane.runJob(created.jobId); expect(completed.outcome).toBe("completed"); + const counterCheckpoint = completed.checkpoints.find((checkpoint) => checkpoint.stage === "counter-slot"); + expect(counterCheckpoint).toMatchObject({ outputArtifacts: [], state: {} }); + expect(reflectorCounterReports).toEqual([null]); expect(modelKeys).toHaveLength(2); expect(new Set(modelKeys).size).toBe(2); expect(writeKeys.size).toBe(1); diff --git a/packages/evolution/test/counter.test.ts b/packages/evolution/test/counter.test.ts new file mode 100644 index 000000000..23b3ab140 --- /dev/null +++ b/packages/evolution/test/counter.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { + COUNTER_OBSERVATION_SCHEMA_VERSION, + COUNTER_REPORT_SCHEMA_VERSION, + DisabledPreReflectionStage, + EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION, + type EvolutionMemoryItem, +} from "../src/index.ts"; + +const HARNESS_VERSION = `sha256:${"1".repeat(64)}`; + +describe("reserved counter contracts", () => { + it("versions memory items, observations, and reports independently", () => { + expect(EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION).toBe(1); + expect(COUNTER_OBSERVATION_SCHEMA_VERSION).toBe(1); + expect(COUNTER_REPORT_SCHEMA_VERSION).toBe(1); + }); + + it("keeps stable memory identity outside rendered memory", () => { + const item: EvolutionMemoryItem = { + schemaVersion: EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION, + memoryItemId: "memory-item-1", + contentDigest: `sha256:${"2".repeat(64)}`, + creationRevision: HARNESS_VERSION, + supersessionRevision: null, + ledger: { + origin: "evolution", + creationEventId: "ledger-event-1", + supersessionEventId: null, + }, + }; + + expect(item.memoryItemId).toBe("memory-item-1"); + expect(item).not.toHaveProperty("content"); + }); + + it("returns no report from the disabled stage", async () => { + const counter = new DisabledPreReflectionStage(); + + await expect(counter.run()).resolves.toBeNull(); + }); +});