From 5f5631585494301f0df071d3ea1f7b16c46c4aef Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Wed, 19 Aug 2026 18:56:28 -0700 Subject: [PATCH 1/3] feat(evolution): add evidence ranker - Prevents unverifiable or weakly recurring proposals from reaching mutation planning - Makes deterministic selection auditable through versioned score components and dispositions --- packages/evolution/src/contracts.ts | 31 +- packages/evolution/src/index.ts | 8 + packages/evolution/src/pipeline.ts | 2 +- packages/evolution/src/proposal-ranking.ts | 956 ++++++++++++++++++ packages/evolution/test/control-plane.test.ts | 15 +- .../evolution/test/proposal-ranking.test.ts | 482 +++++++++ 6 files changed, 1488 insertions(+), 6 deletions(-) create mode 100644 packages/evolution/src/proposal-ranking.ts create mode 100644 packages/evolution/test/proposal-ranking.test.ts diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index 8247c3577..f8b1785a9 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -488,21 +488,42 @@ export interface ValidationIssue { export interface ProposalValidationResult { readonly proposalId: string; - readonly disposition: "accepted" | "deferred" | "rejected"; + readonly disposition: "accepted" | "deferred" | "quarantined" | "rejected"; + readonly reasonCodes: readonly string[]; readonly issues: readonly ValidationIssue[]; } export interface ValidationReport { readonly metadata: EvolutionArtifactMetadata; + readonly validatorVersion: string; + readonly assurance: "provenance-recurrence-and-structure-only"; readonly proposalResults: readonly ProposalValidationResult[]; } -export type ScoredProposalDisposition = "deferred" | "rejected" | "selected" | "superseded"; +export interface EvidenceRankingScoreComponents { + readonly recurrence: number; + readonly provenanceIntegrity: number; + readonly sourceSignalStrength: number; + readonly scopeCoverage: number; + readonly curatorConfidence: number; + readonly proposalCompleteness: number; + readonly contradictionPenalty: number; +} + +export type ScoredProposalDisposition = + | "deferred" + | "quarantined" + | "rejected" + | "selected" + | "superseded" + | "unselected"; export interface ScoredProposal { readonly metadata: EvolutionArtifactMetadata; readonly proposal: EvolutionProposal; - readonly score: number | null; + readonly scorerVersion: string; + readonly evidenceRankingScore: number | null; + readonly scoreComponents: EvidenceRankingScoreComponents | null; readonly disposition: ScoredProposalDisposition; readonly reasonCodes: readonly string[]; } @@ -658,9 +679,11 @@ export type EvolutionProposalLedgerDisposition = | "applied" | "deferred" | "failed" + | "quarantined" | "rejected" | "rolled-back" - | "superseded"; + | "superseded" + | "unselected"; interface EvolutionLedgerEventBase { readonly metadata: EvolutionArtifactMetadata; diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 14d6511f2..a52d9c85c 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -20,6 +20,7 @@ export { type EvidenceCoverageDisposition, type EvidenceCoverageEntry, type EvidenceCoverageMap, + type EvidenceRankingScoreComponents, type EvidenceSemanticUnit, type EvidenceSemanticUnitKind, type EvidenceSlice, @@ -168,6 +169,13 @@ export type { ReflectorInput, SuccessfulMaterialization, } from "./ports.ts"; +export { + DeterministicProposalRanker, + type DeterministicProposalRankerDependencies, + type DeterministicProposalRankerOptions, + EVIDENCE_RANKING_ASSURANCE, + EVIDENCE_RANKING_SCORER_VERSION, +} from "./proposal-ranking.ts"; export { type CreateEvolutionArtifactMetadataOptions, computeEvolutionContentDigest, diff --git a/packages/evolution/src/pipeline.ts b/packages/evolution/src/pipeline.ts index 0ba36a831..328fd3054 100644 --- a/packages/evolution/src/pipeline.ts +++ b/packages/evolution/src/pipeline.ts @@ -85,7 +85,7 @@ function isProposal(artifact: EvolutionArtifact): artifact is EvolutionProposal } function isScoredProposal(artifact: EvolutionArtifact): artifact is ScoredProposal { - return "proposal" in artifact && "score" in artifact; + return "proposal" in artifact && "evidenceRankingScore" in artifact; } function isMutationBatch(artifact: EvolutionArtifact): artifact is MutationBatch { diff --git a/packages/evolution/src/proposal-ranking.ts b/packages/evolution/src/proposal-ranking.ts new file mode 100644 index 000000000..c451df612 --- /dev/null +++ b/packages/evolution/src/proposal-ranking.ts @@ -0,0 +1,956 @@ +import type { MaterializedEvidence, MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import type { + EvidenceCorpusDocument, + EvidenceRankingScoreComponents, + EvolutionArtifactProducer, + EvolutionMemoryDelta, + EvolutionMemoryProposal, + EvolutionProposal, + InstructionDocumentSnapshot, + Lesson, + ProposalValidationResult, + ScoredProposal, + ValidationIssue, + ValidationReport, +} from "./contracts.ts"; +import type { ProposalScorer, ProposalScoringInput, ProposalValidationInput, ProposalValidator } from "./ports.ts"; +import { computeEvolutionContentDigest, createEvolutionArtifactMetadata } from "./provenance.ts"; + +export const EVIDENCE_RANKING_SCORER_VERSION = "evidence-ranking-v1"; +export const EVIDENCE_RANKING_ASSURANCE = "provenance-recurrence-and-structure-only" as const; + +const PROPOSAL_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const DEFAULT_ELIGIBILITY_THRESHOLD = 40; +const DEFAULT_MAX_SELECTED_PROPOSALS = 5; +const DEFAULT_MINIMUM_INDEPENDENT_TASKS = 2; +const DEFAULT_BROAD_SCOPE_INDEPENDENT_TASKS = 3; +const DEFAULT_MAX_DELTA_BYTES = 8 * 1024; + +type EvidenceSignal = "deterministic-outcome" | "direct-user" | "model-self-assessment" | "observable" | "tool-outcome"; + +interface EvidenceContext { + readonly evidence: MaterializedEvidence; + readonly invocation: MaterializedInvocation; + readonly sourceId: string; + readonly materializationId: string; + readonly ingestionCursor: number; + readonly traceId: string | null; + readonly document: EvidenceCorpusDocument; + readonly taskLineageId: string | null; + readonly signal: EvidenceSignal; + readonly signalPoints: number; + readonly provenancePoints: number; +} + +interface ValidatedProposalContext { + readonly proposal: EvolutionProposal; + readonly supporting: readonly EvidenceContext[]; + readonly contradicting: readonly EvidenceContext[]; + readonly independentSupportCount: number; + readonly earliestSupportingTime: string; + readonly fingerprint: string; +} + +export interface DeterministicProposalRankerDependencies { + readonly version: string; +} + +export interface DeterministicProposalRankerOptions { + readonly eligibilityThreshold?: number; + readonly maxSelectedProposals?: number; + readonly minimumIndependentTasks?: number; + readonly broadScopeIndependentTasks?: number; + readonly maxDeltaBytes?: number; +} + +interface ResolvedOptions { + readonly eligibilityThreshold: number; + readonly maxSelectedProposals: number; + readonly minimumIndependentTasks: number; + readonly broadScopeIndependentTasks: number; + readonly maxDeltaBytes: number; +} + +function unique(values: readonly string[]): string[] { + return [...new Set(values)]; +} + +function sorted(values: readonly string[]): string[] { + return unique(values).sort((left, right) => left.localeCompare(right)); +} + +function sameSet(left: readonly string[], right: readonly string[]): boolean { + const sortedLeft = sorted(left); + const sortedRight = sorted(right); + return sortedLeft.length === sortedRight.length && sortedLeft.every((value, index) => value === sortedRight[index]); +} + +function clamp(value: number, minimum: number, maximum: number): number { + return Math.min(maximum, Math.max(minimum, value)); +} + +function positiveInteger(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback; + if (!Number.isSafeInteger(resolved) || resolved < 1) throw new Error(`${name} must be a positive integer`); + return resolved; +} + +function boundedScore(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback; + if (!Number.isFinite(resolved) || resolved < 0 || resolved > 100) { + throw new Error(`${name} must be a finite number from 0 through 100`); + } + return resolved; +} + +function normalizedText(value: string): string { + return value.normalize("NFKC").trim().replace(/\s+/g, " ").toLocaleLowerCase("en-US"); +} + +function issue( + proposalId: string, + code: string, + message: string, + severity: ValidationIssue["severity"] = "error", +): ValidationIssue { + return { code, message, severity, proposalId }; +} + +function contentWithoutMetadata(artifact: T): Omit { + const { metadata: _metadata, ...content } = artifact; + return content; +} + +function taskLineageId(evidence: MaterializedEvidence): string | null { + const identifier = evidence.order.causalIdentifiers.find((candidate) => { + const name = candidate.name.toLocaleLowerCase("en-US"); + return name.includes("task") && name.includes("lineage"); + }); + return identifier?.value ?? null; +} + +function recordValue(value: unknown): Readonly> | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Readonly>) + : null; +} + +function evidenceSignal(evidence: MaterializedEvidence): { readonly signal: EvidenceSignal; readonly points: number } { + if (evidence.kind === "session_item") { + const payload = recordValue(evidence.payload); + const message = recordValue(payload?.message); + if (message?.role === "user") return { signal: "direct-user", points: 15 }; + if (message?.role === "toolResult" || evidence.itemType.toLocaleLowerCase("en-US").includes("tool")) { + return { signal: "tool-outcome", points: 9 }; + } + return { signal: "model-self-assessment", points: 3 }; + } + if (evidence.kind === "span") { + return evidence.payload.settled && evidence.payload.status !== null + ? { signal: "deterministic-outcome", points: 12 } + : { signal: "observable", points: 6 }; + } + if (evidence.payload.type === "trace_footer") return { signal: "deterministic-outcome", points: 12 }; + if (evidence.payload.type === "span_end" || evidence.payload.type === "span_event") { + return { signal: "tool-outcome", points: 9 }; + } + return { signal: "observable", points: 6 }; +} + +function evidenceProvenancePoints(evidence: MaterializedEvidence, invocation: MaterializedInvocation): number { + if (invocation.integrity === "complete") return 20; + if (invocation.integrity === "incompatible") return 0; + const invalidForBinding = invocation.diagnostics.some( + (diagnostic) => + diagnostic.bindingId !== undefined && + diagnostic.bindingId === evidence.sessionSource?.bindingId && + [ + "session_changed", + "session_incompatible", + "session_invalid", + "session_missing", + "session_truncated", + ].includes(diagnostic.code), + ); + if (invalidForBinding) return 0; + const beforeVerifiedPrefix = invocation.diagnostics.some( + (diagnostic) => + diagnostic.recoverable === false && + diagnostic.traceSequence !== undefined && + evidence.order.traceSequence >= diagnostic.traceSequence, + ); + return beforeVerifiedPrefix ? 0 : 12; +} + +function evidenceIndex(input: ProposalValidationInput): { + readonly index: ReadonlyMap; + readonly globalIssues: readonly ValidationIssue[]; +} { + const index = new Map(); + const globalIssues: ValidationIssue[] = []; + const documents = new Map(input.corpus.documents.map((document) => [document.sourceId, document])); + for (const source of input.input.evidenceSources) { + const document = documents.get(source.sourceId); + if (document === undefined || document.materializationId !== source.materializationId) { + globalIssues.push({ + code: "materialization-registry-mismatch", + message: `Evidence source "${source.sourceId}" does not match the materializer-owned corpus registry`, + severity: "error", + }); + continue; + } + if ( + source.origin !== "product-runtime" || + source.ingestionCursor > input.input.ingestionHighWaterCursor || + source.contentDigest !== computeEvolutionContentDigest(source.invocation) || + document.contentDigest !== source.contentDigest || + document.integrity !== source.invocation.integrity || + document.traceId !== source.invocation.traceId || + document.materializedInvocationVersion !== source.invocation.version + ) { + globalIssues.push({ + code: "materialization-registry-mismatch", + message: `Evidence source "${source.sourceId}" failed frozen materialization validation`, + severity: "error", + }); + continue; + } + for (const evidence of source.invocation.evidence) { + if (!document.evidenceIds.includes(evidence.evidenceId) || index.has(evidence.evidenceId)) { + globalIssues.push({ + code: index.has(evidence.evidenceId) ? "duplicate-evidence-id" : "materialization-registry-mismatch", + message: `Evidence "${evidence.evidenceId}" is not uniquely registered by its materialization`, + severity: "error", + }); + continue; + } + const signal = evidenceSignal(evidence); + index.set(evidence.evidenceId, { + evidence, + invocation: source.invocation, + sourceId: source.sourceId, + materializationId: source.materializationId, + ingestionCursor: source.ingestionCursor, + traceId: source.invocation.traceId, + document, + taskLineageId: taskLineageId(evidence), + signal: signal.signal, + signalPoints: signal.points, + provenancePoints: evidenceProvenancePoints(evidence, source.invocation), + }); + } + } + return { index, globalIssues }; +} + +function validateInstructionSnapshots(input: ProposalValidationInput): ValidationIssue[] { + const issues: ValidationIssue[] = []; + if (input.input.instructions.length !== input.instructions.documents.length) { + issues.push({ + code: "instruction-snapshot-stale", + message: "Applicable instruction paths changed after curation", + severity: "error", + }); + return issues; + } + for (let index = 0; index < input.input.instructions.length; index += 1) { + const frozen = input.input.instructions[index]; + const current = input.instructions.documents[index]; + if ( + frozen === undefined || + current === undefined || + frozen.path !== current.path || + frozen.precedence !== current.precedence || + frozen.content !== current.content || + frozen.contentDigest !== current.contentDigest || + computeEvolutionContentDigest(current.content) !== current.contentDigest + ) { + issues.push({ + code: "instruction-snapshot-stale", + message: `Applicable instruction at precedence position ${index} changed after curation`, + severity: "error", + }); + } + } + return issues; +} + +function proposalFingerprint(proposal: EvolutionProposal): string { + return computeEvolutionContentDigest({ + target: proposal.target.kind, + action: proposal.action, + delta: proposal.exactDelta, + }); +} + +function removedMemoryItemIds(delta: EvolutionMemoryDelta): readonly string[] { + if (delta.operation === "ADD") return []; + return [delta.operation === "REMOVE" ? delta.memoryItemId : delta.removal.memoryItemId]; +} + +function addContents(delta: EvolutionMemoryDelta): readonly string[] { + if (delta.operation === "REMOVE") return []; + return [delta.operation === "ADD" ? delta.content : delta.addition.content]; +} + +function isBroadScope(proposal: EvolutionMemoryProposal): boolean { + const scopes = + proposal.exactDelta.operation === "ADD" + ? proposal.exactDelta.intendedScope + : proposal.exactDelta.operation === "REPLACE" + ? proposal.exactDelta.addition.intendedScope + : proposal.scope.applicabilityBoundaries; + return scopes.some((value) => /\bglobal\b|repository[- ]wide|all repositories|all tasks/i.test(value)); +} + +function validatesInstructionReference( + documents: readonly InstructionDocumentSnapshot[], + proposal: EvolutionProposal, +): ValidationIssue[] { + const issues: ValidationIssue[] = []; + for (const reference of proposal.instructionRelationship.relevantEntries) { + if (reference.kind === "applicable-instruction") { + const document = documents.find((candidate) => candidate.path === reference.path); + if ( + document === undefined || + document.precedence !== reference.precedence || + document.contentDigest !== reference.contentDigest + ) { + issues.push( + issue( + proposal.proposalId, + "instruction-reference-stale", + `Instruction reference "${reference.path}" is stale`, + ), + ); + } + } + } + return issues; +} + +function validateMemoryReference(input: ProposalValidationInput, proposal: EvolutionProposal): ValidationIssue[] { + const issues: ValidationIssue[] = []; + for (const reference of proposal.instructionRelationship.relevantEntries) { + if (reference.kind !== "evolution-memory") continue; + const digest = + reference.memoryItemId === null + ? input.input.harness.memoryContentDigest + : input.input.harness.activeMemoryItems.find((item) => item.memoryItemId === reference.memoryItemId) + ?.contentDigest; + if (digest === undefined || digest !== reference.contentDigest) { + issues.push(issue(proposal.proposalId, "memory-reference-stale", "Evolution memory reference is stale")); + } + } + return issues; +} + +function proposalShapeIssues( + input: ProposalValidationInput, + proposal: EvolutionProposal, + options: ResolvedOptions, +): ValidationIssue[] { + const issues: ValidationIssue[] = []; + if (!PROPOSAL_ID_PATTERN.test(proposal.proposalId) || proposal.metadata.artifactId !== proposal.proposalId) { + issues.push(issue(proposal.proposalId, "invalid-proposal-id", "Proposal identity is invalid or inconsistent")); + } + if ( + proposal.metadata.schemaVersion !== 1 || + proposal.metadata.scope.repositoryId !== input.input.metadata.scope.repositoryId || + proposal.metadata.scope.scopeId !== input.input.metadata.scope.scopeId || + proposal.metadata.contentDigest !== computeEvolutionContentDigest(contentWithoutMetadata(proposal)) + ) { + issues.push( + issue(proposal.proposalId, "proposal-schema-invalid", "Proposal metadata or content digest is invalid"), + ); + } + if ( + proposal.scope.repository.repositoryId !== input.input.metadata.scope.repositoryId || + proposal.scope.repository.scopeId !== input.input.metadata.scope.scopeId || + proposal.scope.repositoryRevision !== input.input.repository.revision || + proposal.lessonIds.length === 0 || + proposal.supportingEvidenceIds.length === 0 || + proposal.summary.trim().length === 0 || + proposal.expectedBenefit.trim().length === 0 || + proposal.acceptanceCriterion.trim().length === 0 || + proposal.risk.trim().length === 0 || + proposal.rollbackDescription.trim().length === 0 || + !Number.isFinite(proposal.curatorConfidence) || + proposal.curatorConfidence < 0 || + proposal.curatorConfidence > 1 + ) { + issues.push( + issue(proposal.proposalId, "proposal-schema-invalid", "Proposal required fields or scope are invalid"), + ); + } + if (proposal.disposition === "deferred") { + const targetKind: string = proposal.target.kind; + const action: string = proposal.action; + if ( + targetKind === "evolution-memory" || + action !== "DEFER" || + proposal.exactDelta !== null || + proposal.expectedBaseDigest !== null || + proposal.finding.trim().length === 0 + ) { + issues.push( + issue(proposal.proposalId, "unsupported-target", "Deferred proposal contains an executable writer route"), + ); + } + return issues; + } + const action: string = proposal.action; + if ( + proposal.target.kind !== "evolution-memory" || + action === "DEFER" || + action !== proposal.exactDelta.operation || + proposal.target.expectedHarnessVersion !== input.input.expectedHarnessBaseVersion || + proposal.expectedBaseDigest !== input.input.harness.memoryContentDigest + ) { + issues.push( + issue( + proposal.proposalId, + "unsupported-target", + "Actionable proposal does not target the frozen evolution memory base", + ), + ); + } + if (Buffer.byteLength(JSON.stringify(proposal.exactDelta), "utf8") > options.maxDeltaBytes) { + issues.push( + issue(proposal.proposalId, "delta-size-limit", "Proposal exact delta exceeds the deterministic size limit"), + ); + } + const exactDelta = proposal.exactDelta; + for (const memoryItemId of removedMemoryItemIds(exactDelta)) { + const item = input.input.harness.activeMemoryItems.find((candidate) => candidate.memoryItemId === memoryItemId); + const expectedDigest = + exactDelta.operation === "REMOVE" + ? exactDelta.expectedContentDigest + : exactDelta.operation === "REPLACE" + ? exactDelta.removal.expectedContentDigest + : null; + if (expectedDigest === null || item === undefined || item.contentDigest !== expectedDigest) { + issues.push( + issue( + proposal.proposalId, + "memory-item-digest-stale", + `Memory item "${memoryItemId}" changed after curation`, + ), + ); + } + } + for (const content of addContents(proposal.exactDelta)) { + if (input.input.harness.memory.includes(content)) { + issues.push( + issue(proposal.proposalId, "duplicate-content-exact", "Proposed memory content already exists exactly"), + ); + } else if (normalizedText(input.input.harness.memory).includes(normalizedText(content))) { + issues.push( + issue( + proposal.proposalId, + "duplicate-content-normalized", + "Proposed memory content already exists after normalized-text comparison", + ), + ); + } + } + return issues; +} + +function lessonClosure( + input: ProposalValidationInput, + proposal: EvolutionProposal, + index: ReadonlyMap, +): { + readonly issues: readonly ValidationIssue[]; + readonly supporting: readonly EvidenceContext[]; + readonly contradicting: readonly EvidenceContext[]; +} { + const issues: ValidationIssue[] = []; + const lessonsById = new Map(input.reflection.lessons.map((lesson) => [lesson.lessonId, lesson])); + const finalLessonIds = new Set(input.reflection.finalLessonIds); + const lessons: Lesson[] = []; + for (const lessonId of proposal.lessonIds) { + const lesson = lessonsById.get(lessonId); + if (lesson === undefined || !finalLessonIds.has(lessonId)) { + issues.push( + issue(proposal.proposalId, "lesson-closure-invalid", `Lesson "${lessonId}" is unavailable or not final`), + ); + continue; + } + if ( + lesson.scope.repositoryId !== input.input.metadata.scope.repositoryId || + lesson.scope.scopeId !== input.input.metadata.scope.scopeId + ) { + issues.push( + issue(proposal.proposalId, "lesson-scope-invalid", `Lesson "${lessonId}" is outside the frozen scope`), + ); + } + lessons.push(lesson); + } + const expectedSupporting = unique(lessons.flatMap((lesson) => lesson.evidenceIds)); + const expectedContradicting = unique(lessons.flatMap((lesson) => lesson.counterevidenceIds)); + if ( + !sameSet(proposal.supportingEvidenceIds, expectedSupporting) || + !sameSet(proposal.contradictingEvidenceIds, expectedContradicting) + ) { + issues.push( + issue( + proposal.proposalId, + "evidence-closure-invalid", + "Proposal evidence does not close exactly through its lessons", + ), + ); + } + const resolve = (evidenceIds: readonly string[], role: "supporting" | "contradicting"): EvidenceContext[] => + evidenceIds.flatMap((evidenceId) => { + const context = index.get(evidenceId); + if (context === undefined) { + issues.push( + issue( + proposal.proposalId, + "evidence-unverifiable", + `${role} evidence "${evidenceId}" is not in the materializer registry`, + ), + ); + return []; + } + if (context.provenancePoints === 0) { + issues.push( + issue( + proposal.proposalId, + "evidence-quarantined", + `${role} evidence "${evidenceId}" is outside a verified materialization prefix`, + ), + ); + } + if (context.ingestionCursor > input.input.ingestionHighWaterCursor) { + issues.push( + issue( + proposal.proposalId, + "evidence-after-cutoff", + `${role} evidence "${evidenceId}" is after the frozen job cutoff`, + ), + ); + } + if ( + context.evidence.order.timestamp !== null && + context.evidence.order.timestamp > input.input.metadata.createdAt + ) { + issues.push( + issue( + proposal.proposalId, + "evidence-after-cutoff", + `${role} evidence "${evidenceId}" did not exist at the frozen job cutoff`, + ), + ); + } + return [context]; + }); + return { + issues, + supporting: resolve(proposal.supportingEvidenceIds, "supporting"), + contradicting: resolve(proposal.contradictingEvidenceIds, "contradicting"), + }; +} + +function strongestSignalsByTask(contexts: readonly EvidenceContext[]): ReadonlyMap { + const result = new Map(); + for (const context of contexts) { + if (context.taskLineageId === null) continue; + result.set(context.taskLineageId, Math.max(result.get(context.taskLineageId) ?? 0, context.signalPoints)); + } + return result; +} + +function provenanceAndRecurrenceIssues( + proposal: EvolutionProposal, + supporting: readonly EvidenceContext[], + contradicting: readonly EvidenceContext[], + options: ResolvedOptions, +): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const supportSignals = strongestSignalsByTask(supporting); + const contradictionSignals = strongestSignalsByTask(contradicting); + const hasDirectUserSupport = supporting.some((context) => context.signal === "direct-user"); + const requiredTasks = + proposal.disposition === "actionable" && isBroadScope(proposal) + ? options.broadScopeIndependentTasks + : options.minimumIndependentTasks; + if (supportSignals.size < requiredTasks && !(supportSignals.size === 1 && hasDirectUserSupport)) { + issues.push( + issue( + proposal.proposalId, + "insufficient-independent-support", + `Proposal requires ${requiredTasks} independent supporting tasks`, + ), + ); + } + const supportStrength = [...supportSignals.values()].reduce((sum, points) => sum + points, 0); + const contradictionStrength = [...contradictionSignals.values()].reduce((sum, points) => sum + points, 0); + if (contradictionStrength >= supportStrength && contradictionStrength > 0) { + issues.push( + issue( + proposal.proposalId, + "counterevidence-not-resolved", + "Independent counterevidence is equal to or stronger than support", + ), + ); + } + return issues; +} + +function scopeIssues(proposal: EvolutionProposal, contexts: readonly EvidenceContext[]): ValidationIssue[] { + const materializationIds = unique(contexts.map((context) => context.materializationId)); + const traceIds = unique(contexts.flatMap((context) => (context.traceId === null ? [] : [context.traceId]))); + const sessionIds = unique(contexts.flatMap((context) => context.evidence.sessionSource?.sessionId ?? [])); + const taskLineageIds = unique(contexts.flatMap((context) => context.taskLineageId ?? [])); + const invocations = [...new Set(contexts.map((context) => context.invocation))]; + const runtimeKinds = unique(invocations.flatMap((invocation) => invocation.producer?.runtime.kind ?? [])); + const runtimeVersions = unique(invocations.flatMap((invocation) => invocation.producer?.runtime.version ?? [])); + const sourceRevisions = unique(invocations.flatMap((invocation) => invocation.producer?.sourceRevision ?? [])); + const fields: readonly [string, readonly string[], readonly string[]][] = [ + ["materialization", proposal.scope.materializationIds, materializationIds], + ["trace", proposal.scope.traceIds, traceIds], + ["session", proposal.scope.sessionIds, sessionIds], + ["task lineage", proposal.scope.taskLineageIds, taskLineageIds], + ["runtime kind", proposal.scope.runtimeKinds, runtimeKinds], + ["runtime version", proposal.scope.runtimeVersions, runtimeVersions], + ["source revision", proposal.scope.sourceRevisions, sourceRevisions], + ]; + return fields.flatMap(([name, actual, expected]) => + sameSet(actual, expected) + ? [] + : [ + issue( + proposal.proposalId, + "proposal-scope-invalid", + `Proposal ${name} scope does not match cited evidence`, + ), + ], + ); +} + +function operationConflicts(left: EvolutionProposal, right: EvolutionProposal): boolean { + if ( + left.conflictingProposalIds.includes(right.proposalId) || + right.conflictingProposalIds.includes(left.proposalId) + ) { + return true; + } + if (left.disposition === "deferred" || right.disposition === "deferred") return false; + const leftRemoved = new Set(removedMemoryItemIds(left.exactDelta)); + return removedMemoryItemIds(right.exactDelta).some((memoryItemId) => leftRemoved.has(memoryItemId)); +} + +function duplicateAndConflictIssues( + proposals: readonly EvolutionProposal[], + proposal: EvolutionProposal, +): ValidationIssue[] { + const issues: ValidationIssue[] = []; + for (const candidate of proposals) { + if (candidate.proposalId === proposal.proposalId) continue; + if (proposalFingerprint(candidate) === proposalFingerprint(proposal)) { + issues.push( + issue( + proposal.proposalId, + "duplicate-operation", + `Proposal duplicates operation "${candidate.proposalId}"`, + "warning", + ), + ); + } else if (operationConflicts(proposal, candidate)) { + issues.push( + issue( + proposal.proposalId, + "operation-conflict", + `Proposal conflicts with "${candidate.proposalId}"`, + "warning", + ), + ); + } + } + return issues; +} + +function validationDisposition( + proposal: EvolutionProposal, + issues: readonly ValidationIssue[], +): ProposalValidationResult["disposition"] { + if (proposal.disposition === "deferred" && !issues.some((candidate) => candidate.severity === "error")) + return "deferred"; + if ( + issues.some( + (candidate) => + candidate.code === "evidence-quarantined" || candidate.code === "materialization-registry-mismatch", + ) + ) { + return "quarantined"; + } + return issues.some((candidate) => candidate.severity === "error") ? "rejected" : "accepted"; +} + +function validationContext( + input: ProposalValidationInput, + proposal: EvolutionProposal, + index: ReadonlyMap, + options: ResolvedOptions, + globalIssues: readonly ValidationIssue[], +): { readonly result: ProposalValidationResult; readonly context: ValidatedProposalContext } { + const closure = lessonClosure(input, proposal, index); + const contexts = [...closure.supporting, ...closure.contradicting]; + const issues = [ + ...globalIssues.map((candidate) => ({ ...candidate, proposalId: proposal.proposalId })), + ...proposalShapeIssues(input, proposal, options), + ...validatesInstructionReference(input.instructions.documents, proposal), + ...validateMemoryReference(input, proposal), + ...closure.issues, + ...scopeIssues(proposal, contexts), + ...provenanceAndRecurrenceIssues(proposal, closure.supporting, closure.contradicting, options), + ...duplicateAndConflictIssues(input.proposals, proposal), + ]; + const disposition = validationDisposition(proposal, issues); + return { + result: { + proposalId: proposal.proposalId, + disposition, + reasonCodes: unique(issues.map((candidate) => candidate.code)), + issues, + }, + context: { + proposal, + supporting: closure.supporting, + contradicting: closure.contradicting, + independentSupportCount: strongestSignalsByTask(closure.supporting).size, + earliestSupportingTime: + sorted(closure.supporting.flatMap((context) => context.evidence.order.timestamp ?? []))[0] ?? + "9999-12-31T23:59:59.999Z", + fingerprint: proposalFingerprint(proposal), + }, + }; +} + +function scoreComponents(context: ValidatedProposalContext): EvidenceRankingScoreComponents { + const supportSignals = [...strongestSignalsByTask(context.supporting).values()]; + const contradictionSignals = [...strongestSignalsByTask(context.contradicting).values()]; + const supportStrength = supportSignals.reduce((sum, value) => sum + value, 0); + const contradictionStrength = contradictionSignals.reduce((sum, value) => sum + value, 0); + const uniqueMaterializations = new Set(context.supporting.map((item) => item.materializationId)).size; + const uniqueSourceRevisions = new Set( + context.supporting.flatMap((item) => item.invocation.producer?.sourceRevision ?? []), + ).size; + const proposal = context.proposal; + return { + recurrence: Math.min(30, context.independentSupportCount * 10), + provenanceIntegrity: + context.supporting.length === 0 + ? 0 + : Math.round( + context.supporting.reduce((sum, item) => sum + item.provenancePoints, 0) / context.supporting.length, + ), + sourceSignalStrength: supportSignals.length === 0 ? 0 : Math.max(...supportSignals), + scopeCoverage: Math.min(15, 5 + uniqueMaterializations * 5 + Math.min(5, uniqueSourceRevisions * 5)), + curatorConfidence: Math.round(clamp(proposal.curatorConfidence, 0, 1) * 10), + proposalCompleteness: 10, + contradictionPenalty: + supportStrength === 0 + ? contradictionStrength > 0 + ? 40 + : 0 + : Math.min(40, Math.round((contradictionStrength / supportStrength) * 40)), + }; +} + +function finalScore(components: EvidenceRankingScoreComponents): number { + return clamp( + components.recurrence + + components.provenanceIntegrity + + components.sourceSignalStrength + + components.scopeCoverage + + components.curatorConfidence + + components.proposalCompleteness - + components.contradictionPenalty, + 0, + 100, + ); +} + +function compareRanked( + left: { readonly context: ValidatedProposalContext; readonly score: number }, + right: { readonly context: ValidatedProposalContext; readonly score: number }, +): number { + return ( + right.score - left.score || + right.context.independentSupportCount - left.context.independentSupportCount || + left.context.earliestSupportingTime.localeCompare(right.context.earliestSupportingTime) || + left.context.proposal.proposalId.localeCompare(right.context.proposal.proposalId) + ); +} + +export class DeterministicProposalRanker implements ProposalValidator, ProposalScorer { + readonly #dependencies: DeterministicProposalRankerDependencies; + readonly #options: ResolvedOptions; + + constructor( + dependencies: DeterministicProposalRankerDependencies, + options: DeterministicProposalRankerOptions = {}, + ) { + this.#dependencies = dependencies; + this.#options = { + eligibilityThreshold: boundedScore( + options.eligibilityThreshold, + DEFAULT_ELIGIBILITY_THRESHOLD, + "eligibilityThreshold", + ), + maxSelectedProposals: positiveInteger( + options.maxSelectedProposals, + DEFAULT_MAX_SELECTED_PROPOSALS, + "maxSelectedProposals", + ), + minimumIndependentTasks: positiveInteger( + options.minimumIndependentTasks, + DEFAULT_MINIMUM_INDEPENDENT_TASKS, + "minimumIndependentTasks", + ), + broadScopeIndependentTasks: positiveInteger( + options.broadScopeIndependentTasks, + DEFAULT_BROAD_SCOPE_INDEPENDENT_TASKS, + "broadScopeIndependentTasks", + ), + maxDeltaBytes: positiveInteger(options.maxDeltaBytes, DEFAULT_MAX_DELTA_BYTES, "maxDeltaBytes"), + }; + } + + async validate(input: ProposalValidationInput): Promise { + input.signal?.throwIfAborted(); + const instructionIssues = validateInstructionSnapshots(input); + const frozenInputIssues: ValidationIssue[] = []; + if (computeEvolutionContentDigest(input.input.harness.memory) !== input.input.harness.memoryContentDigest) { + frozenInputIssues.push({ + code: "memory-base-digest-stale", + message: "Frozen EVOLVE.md content digest is invalid", + severity: "error", + }); + } + const evidence = evidenceIndex(input); + const globalIssues = [...instructionIssues, ...frozenInputIssues, ...evidence.globalIssues]; + const proposalResults = input.proposals.map( + (proposal) => validationContext(input, proposal, evidence.index, this.#options, globalIssues).result, + ); + const producer: EvolutionArtifactProducer = { + name: "diffex-evolution-validator", + version: this.#dependencies.version, + promptVersion: input.input.promptVersions.validator, + model: null, + }; + const content = { + validatorVersion: EVIDENCE_RANKING_SCORER_VERSION, + assurance: EVIDENCE_RANKING_ASSURANCE, + proposalResults, + }; + const artifactId = `validation-${computeEvolutionContentDigest({ input: input.input.metadata.contentDigest, content }).slice("sha256:".length, "sha256:".length + 32)}`; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId, + producer, + sourceIds: input.proposals.map((proposal) => proposal.proposalId), + scope: input.input.metadata.scope, + createdAt: input.input.metadata.createdAt, + content, + }), + ...content, + }; + } + + async score(input: ProposalScoringInput): Promise { + input.signal?.throwIfAborted(); + const evidence = evidenceIndex(input); + const validationById = new Map(input.validation.proposalResults.map((result) => [result.proposalId, result])); + const contexts = new Map( + input.proposals.map((proposal) => [ + proposal.proposalId, + validationContext(input, proposal, evidence.index, this.#options, evidence.globalIssues).context, + ]), + ); + const ranked = input.proposals + .flatMap((proposal) => { + const validation = validationById.get(proposal.proposalId); + const context = contexts.get(proposal.proposalId); + if (validation?.disposition !== "accepted" || context === undefined) return []; + const components = scoreComponents(context); + return [{ context, components, score: finalScore(components) }]; + }) + .sort(compareRanked); + const representatives = new Map(); + for (const item of ranked) + if (!representatives.has(item.context.fingerprint)) representatives.set(item.context.fingerprint, item); + const selected: (typeof ranked)[number][] = []; + for (const candidate of representatives.values()) { + if (candidate.score < this.#options.eligibilityThreshold) continue; + if (selected.length >= this.#options.maxSelectedProposals) continue; + if (selected.some((existing) => operationConflicts(candidate.context.proposal, existing.context.proposal))) + continue; + selected.push(candidate); + } + const selectedIds = new Set(selected.map((item) => item.context.proposal.proposalId)); + const representativeIds = new Set([...representatives.values()].map((item) => item.context.proposal.proposalId)); + const rankedById = new Map(ranked.map((item) => [item.context.proposal.proposalId, item])); + const producer: EvolutionArtifactProducer = { + name: "diffex-evolution-scorer", + version: this.#dependencies.version, + promptVersion: input.input.promptVersions.validator, + model: null, + }; + return input.proposals.map((proposal): ScoredProposal => { + const validation = validationById.get(proposal.proposalId); + const rankedItem = rankedById.get(proposal.proposalId); + let disposition: ScoredProposal["disposition"]; + let reasonCodes: readonly string[]; + if (validation === undefined) { + disposition = "rejected"; + reasonCodes = ["validation-result-missing"]; + } else if (validation.disposition !== "accepted") { + disposition = validation.disposition; + reasonCodes = validation.reasonCodes.length === 0 ? [validation.disposition] : validation.reasonCodes; + } else if (rankedItem === undefined) { + disposition = "rejected"; + reasonCodes = ["scoring-context-missing"]; + } else if (!representativeIds.has(proposal.proposalId)) { + disposition = "superseded"; + reasonCodes = ["exact-proposal-fingerprint-collapsed"]; + } else if (selectedIds.has(proposal.proposalId)) { + disposition = "selected"; + reasonCodes = ["selected-by-evidence-ranking"]; + } else { + disposition = "unselected"; + reasonCodes = + rankedItem.score < this.#options.eligibilityThreshold + ? ["below-eligibility-threshold"] + : selected.length >= this.#options.maxSelectedProposals + ? ["selection-limit-reached"] + : ["conflicts-with-higher-ranked-proposal"]; + } + const content = { + proposalId: proposal.proposalId, + scorerVersion: EVIDENCE_RANKING_SCORER_VERSION, + evidenceRankingScore: rankedItem?.score ?? null, + scoreComponents: rankedItem?.components ?? null, + disposition, + reasonCodes, + }; + const artifactId = `score-${computeEvolutionContentDigest({ validation: input.validation.metadata.contentDigest, content }).slice("sha256:".length, "sha256:".length + 32)}`; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId, + producer, + sourceIds: [proposal.proposalId, input.validation.metadata.artifactId], + scope: input.input.metadata.scope, + createdAt: input.input.metadata.createdAt, + content, + }), + proposal, + ...content, + }; + }); + } +} diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index 414fa1d47..fe19496f6 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -504,6 +504,7 @@ describe("checkpointed evolution pipeline", () => { const proposalResults = input.proposals.map((proposal) => ({ proposalId: proposal.proposalId, disposition: "accepted" as const, + reasonCodes: [], issues: [], })); const report: ValidationReport = { @@ -514,6 +515,8 @@ describe("checkpointed evolution pipeline", () => { proposalResults, }, ), + validatorVersion: "fixture-validator-v1", + assurance: "provenance-recurrence-and-structure-only", proposalResults, }; return report; @@ -527,7 +530,17 @@ describe("checkpointed evolution pipeline", () => { disposition: "selected", }), proposal, - score: 100, + scorerVersion: "fixture-scorer-v1", + evidenceRankingScore: 100, + scoreComponents: { + recurrence: 30, + provenanceIntegrity: 20, + sourceSignalStrength: 15, + scopeCoverage: 15, + curatorConfidence: 10, + proposalCompleteness: 10, + contradictionPenalty: 0, + }, disposition: "selected", reasonCodes: ["selected"], }), diff --git a/packages/evolution/test/proposal-ranking.test.ts b/packages/evolution/test/proposal-ranking.test.ts new file mode 100644 index 000000000..b78c90536 --- /dev/null +++ b/packages/evolution/test/proposal-ranking.test.ts @@ -0,0 +1,482 @@ +import type { MaterializedEvidence, MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import { describe, expect, it } from "vitest"; +import { + computeEvolutionContentDigest, + createEvolutionArtifactMetadata, + DeterministicProposalRanker, + EVIDENCE_RANKING_ASSURANCE, + EVIDENCE_RANKING_SCORER_VERSION, + type EvidenceCorpusManifest, + type EvolutionArtifactProducer, + type EvolutionInputSnapshot, + type EvolutionMemoryProposal, + type InstructionSnapshot, + type Lesson, + type ProposalScoringInput, + type ProposalValidationInput, + type ReflectionResult, +} from "../src/index.ts"; + +const HARNESS_VERSION = `sha256:${"1".repeat(64)}`; +const MODEL_DIGEST = `sha256:${"2".repeat(64)}`; +const createdAt = "2026-08-19T12:10:00.000Z"; +const scope = { repositoryId: "diffex", scopeId: "repository" } as const; +const producer: EvolutionArtifactProducer = { name: "fixture", version: "1", promptVersion: null, model: null }; + +interface EvidenceFixture { + readonly evidenceId: string; + readonly lineageId: string; + readonly role?: "assistant" | "toolResult" | "user"; + readonly integrity?: MaterializedInvocation["integrity"]; + readonly diagnostics?: MaterializedInvocation["diagnostics"]; +} + +function metadata(artifactId: string, sourceIds: readonly string[], content: unknown) { + return createEvolutionArtifactMetadata({ artifactId, producer, sourceIds, scope, createdAt, content }); +} + +function evidence(fixture: EvidenceFixture, index: number): MaterializedEvidence { + return { + kind: "session_item", + evidenceId: fixture.evidenceId, + itemType: fixture.role === "toolResult" ? "tool_result" : "message", + payload: { + type: "message", + id: `entry-${fixture.evidenceId}`, + message: { role: fixture.role ?? "toolResult", content: fixture.evidenceId }, + }, + order: { + traceSequence: index + 2, + timestamp: new Date(Date.parse(createdAt) - 60_000 + index).toISOString(), + sessionPosition: index, + parentSpanId: null, + causalIdentifiers: [{ name: "diffex.task.lineage", value: fixture.lineageId }], + ambiguity: [], + }, + traceSources: [], + sessionSource: { + bindingId: `binding-${fixture.evidenceId}`, + sessionId: `session-${fixture.evidenceId}`, + path: `/private/${fixture.evidenceId}.jsonl`, + sessionFormat: "coding-agent-v3", + formatVersion: 3, + segmentTraceSequences: [index + 2], + entryId: `entry-${fixture.evidenceId}`, + entryOrdinal: index, + }, + }; +} + +function invocation(fixture: EvidenceFixture, index: number): MaterializedInvocation { + return { + version: 1, + tracePath: `/private/trace-${fixture.evidenceId}.jsonl`, + traceId: `trace-${fixture.evidenceId}`, + producer: { + diffexVersion: "0.2.0", + sourceRevision: "revision-1", + runtime: { kind: "node", version: "24.0.0" }, + telemetrySchemaVersions: {}, + metadata: {}, + }, + integrity: fixture.integrity ?? "complete", + terminationStatus: fixture.integrity === "partial" ? "crashed" : "completed", + health: null, + ordering: { + criteria: ["trace_sequence", "session_cursor", "timestamp", "span_parentage", "causal_id"], + ambiguity: [], + }, + evidence: [evidence(fixture, index)], + diagnostics: fixture.diagnostics ?? [], + }; +} + +function lesson(lessonId: string, supportIds: readonly string[], counterIds: readonly string[] = []): Lesson { + const content = { + lessonId, + kind: "successful-strategy" as const, + summary: `Lesson ${lessonId}`, + preconditions: ["Repository work"], + observedOutcome: "The behavior was observed", + evidenceIds: supportIds, + counterevidenceIds: counterIds, + scope, + applicabilityBoundaries: ["Repository work"], + uncertainty: null, + actionableTakeaway: "Keep the verified behavior", + }; + return { metadata: metadata(lessonId, [...supportIds, ...counterIds], content), ...content }; +} + +function proposal( + proposalId: string, + input: EvolutionInputSnapshot, + lessons: readonly Lesson[], + content = `Rule ${proposalId}`, + overrides: Partial = {}, +): EvolutionMemoryProposal { + const supportIds = [...new Set(lessons.flatMap((item) => item.evidenceIds))]; + const counterIds = [...new Set(lessons.flatMap((item) => item.counterevidenceIds))]; + const contexts = input.evidenceSources.flatMap((source) => + source.invocation.evidence + .filter((item) => supportIds.includes(item.evidenceId) || counterIds.includes(item.evidenceId)) + .map((item) => ({ source, item })), + ); + const exactDelta = { + operation: "ADD" as const, + destination: { section: "Engineering", anchor: null }, + content, + intendedScope: ["Repository work"], + }; + const body = { + proposalId, + summary: `Add ${content}`, + scope: { + repository: scope, + repositoryRevision: input.repository.revision, + materializationIds: [...new Set(contexts.map(({ source }) => source.materializationId))], + traceIds: [...new Set(contexts.flatMap(({ source }) => source.invocation.traceId ?? []))], + sessionIds: [...new Set(contexts.flatMap(({ item }) => item.sessionSource?.sessionId ?? []))], + taskLineageIds: [ + ...new Set( + contexts.flatMap(({ item }) => + item.order.causalIdentifiers + .filter((identifier) => identifier.name === "diffex.task.lineage") + .map((identifier) => identifier.value), + ), + ), + ], + runtimeKinds: ["node"], + runtimeVersions: ["24.0.0"], + sourceRevisions: ["revision-1"], + harnessVersions: [], + applicabilityBoundaries: ["Repository work"], + }, + lessonIds: lessons.map((item) => item.lessonId), + supportingEvidenceIds: supportIds, + contradictingEvidenceIds: counterIds, + supportingTraceIds: [ + ...new Set( + contexts + .filter(({ item }) => supportIds.includes(item.evidenceId)) + .flatMap(({ source }) => source.invocation.traceId ?? []), + ), + ], + contradictingTraceIds: [ + ...new Set( + contexts + .filter(({ item }) => counterIds.includes(item.evidenceId)) + .flatMap(({ source }) => source.invocation.traceId ?? []), + ), + ], + expectedBenefit: "Future behavior is more reliable", + acceptanceCriterion: "The exact rule appears once", + risk: "The scope may be too broad", + rollbackDescription: "Remove the exact rule", + curatorConfidence: 0.8, + uncertainty: null, + counterexamples: [], + conflictingProposalIds: [], + instructionRelationship: { classification: "novel" as const, relevantEntries: [], strengthening: null }, + disposition: "actionable" as const, + target: { kind: "evolution-memory" as const, expectedHarnessVersion: input.expectedHarnessBaseVersion }, + action: "ADD" as const, + exactDelta, + expectedBaseDigest: input.harness.memoryContentDigest, + }; + const merged = { ...body, ...overrides }; + return { + metadata: createEvolutionArtifactMetadata({ + artifactId: proposalId, + producer, + sourceIds: [...merged.lessonIds, ...merged.supportingEvidenceIds, ...merged.contradictingEvidenceIds], + scope, + createdAt, + content: merged, + }), + ...merged, + }; +} + +function fixture(evidenceFixtures: readonly EvidenceFixture[]) { + const evidenceSources = evidenceFixtures.map((item, index) => { + const materialized = invocation(item, index); + return { + sourceId: `source-${item.evidenceId}`, + materializationId: `materialization-${item.evidenceId}`, + ingestionCursor: index + 1, + origin: "product-runtime" as const, + contentDigest: computeEvolutionContentDigest(materialized), + invocation: materialized, + }; + }); + const memory = "# Evolution memory\n"; + const instructionContent = "Repository guidance.\n"; + const input: EvolutionInputSnapshot = { + metadata: metadata( + "input-1", + evidenceSources.map((source) => source.sourceId), + { evidenceSources: evidenceSources.map((source) => source.sourceId) }, + ), + trigger: "manual", + repository: { repositoryId: "diffex", rootPath: "/repo", revision: "revision-current" }, + harness: { + harnessVersion: HARNESS_VERSION, + baseVersion: null, + memory, + memoryContentDigest: computeEvolutionContentDigest(memory), + activeMemoryItems: [], + }, + expectedHarnessBaseVersion: HARNESS_VERSION, + ingestionHighWaterCursor: evidenceSources.length, + instructions: [ + { + path: "/repo/AGENTS.md", + precedence: 10, + content: instructionContent, + contentDigest: computeEvolutionContentDigest(instructionContent), + }, + ], + modelConfiguration: { provider: "faux", model: "fixture", configurationDigest: MODEL_DIGEST }, + pipelineVersion: "pipeline-v1", + promptVersions: { + reflector: "reflector-v1", + curator: "curator-v1", + validator: "validator-v1", + writer: "writer-v1", + }, + tokenMeasurement: { estimatorVersion: "fixture", estimatedTokens: 100 }, + evidenceSources, + }; + const documents = evidenceSources.map((source) => ({ + documentId: `document-${source.materializationId}`, + sourceId: source.sourceId, + materializationId: source.materializationId, + contentDigest: source.contentDigest, + materializedInvocationVersion: source.invocation.version, + traceId: source.invocation.traceId, + integrity: source.invocation.integrity, + terminationStatus: source.invocation.terminationStatus, + evidenceIds: source.invocation.evidence.map((item) => item.evidenceId), + semanticUnits: [], + diagnostics: source.invocation.diagnostics.map((diagnostic) => ({ + code: diagnostic.code, + message: diagnostic.message, + recoverable: diagnostic.recoverable, + traceSequence: diagnostic.traceSequence ?? null, + bindingId: diagnostic.bindingId ?? null, + sourceCode: diagnostic.sourceCode ?? null, + })), + orderingAmbiguity: [], + })); + const corpus: EvidenceCorpusManifest = { + metadata: metadata( + "corpus-1", + evidenceSources.map((source) => source.sourceId), + { documents, excludedSources: [] }, + ), + documents, + excludedSources: [], + }; + const instructions: InstructionSnapshot = { + metadata: metadata("instructions-1", [input.metadata.artifactId], input.instructions), + documents: input.instructions, + }; + return { input, corpus, instructions }; +} + +function validationInput( + fixtureValue: ReturnType, + lessons: readonly Lesson[], + proposals: readonly EvolutionMemoryProposal[], +): ProposalValidationInput { + const reflection: ReflectionResult = { + nodes: [], + lessons, + coverage: { + metadata: metadata( + "coverage-1", + fixtureValue.corpus.documents.map((document) => document.documentId), + { lessons: lessons.map((item) => item.lessonId) }, + ), + manifestDigest: fixtureValue.corpus.metadata.contentDigest, + entries: [], + rootNodeIds: [], + finalLessonIds: lessons.map((item) => item.lessonId), + }, + finalLessonIds: lessons.map((item) => item.lessonId), + }; + return { ...fixtureValue, reflection, proposals }; +} + +async function validateAndScore(ranker: DeterministicProposalRanker, input: ProposalValidationInput) { + const validation = await ranker.validate(input); + const scoringInput: ProposalScoringInput = { ...input, validation }; + return { validation, scored: await ranker.score(scoringInput) }; +} + +describe("deterministic proposal evidence ranking", () => { + it("reproduces identical validation, component scores, and selection", async () => { + const data = fixture([ + { evidenceId: "support-a", lineageId: "task-a" }, + { evidenceId: "support-b", lineageId: "task-b" }, + ]); + const lessons = [lesson("lesson-1", ["support-a", "support-b"])]; + const proposals = [proposal("proposal-a", data.input, lessons)]; + const input = validationInput(data, lessons, proposals); + const ranker = new DeterministicProposalRanker({ version: "1.2.3" }); + + const first = await validateAndScore(ranker, input); + const second = await validateAndScore(ranker, input); + + expect(first).toEqual(second); + expect(first.validation).toMatchObject({ + validatorVersion: EVIDENCE_RANKING_SCORER_VERSION, + assurance: EVIDENCE_RANKING_ASSURANCE, + proposalResults: [{ proposalId: "proposal-a", disposition: "accepted" }], + }); + expect(first.scored[0]).toMatchObject({ + disposition: "selected", + scorerVersion: EVIDENCE_RANKING_SCORER_VERSION, + scoreComponents: { + recurrence: 20, + provenanceIntegrity: 20, + sourceSignalStrength: 9, + curatorConfidence: 8, + proposalCompleteness: 10, + contradictionPenalty: 0, + }, + }); + expect(first.scored[0]?.evidenceRankingScore).toBeGreaterThanOrEqual(40); + }); + + it("rejects invented evidence and quarantines evidence outside a verified prefix", async () => { + const data = fixture([ + { evidenceId: "support-a", lineageId: "task-a", integrity: "partial" }, + { + evidenceId: "support-b", + lineageId: "task-b", + integrity: "partial", + diagnostics: [ + { + code: "session_changed", + message: "Session changed", + recoverable: false, + bindingId: "binding-support-b", + }, + ], + }, + ]); + const validPrefixLesson = lesson("lesson-prefix", ["support-a", "support-a"]); + const quarantinedLesson = lesson("lesson-quarantine", ["support-a", "support-b"]); + const validPrefixProposal = proposal("proposal-prefix", data.input, [validPrefixLesson]); + const quarantinedProposal = proposal("proposal-quarantine", data.input, [quarantinedLesson]); + const inventedProposal = proposal("proposal-invented", data.input, [validPrefixLesson], "Invented rule", { + supportingEvidenceIds: ["invented-evidence"], + }); + const input = validationInput( + data, + [validPrefixLesson, quarantinedLesson], + [validPrefixProposal, quarantinedProposal, inventedProposal], + ); + const report = await new DeterministicProposalRanker({ version: "1" }, { minimumIndependentTasks: 1 }).validate( + input, + ); + + expect(report.proposalResults).toEqual([ + expect.objectContaining({ proposalId: "proposal-prefix", disposition: "accepted" }), + expect.objectContaining({ + proposalId: "proposal-quarantine", + disposition: "quarantined", + reasonCodes: expect.arrayContaining(["evidence-quarantined"]), + }), + expect.objectContaining({ + proposalId: "proposal-invented", + disposition: "rejected", + reasonCodes: expect.arrayContaining(["evidence-closure-invalid", "evidence-unverifiable"]), + }), + ]); + }); + + it("deduplicates task lineages and allows one direct user instruction", async () => { + const data = fixture([ + { evidenceId: "repeat-a", lineageId: "same-task" }, + { evidenceId: "repeat-b", lineageId: "same-task" }, + { evidenceId: "user-direct", lineageId: "direct-task", role: "user" }, + ]); + const repeatedLesson = lesson("lesson-repeat", ["repeat-a", "repeat-b"]); + const directLesson = lesson("lesson-direct", ["user-direct"]); + const proposals = [ + proposal("proposal-repeat", data.input, [repeatedLesson]), + proposal("proposal-direct", data.input, [directLesson]), + ]; + const report = await new DeterministicProposalRanker({ version: "1" }).validate( + validationInput(data, [repeatedLesson, directLesson], proposals), + ); + + expect(report.proposalResults).toEqual([ + expect.objectContaining({ + proposalId: "proposal-repeat", + disposition: "rejected", + reasonCodes: expect.arrayContaining(["insufficient-independent-support"]), + }), + expect.objectContaining({ proposalId: "proposal-direct", disposition: "accepted" }), + ]); + }); + + it("rejects stale instructions, duplicate text, and unresolved strong counterevidence", async () => { + const data = fixture([ + { evidenceId: "support-a", lineageId: "task-a", role: "assistant" }, + { evidenceId: "support-b", lineageId: "task-b", role: "assistant" }, + { evidenceId: "counter-a", lineageId: "counter-task", role: "user" }, + ]); + const contradictoryLesson = lesson("lesson-1", ["support-a", "support-b"], ["counter-a"]); + const duplicate = proposal("proposal-duplicate", data.input, [contradictoryLesson], "evolution MEMORY"); + const staleInstructions: InstructionSnapshot = { + ...data.instructions, + documents: data.instructions.documents.map((document) => ({ ...document, content: "Changed guidance.\n" })), + }; + const input = validationInput({ ...data, instructions: staleInstructions }, [contradictoryLesson], [duplicate]); + const report = await new DeterministicProposalRanker({ version: "1" }).validate(input); + + expect(report.proposalResults[0]).toMatchObject({ + disposition: "rejected", + reasonCodes: expect.arrayContaining([ + "instruction-snapshot-stale", + "duplicate-content-normalized", + "counterevidence-not-resolved", + ]), + }); + }); + + it("selects at most five compatible eligible fingerprints without filling below threshold", async () => { + const data = fixture([ + { evidenceId: "support-a", lineageId: "task-a" }, + { evidenceId: "support-b", lineageId: "task-b" }, + ]); + const lessons = [lesson("lesson-1", ["support-a", "support-b"])]; + const proposals = Array.from({ length: 7 }, (_, index) => + proposal(`proposal-${String(index).padStart(2, "0")}`, data.input, lessons, `Distinct rule ${index}`), + ); + const duplicate = proposal("proposal-duplicate", data.input, lessons, "Distinct rule 0"); + const ranker = new DeterministicProposalRanker({ version: "1" }); + const result = await validateAndScore(ranker, validationInput(data, lessons, [...proposals, duplicate])); + + expect(result.scored.filter((item) => item.disposition === "selected")).toHaveLength(5); + expect(result.scored.find((item) => item.proposal.proposalId === "proposal-duplicate")).toMatchObject({ + disposition: "superseded", + reasonCodes: ["exact-proposal-fingerprint-collapsed"], + }); + expect(result.scored.filter((item) => item.disposition === "unselected")).toHaveLength(2); + + const strict = new DeterministicProposalRanker({ version: "1" }, { eligibilityThreshold: 100 }); + const belowThreshold = await validateAndScore( + strict, + validationInput(data, lessons, [proposals[0] as EvolutionMemoryProposal]), + ); + expect(belowThreshold.scored[0]).toMatchObject({ + disposition: "unselected", + reasonCodes: ["below-eligibility-threshold"], + }); + }); +}); From ab6d9459a91cde08edf6b56be993c8e17e1e3e67 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Wed, 19 Aug 2026 19:06:13 -0700 Subject: [PATCH 2/3] fix(evolution): harden ranking gates - Keeps accepted but ineligible proposals as a durable no-op instead of forcing a write - Verifies validation artifacts, citation closure, and the five-selection invariant at pipeline boundaries --- packages/evolution/src/pipeline.ts | 31 +++-- packages/evolution/src/proposal-ranking.ts | 124 ++++++++++++++--- .../evolution/test/proposal-ranking.test.ts | 125 ++++++++++++++++++ 3 files changed, 250 insertions(+), 30 deletions(-) diff --git a/packages/evolution/src/pipeline.ts b/packages/evolution/src/pipeline.ts index 328fd3054..5146d8a56 100644 --- a/packages/evolution/src/pipeline.ts +++ b/packages/evolution/src/pipeline.ts @@ -165,6 +165,12 @@ function assertScoredProposals( retryable: false, }); } + if (scoredProposals.filter((scored) => scored.disposition === "selected").length > 5) { + throw new EvolutionExecutionError("Scorer selected more than five proposal groups", { + reasonCode: "pipeline-invariant", + retryable: false, + }); + } } function assertValidationReport( @@ -186,15 +192,18 @@ function assertValidationReport( retryable: false, }); } - const accepted = new Set( - validation.proposalResults - .filter((result) => result.disposition === "accepted") - .map((result) => result.proposalId), - ); + const validationById = new Map(validation.proposalResults.map((result) => [result.proposalId, result])); if ( - scoredProposals.some((scored) => scored.disposition === "selected" && !accepted.has(scored.proposal.proposalId)) + scoredProposals.some((scored) => { + const validationResult = validationById.get(scored.proposal.proposalId); + if (validationResult === undefined) return true; + if (validationResult.disposition === "accepted") { + return !["selected", "superseded", "unselected"].includes(scored.disposition); + } + return scored.disposition !== validationResult.disposition; + }) ) { - throw new EvolutionExecutionError("Scorer selected a proposal that deterministic validation did not accept", { + throw new EvolutionExecutionError("Scorer disposition does not match deterministic validation", { reasonCode: "pipeline-invariant", retryable: false, }); @@ -543,12 +552,6 @@ export class EvolutionPipeline implements EvolutionStageExecutor { const selectedProposalIds = scoredProposals .filter((scored) => scored.disposition === "selected") .map((scored) => scored.proposal.proposalId); - if (acceptedProposalIds.length > 0 && selectedProposalIds.length === 0) { - throw new EvolutionExecutionError("Accepted proposals produced no writable selection", { - reasonCode: "validation-rejected", - retryable: false, - }); - } const ledgerEvents = scoredProposals.flatMap((scored): EvolutionLedgerEvent[] => { const disposition: EvolutionProposalLedgerDisposition | null = scored.disposition === "selected" ? null : scored.disposition; @@ -563,7 +566,7 @@ export class EvolutionPipeline implements EvolutionStageExecutor { acceptedProposalIds, selectedProposalIds, }, - directive: acceptedProposalIds.length === 0 ? "complete-no-op" : "continue", + directive: selectedProposalIds.length === 0 ? "complete-no-op" : "continue", }; } case "writing": { diff --git a/packages/evolution/src/proposal-ranking.ts b/packages/evolution/src/proposal-ranking.ts index c451df612..becc833b3 100644 --- a/packages/evolution/src/proposal-ranking.ts +++ b/packages/evolution/src/proposal-ranking.ts @@ -1,6 +1,5 @@ import type { MaterializedEvidence, MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; import type { - EvidenceCorpusDocument, EvidenceRankingScoreComponents, EvolutionArtifactProducer, EvolutionMemoryDelta, @@ -31,11 +30,9 @@ type EvidenceSignal = "deterministic-outcome" | "direct-user" | "model-self-asse interface EvidenceContext { readonly evidence: MaterializedEvidence; readonly invocation: MaterializedInvocation; - readonly sourceId: string; readonly materializationId: string; readonly ingestionCursor: number; readonly traceId: string | null; - readonly document: EvidenceCorpusDocument; readonly taskLineageId: string | null; readonly signal: EvidenceSignal; readonly signalPoints: number; @@ -85,6 +82,10 @@ function sameSet(left: readonly string[], right: readonly string[]): boolean { return sortedLeft.length === sortedRight.length && sortedLeft.every((value, index) => value === sortedRight[index]); } +function hasDuplicates(values: readonly string[]): boolean { + return new Set(values).size !== values.length; +} + function clamp(value: number, minimum: number, maximum: number): number { return Math.min(maximum, Math.max(minimum, value)); } @@ -95,6 +96,14 @@ function positiveInteger(value: number | undefined, fallback: number, name: stri return resolved; } +function selectionLimit(value: number | undefined): number { + const resolved = positiveInteger(value, DEFAULT_MAX_SELECTED_PROPOSALS, "maxSelectedProposals"); + if (resolved > DEFAULT_MAX_SELECTED_PROPOSALS) { + throw new Error(`maxSelectedProposals cannot exceed ${DEFAULT_MAX_SELECTED_PROPOSALS}`); + } + return resolved; +} + function boundedScore(value: number | undefined, fallback: number, name: string): number { const resolved = value ?? fallback; if (!Number.isFinite(resolved) || resolved < 0 || resolved > 100) { @@ -189,6 +198,21 @@ function evidenceIndex(input: ProposalValidationInput): { const index = new Map(); const globalIssues: ValidationIssue[] = []; const documents = new Map(input.corpus.documents.map((document) => [document.sourceId, document])); + if (documents.size !== input.corpus.documents.length) { + globalIssues.push({ + code: "materialization-registry-mismatch", + message: "Corpus contains duplicate evidence source registrations", + severity: "error", + }); + } + const sourceIds = new Set(input.input.evidenceSources.map((source) => source.sourceId)); + if (input.corpus.documents.some((document) => !sourceIds.has(document.sourceId))) { + globalIssues.push({ + code: "materialization-registry-mismatch", + message: "Corpus contains evidence outside the frozen input snapshot", + severity: "error", + }); + } for (const source of input.input.evidenceSources) { const document = documents.get(source.sourceId); if (document === undefined || document.materializationId !== source.materializationId) { @@ -228,11 +252,9 @@ function evidenceIndex(input: ProposalValidationInput): { index.set(evidence.evidenceId, { evidence, invocation: source.invocation, - sourceId: source.sourceId, materializationId: source.materializationId, ingestionCursor: source.ingestionCursor, traceId: source.invocation.traceId, - document, taskLineageId: taskLineageId(evidence), signal: signal.signal, signalPoints: signal.points, @@ -329,6 +351,34 @@ function validatesInstructionReference( return issues; } +function instructionDuplicateIssues( + documents: readonly InstructionDocumentSnapshot[], + proposal: EvolutionProposal, +): ValidationIssue[] { + if (proposal.disposition === "deferred") return []; + return addContents(proposal.exactDelta).flatMap((content) => { + if (documents.some((document) => document.content.includes(content))) { + return [ + issue( + proposal.proposalId, + "duplicate-instruction-exact", + "Proposed content already exists in an applicable instruction", + ), + ]; + } + const normalized = normalizedText(content); + return documents.some((document) => normalizedText(document.content).includes(normalized)) + ? [ + issue( + proposal.proposalId, + "duplicate-instruction-normalized", + "Proposed content duplicates an applicable instruction after normalization", + ), + ] + : []; + }); +} + function validateMemoryReference(input: ProposalValidationInput, proposal: EvolutionProposal): ValidationIssue[] { const issues: ValidationIssue[] = []; for (const reference of proposal.instructionRelationship.relevantEntries) { @@ -364,6 +414,16 @@ function proposalShapeIssues( issue(proposal.proposalId, "proposal-schema-invalid", "Proposal metadata or content digest is invalid"), ); } + if ( + hasDuplicates(proposal.lessonIds) || + hasDuplicates(proposal.supportingEvidenceIds) || + hasDuplicates(proposal.contradictingEvidenceIds) || + hasDuplicates(proposal.conflictingProposalIds) + ) { + issues.push( + issue(proposal.proposalId, "duplicate-proposal-reference", "Proposal contains duplicate identifiers"), + ); + } if ( proposal.scope.repository.repositoryId !== input.input.metadata.scope.repositoryId || proposal.scope.repository.scopeId !== input.input.metadata.scope.scopeId || @@ -486,6 +546,18 @@ function lessonClosure( issue(proposal.proposalId, "lesson-scope-invalid", `Lesson "${lessonId}" is outside the frozen scope`), ); } + if ( + lesson.metadata.artifactId !== lesson.lessonId || + lesson.metadata.contentDigest !== computeEvolutionContentDigest(contentWithoutMetadata(lesson)) + ) { + issues.push( + issue( + proposal.proposalId, + "lesson-digest-invalid", + `Lesson "${lessonId}" failed immutable artifact validation`, + ), + ); + } lessons.push(lesson); } const expectedSupporting = unique(lessons.flatMap((lesson) => lesson.evidenceIds)); @@ -547,10 +619,22 @@ function lessonClosure( } return [context]; }); + const supporting = resolve(proposal.supportingEvidenceIds, "supporting"); + const contradicting = resolve(proposal.contradictingEvidenceIds, "contradicting"); + const supportingTraceIds = unique(supporting.flatMap((context) => context.traceId ?? [])); + const contradictingTraceIds = unique(contradicting.flatMap((context) => context.traceId ?? [])); + if ( + !sameSet(proposal.supportingTraceIds, supportingTraceIds) || + !sameSet(proposal.contradictingTraceIds, contradictingTraceIds) + ) { + issues.push( + issue(proposal.proposalId, "trace-closure-invalid", "Proposal trace references do not match cited evidence"), + ); + } return { issues, - supporting: resolve(proposal.supportingEvidenceIds, "supporting"), - contradicting: resolve(proposal.contradictingEvidenceIds, "contradicting"), + supporting, + contradicting, }; } @@ -703,6 +787,7 @@ function validationContext( ...globalIssues.map((candidate) => ({ ...candidate, proposalId: proposal.proposalId })), ...proposalShapeIssues(input, proposal, options), ...validatesInstructionReference(input.instructions.documents, proposal), + ...instructionDuplicateIssues(input.instructions.documents, proposal), ...validateMemoryReference(input, proposal), ...closure.issues, ...scopeIssues(proposal, contexts), @@ -802,11 +887,7 @@ export class DeterministicProposalRanker implements ProposalValidator, ProposalS DEFAULT_ELIGIBILITY_THRESHOLD, "eligibilityThreshold", ), - maxSelectedProposals: positiveInteger( - options.maxSelectedProposals, - DEFAULT_MAX_SELECTED_PROPOSALS, - "maxSelectedProposals", - ), + maxSelectedProposals: selectionLimit(options.maxSelectedProposals), minimumIndependentTasks: positiveInteger( options.minimumIndependentTasks, DEFAULT_MINIMUM_INDEPENDENT_TASKS, @@ -865,7 +946,17 @@ export class DeterministicProposalRanker implements ProposalValidator, ProposalS async score(input: ProposalScoringInput): Promise { input.signal?.throwIfAborted(); const evidence = evidenceIndex(input); - const validationById = new Map(input.validation.proposalResults.map((result) => [result.proposalId, result])); + const expectedValidation = await this.validate(input); + if ( + input.validation.validatorVersion !== expectedValidation.validatorVersion || + input.validation.assurance !== expectedValidation.assurance || + computeEvolutionContentDigest(input.validation.proposalResults) !== + computeEvolutionContentDigest(expectedValidation.proposalResults) || + input.validation.metadata.contentDigest !== expectedValidation.metadata.contentDigest + ) { + throw new Error("Scorer received a validation report that does not match deterministic validation"); + } + const validationById = new Map(expectedValidation.proposalResults.map((result) => [result.proposalId, result])); const contexts = new Map( input.proposals.map((proposal) => [ proposal.proposalId, @@ -923,12 +1014,13 @@ export class DeterministicProposalRanker implements ProposalValidator, ProposalS reasonCodes = ["selected-by-evidence-ranking"]; } else { disposition = "unselected"; + const conflicts = selected.some((item) => operationConflicts(proposal, item.context.proposal)); reasonCodes = rankedItem.score < this.#options.eligibilityThreshold ? ["below-eligibility-threshold"] - : selected.length >= this.#options.maxSelectedProposals - ? ["selection-limit-reached"] - : ["conflicts-with-higher-ranked-proposal"]; + : conflicts + ? ["conflicts-with-higher-ranked-proposal"] + : ["selection-limit-reached"]; } const content = { proposalId: proposal.proposalId, diff --git a/packages/evolution/test/proposal-ranking.test.ts b/packages/evolution/test/proposal-ranking.test.ts index b78c90536..4868fc06b 100644 --- a/packages/evolution/test/proposal-ranking.test.ts +++ b/packages/evolution/test/proposal-ranking.test.ts @@ -9,7 +9,10 @@ import { type EvidenceCorpusManifest, type EvolutionArtifactProducer, type EvolutionInputSnapshot, + type EvolutionJob, type EvolutionMemoryProposal, + EvolutionPipeline, + EvolutionTargetAdapterRegistry, type InstructionSnapshot, type Lesson, type ProposalScoringInput, @@ -479,4 +482,126 @@ describe("deterministic proposal evidence ranking", () => { reasonCodes: ["below-eligibility-threshold"], }); }); + + it("rejects stale trace closure and duplicates against applicable instructions", async () => { + const data = fixture([ + { evidenceId: "support-a", lineageId: "task-a" }, + { evidenceId: "support-b", lineageId: "task-b" }, + ]); + const lessons = [lesson("lesson-1", ["support-a", "support-b"])]; + const staleTrace = proposal("proposal-trace", data.input, lessons, "Distinct guidance", { + supportingTraceIds: ["invented-trace"], + }); + const instructionDuplicate = proposal("proposal-instruction", data.input, lessons, "repository GUIDANCE."); + const report = await new DeterministicProposalRanker({ version: "1" }).validate( + validationInput(data, lessons, [staleTrace, instructionDuplicate]), + ); + + expect(report.proposalResults).toEqual([ + expect.objectContaining({ + proposalId: "proposal-trace", + disposition: "rejected", + reasonCodes: expect.arrayContaining(["trace-closure-invalid"]), + }), + expect.objectContaining({ + proposalId: "proposal-instruction", + disposition: "rejected", + reasonCodes: expect.arrayContaining(["duplicate-instruction-normalized"]), + }), + ]); + }); + + it("rejects altered validation reports and selection limits above five", async () => { + const data = fixture([ + { evidenceId: "support-a", lineageId: "task-a" }, + { evidenceId: "support-b", lineageId: "task-b" }, + ]); + const lessons = [lesson("lesson-1", ["support-a", "support-b"])]; + const proposals = [proposal("proposal-a", data.input, lessons)]; + const input = validationInput(data, lessons, proposals); + const ranker = new DeterministicProposalRanker({ version: "1" }); + const validation = await ranker.validate(input); + + await expect( + ranker.score({ + ...input, + validation: { + ...validation, + proposalResults: validation.proposalResults.map((result) => ({ ...result, disposition: "rejected" })), + }, + }), + ).rejects.toThrow("does not match deterministic validation"); + expect(() => new DeterministicProposalRanker({ version: "1" }, { maxSelectedProposals: 6 })).toThrow( + "cannot exceed 5", + ); + }); + + it("completes validation as a no-op when accepted proposals are below threshold", async () => { + const data = fixture([ + { evidenceId: "support-a", lineageId: "task-a" }, + { evidenceId: "support-b", lineageId: "task-b" }, + ]); + const lessons = [lesson("lesson-1", ["support-a", "support-b"])]; + const proposals = [proposal("proposal-a", data.input, lessons)]; + const validationInputValue = validationInput(data, lessons, proposals); + const ranker = new DeterministicProposalRanker({ version: "1" }, { eligibilityThreshold: 100 }); + const pipeline = new EvolutionPipeline({ + version: "1", + model: { execute: () => Promise.reject(new Error("Model execution is not expected")) }, + reflector: { reflect: () => Promise.reject(new Error("Reflection is not expected")) }, + curator: { curate: () => Promise.reject(new Error("Curation is not expected")) }, + validator: ranker, + scorer: ranker, + targets: new EvolutionTargetAdapterRegistry({ writers: [] }), + }); + const job: EvolutionJob = { + metadata: metadata("job-1", [data.input.metadata.artifactId], { stage: "validating" }), + jobId: "job-1", + inputSnapshot: { + artifactId: data.input.metadata.artifactId, + contentDigest: data.input.metadata.contentDigest, + }, + trigger: "manual", + evidenceRange: { afterCursor: 0, throughCursor: data.input.ingestionHighWaterCursor }, + stage: "validating", + outcome: "active", + reasonCode: null, + failureMessage: null, + completion: null, + resolution: null, + checkpoints: [], + attempt: 1, + revision: 1, + createdAt, + updatedAt: createdAt, + terminalAt: null, + retryNotBefore: null, + }; + + const result = await pipeline.execute({ + job, + input: data.input, + priorArtifacts: [ + data.corpus, + data.instructions, + ...lessons, + validationInputValue.reflection.coverage, + ...proposals, + ], + idempotencyKey: "job-1:validating", + scopeLease: { + kind: "scope", + key: "diffex:repository", + ownerId: "worker-1", + fence: 1, + expiresAt: "2026-08-19T12:20:00.000Z", + }, + writerLease: null, + }); + + expect(result).toMatchObject({ + directive: "complete-no-op", + state: { acceptedProposalIds: ["proposal-a"], selectedProposalIds: [] }, + }); + }); }); From a1edc82956fcd8051315635522f76f14edde7311 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Wed, 19 Aug 2026 19:07:33 -0700 Subject: [PATCH 3/3] fix(coding-agent): remove startup test race - Allows concurrently admitted children to reach the provider in either order - Continues to verify FIFO admission for queued children as permits become available --- packages/coding-agent/test/suite/subagent-startup.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/test/suite/subagent-startup.test.ts b/packages/coding-agent/test/suite/subagent-startup.test.ts index 2ab15f553..8ecf400fd 100644 --- a/packages/coding-agent/test/suite/subagent-startup.test.ts +++ b/packages/coding-agent/test/suite/subagent-startup.test.ts @@ -97,13 +97,14 @@ describe("asynchronous subagent startup", () => { gates.get("second")!.resolve(fauxAssistantMessage("second done")); await waitForStatus(controller, "second", "idle"); await providerStarts.get("third")!.promise; - expect(started).toEqual(["first", "second", "third"]); + expect(started.slice(0, 2).sort()).toEqual(["first", "second"]); + expect(started[2]).toBe("third"); expect(controller.snapshot().agents.find((agent) => agent.name === "fourth")?.status).toBe("queued"); gates.get("first")!.resolve(fauxAssistantMessage("first done")); await waitForStatus(controller, "first", "idle"); await providerStarts.get("fourth")!.promise; - expect(started).toEqual(["first", "second", "third", "fourth"]); + expect(started[3]).toBe("fourth"); } finally { for (const [name, gate] of gates) gate.resolve(fauxAssistantMessage(`${name} done`)); }