From e4f5ac6d2b8b75010f1eb29164ac454f78c28c4a Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Fri, 28 Aug 2026 23:31:00 -0400 Subject: [PATCH 1/3] feat(agent): expose evolved skill activation reviews --- .../src/core/evolution-runtime.ts | 53 +++++ .../coding-agent/test/evolution-menu.test.ts | 1 + packages/evolution/src/control-service.ts | 198 +++++++++++++++++- packages/evolution/src/index.ts | 5 + .../evolution/test/control-service.test.ts | 162 ++++++++++++++ 5 files changed, 418 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 3303b3370..70b04912f 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -48,6 +48,7 @@ import { RecursiveReflector, SkillCandidateBackgroundWorker, SkillCandidateRegistryWorker, + SkillWriter, StructuredDeltaCurator, StructuredSkillCurator, StructuredSkillSynthesizer, @@ -79,6 +80,7 @@ const MAX_MODEL_ROUNDS = 10; const PROVIDER_FIRST_BYTE_TIMEOUT_MS = 60_000; const PROVIDER_STREAM_IDLE_TIMEOUT_MS = 120_000; const PROVIDER_ABSOLUTE_TIMEOUT_MS = 300_000; +const SKILL_ACTIVATION_LEASE_TTL_MS = 5 * 60_000; export async function resolveEvolutionDirectoryIdentity(cwd: string): Promise<{ readonly directoryId: string; @@ -880,6 +882,16 @@ export async function createCodingAgentEvolutionRuntime( artifacts: persistence, jobs: persistence, }); + const skillWriter = new SkillWriter({ + version: VERSION, + workspace: { repositoryId: scope.repositoryId, directoryId: currentDirectory.directoryId }, + harnessState: options.harnessState, + artifacts: persistence, + registry: candidateRegistry, + leases: persistence, + clock, + statePath: join(storageRoot, "skill-writer"), + }); const skillValidator = new DeterministicSkillDraftValidator({ version: VERSION, artifacts: persistence, @@ -935,6 +947,39 @@ export async function createCodingAgentEvolutionRuntime( ledger: persistence, harnesses: options.harnessState, progress, + skillCandidates: candidateRegistry, + skillActivation: { + async activate(activationScope, groupId) { + const lease = await persistence.acquire( + "writer", + "global", + `skill-activation-${randomUUID()}`, + clock.now(), + SKILL_ACTIVATION_LEASE_TTL_MS, + ); + if (lease === null) throw new Error("Another harness writer is active"); + try { + const batch = await skillWriter.buildMutationBatch({ + scope: activationScope, + groupId, + createdAt: clock.now(), + }); + const writeReceipt = await skillWriter.write(batch, { + idempotencyKey: `skill-write-${batch.batchId}`, + lease, + }); + const receipt = await skillWriter.activate(batch, writeReceipt, { + idempotencyKey: `skill-activate-${batch.batchId}`, + lease, + }); + const targetHarnessVersion = receipt.result.resultingHarnessVersion; + if (targetHarnessVersion === null) throw new Error("Skill activation did not select a harness revision"); + return { groupId, batchId: batch.batchId, targetHarnessVersion, receipt }; + } finally { + await persistence.release(lease); + } + }, + }, observedHarnessVersion: () => options.session.activeHarnessVersion, currentDirectory: { ...currentDirectory, @@ -1056,6 +1101,14 @@ export class EvolutionRuntimeCoordinator implements AgentSessionLifecycleObserve return this.#control().getProposal(jobId, proposalId); } + skillCandidateDetail(jobId: string, candidateId: string) { + return this.#control().getSkillCandidate(jobId, candidateId); + } + + activateSkillCandidate(jobId: string, candidateId: string) { + return this.#control().activateSkillCandidate(jobId, candidateId); + } + startManual(): Promise { return this.#control().startManual(); } diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index 6b6c8b1e5..e0d8f9ec4 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -87,6 +87,7 @@ function jobDetail(overrides: Partial = {}): EvolutionJobDet recoveryAttempts: 1, }, ...overrides, + skillCandidates: overrides.skillCandidates ?? [], }; } diff --git a/packages/evolution/src/control-service.ts b/packages/evolution/src/control-service.ts index b62734555..2d23a3810 100644 --- a/packages/evolution/src/control-service.ts +++ b/packages/evolution/src/control-service.ts @@ -1,4 +1,9 @@ -import type { HarnessSelectResult, HarnessVersion, HarnessVersionList } from "@earendil-works/diffex-harness-state"; +import { + computeHarnessSkillId, + type HarnessSelectResult, + type HarnessVersion, + type HarnessVersionList, +} from "@earendil-works/diffex-harness-state"; import type { MaterializedEvidence } from "@earendil-works/diffex-harness-trace"; import type { EvolutionFailureReasonCode, @@ -12,8 +17,17 @@ import type { EvolutionProposalLedgerDisposition, EvolutionRepositorySnapshot, EvolutionScope, + EvolutionSkillContentArtifact, HarnessActivatedLedgerEvent, + HarnessActivationReceipt, ScoredProposal, + SkillCandidate, + SkillCandidateGroup, + SkillCandidateOperation, + SkillCandidateState, + SkillDraft, + SkillDraftValidationReport, + SkillEvaluationReport, ValidationReport, } from "./contracts.ts"; import type { EvolutionControlPlane, EvolutionJobCreationBlockReason } from "./control-plane.ts"; @@ -27,7 +41,9 @@ import type { EvolutionJobStore, EvolutionLedger, EvolutionProgressEvent, + SkillCandidateRegistry, } from "./ports.ts"; +import { computeEvolutionTextDigest } from "./provenance.ts"; import { AUTOMATIC_EVOLUTION_COOLDOWN_MS, AUTOMATIC_EVOLUTION_MINIMUM_INVOCATIONS, @@ -90,6 +106,46 @@ export interface EvolutionOverview { readonly rollbackAction: EvolutionRollbackAction | null; } +export interface EvolutionSkillCandidateSummary { + readonly candidateId: string; + readonly groupId: string | null; + readonly proposedName: string; + readonly operation: SkillCandidateOperation; + readonly state: SkillCandidateState; +} + +export interface EvolutionSkillCandidateExistingContent { + readonly skillId: string; + readonly name: string; + readonly version: string; + readonly content: string; +} + +export interface EvolutionSkillCandidateDetail { + readonly summary: EvolutionSkillCandidateSummary; + readonly candidate: SkillCandidate; + readonly sourceEvolutionJobIds: readonly string[]; + readonly proposalRevisionIds: readonly string[]; + readonly existingContents: readonly EvolutionSkillCandidateExistingContent[]; + readonly draft: SkillDraft | null; + readonly validation: SkillDraftValidationReport | null; + readonly evaluation: SkillEvaluationReport | null; + readonly targetHarnessVersion: HarnessVersion | null; + readonly rollbackAvailable: boolean; + readonly activationAvailable: boolean; +} + +export interface EvolutionSkillActivationResult { + readonly groupId: string; + readonly batchId: string; + readonly targetHarnessVersion: HarnessVersion; + readonly receipt: HarnessActivationReceipt; +} + +export interface EvolutionSkillActivationControl { + activate(scope: EvolutionScope, groupId: string): Promise; +} + export interface EvolutionJobDetail extends EvolutionJobSummary { readonly evidenceHighWaterCursor: number; readonly completedSlices: number; @@ -107,6 +163,7 @@ export interface EvolutionJobDetail extends EvolutionJobSummary { readonly nextPromptWillRefresh: boolean; readonly proposalIds: readonly string[]; readonly proposals: readonly EvolutionProposal[]; + readonly skillCandidates: readonly EvolutionSkillCandidateSummary[]; readonly observability: EvolutionJobObservability; } @@ -164,6 +221,8 @@ export interface EvolutionControlServiceDependencies { readonly ledger: EvolutionLedger; readonly harnesses: EvolutionHarnessControl; readonly progress: EvolutionProgressBus; + readonly skillCandidates?: SkillCandidateRegistry; + readonly skillActivation?: EvolutionSkillActivationControl; readonly observedHarnessVersion: () => HarnessVersion | undefined; readonly currentDirectory: { readonly directoryId: string; @@ -188,6 +247,26 @@ function isCoverageMap(artifact: EvolutionArtifact): artifact is Extract typeof entry === "string") ? value : []; } @@ -249,6 +328,25 @@ function harnessActivation(events: readonly EvolutionLedgerEvent[]): HarnessActi return null; } +function candidateGroup(groups: readonly SkillCandidateGroup[], candidateId: string): SkillCandidateGroup | null { + return ( + groups.find((group) => group.proposalRevisions.some((revision) => revision.candidateId === candidateId)) ?? null + ); +} + +function candidateSummary( + candidate: SkillCandidate, + group: SkillCandidateGroup | null, +): EvolutionSkillCandidateSummary { + return { + candidateId: candidate.candidateId, + groupId: group?.groupId ?? null, + proposedName: candidate.proposedName, + operation: candidate.operation, + state: group?.state ?? "proposed", + }; +} + export class EvolutionProgressBus implements EvolutionEventSink { readonly #listeners = new Set<(event: EvolutionProgressEvent) => void>(); @@ -529,6 +627,8 @@ export class EvolutionControlService { const input = artifacts.find(isInputSnapshot); if (input === undefined) throw new Error(`Evolution job "${jobId}" has no input snapshot`); const scored = artifacts.filter(isScoredProposal); + const skillCandidates = artifacts.filter(isSkillCandidate); + const candidateGroups = (await this.#dependencies.skillCandidates?.list(this.#dependencies.scope)) ?? []; const discoveredSlices = new Set( artifacts.flatMap((artifact) => ("sliceIds" in artifact ? stringArray(artifact.sliceIds) : [])), ).size; @@ -637,6 +737,9 @@ export class EvolutionControlService { versions.currentVersion !== null && observed !== null && versions.currentVersion !== observed, proposalIds, proposals, + skillCandidates: skillCandidates.map((candidate) => + candidateSummary(candidate, candidateGroup(candidateGroups, candidate.candidateId)), + ), observability: { queueAgeMs, stageLatencies, @@ -654,6 +757,99 @@ export class EvolutionControlService { }; } + async getSkillCandidate(jobId: string, candidateId: string): Promise { + const job = await this.#dependencies.jobs.load(jobId); + if ( + job === null || + isCancelledJob(job) || + job.metadata.scope.repositoryId !== this.#dependencies.scope.repositoryId || + job.metadata.scope.scopeId !== this.#dependencies.scope.scopeId + ) { + throw new Error(`Unknown evolution job "${jobId}"`); + } + const [artifacts, groups, versions] = await Promise.all([ + loadArtifacts(job, this.#dependencies.artifacts), + this.#dependencies.skillCandidates?.list(this.#dependencies.scope) ?? Promise.resolve([]), + this.#dependencies.harnesses.listVersions(), + ]); + const candidate = artifacts.filter(isSkillCandidate).find((item) => item.candidateId === candidateId); + if (candidate === undefined) throw new Error(`Unknown skill candidate "${candidateId}"`); + const input = artifacts.find(isInputSnapshot); + if (input === undefined) throw new Error(`Evolution job "${jobId}" has no input snapshot`); + const group = candidateGroup(groups, candidateId); + const loadGroupArtifact = async ( + reference: { readonly artifactId: string; readonly contentDigest: string } | null | undefined, + guard: (artifact: EvolutionArtifact) => artifact is T, + ): Promise => { + if (reference === null || reference === undefined) return null; + const artifact = await this.#dependencies.artifacts.get(reference); + if (!guard(artifact)) throw new Error(`Skill candidate "${candidateId}" has an invalid review artifact`); + return artifact; + }; + const [draft, validation, evaluation] = await Promise.all([ + loadGroupArtifact(group?.draft, isSkillDraft), + loadGroupArtifact(group?.validation, isSkillValidation), + loadGroupArtifact(group?.evaluation, isSkillEvaluation), + ]); + const existingContents = await Promise.all( + candidate.existingSkillIds.map(async (skillId) => { + const entry = input.skillCatalog.find((skill) => skill.skillId === skillId); + if (entry === undefined) throw new Error(`Skill candidate source "${skillId}" is unavailable`); + const artifact = await this.#dependencies.artifacts.get(entry.contentArtifact); + if (!isSkillContent(artifact)) throw new Error(`Skill candidate source "${skillId}" is invalid`); + return { + skillId, + name: entry.name, + version: entry.version, + content: artifact.skill.content, + }; + }), + ); + const current = versions.versions.find((bundle) => bundle.manifest.version === versions.currentVersion); + const draftDigest = draft === null ? null : computeEvolutionTextDigest(draft.content); + const active = + draft !== null && + current !== undefined && + (draft.operation === "RETIRE" + ? candidate.existingSkillIds.every( + (skillId) => + !current.state.skills.some( + (skill) => computeHarnessSkillId(skill.workspace, skill.path) === skillId, + ), + ) + : current.state.skills.some((skill) => skill.contentHash === draftDigest)); + const rollbackAvailable = + active && + current?.manifest.base !== null && + versions.versions.some((bundle) => bundle.manifest.version === current?.manifest.base); + return { + summary: candidateSummary(candidate, group), + candidate, + sourceEvolutionJobIds: group?.proposalRevisions.map((revision) => revision.sourceEvolutionJobId) ?? [jobId], + proposalRevisionIds: group?.proposalRevisions.map((revision) => revision.revisionId) ?? [], + existingContents, + draft, + validation, + evaluation, + targetHarnessVersion: active ? (versions.currentVersion ?? null) : null, + rollbackAvailable, + activationAvailable: + group?.state === "accepted" && !active && this.#dependencies.skillActivation !== undefined, + }; + } + + async activateSkillCandidate(jobId: string, candidateId: string): Promise { + const detail = await this.getSkillCandidate(jobId, candidateId); + if ( + !detail.activationAvailable || + detail.summary.groupId === null || + this.#dependencies.skillActivation === undefined + ) { + throw new Error(`Skill candidate "${candidateId}" is not eligible for activation`); + } + return this.#dependencies.skillActivation.activate(this.#dependencies.scope, detail.summary.groupId); + } + async getProposal(jobId: string, proposalId: string): Promise { const job = await this.#dependencies.jobs.load(jobId); if (job === null || isCancelledJob(job)) throw new Error(`Unknown evolution job "${jobId}"`); diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index f04513e81..aa262e423 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -211,6 +211,11 @@ export { EvolutionProgressBus, type EvolutionProposalDetail, type EvolutionRollbackAction, + type EvolutionSkillActivationControl, + type EvolutionSkillActivationResult, + type EvolutionSkillCandidateDetail, + type EvolutionSkillCandidateExistingContent, + type EvolutionSkillCandidateSummary, type EvolutionStageLatency, type EvolutionStartResult, } from "./control-service.ts"; diff --git a/packages/evolution/test/control-service.test.ts b/packages/evolution/test/control-service.test.ts index a26f7c50e..d4cad3579 100644 --- a/packages/evolution/test/control-service.test.ts +++ b/packages/evolution/test/control-service.test.ts @@ -22,6 +22,11 @@ import { estimateEvolutionTokens, FileEvolutionPersistence, PollingEvolutionWorkSource, + type SkillCandidate, + type SkillCandidateGroup, + type SkillDraft, + type SkillDraftValidationReport, + type SkillEvaluationReport, type SuccessfulMaterialization, } from "../src/index.ts"; @@ -537,6 +542,163 @@ describe("evolution control service", () => { } }); + it("exposes candidate provenance and gates manual activation on accepted evaluation", async () => { + const inputReference = { artifactId: "input-1", contentDigest: `sha256:${"1".repeat(64)}` }; + const candidateReference = { artifactId: "candidate-1", contentDigest: `sha256:${"2".repeat(64)}` }; + const draftReference = { artifactId: "draft-1", contentDigest: `sha256:${"3".repeat(64)}` }; + const validationReference = { artifactId: "validation-1", contentDigest: `sha256:${"4".repeat(64)}` }; + const evaluationReference = { artifactId: "evaluation-1", contentDigest: `sha256:${"5".repeat(64)}` }; + const candidate = { + artifactKind: "skill-candidate", + candidateId: "candidate-1", + operation: "CREATE", + proposedName: "focused-verification", + existingSkillIds: [], + sourceLessonIds: ["lesson-1"], + sourceProceduralObservationIds: [], + supportingEvidenceIds: ["evidence-1"], + contradictingEvidenceIds: [], + supportingTraceIds: ["trace-1"], + contradictingTraceIds: [], + taskLineageIds: ["task-1"], + } as unknown as SkillCandidate; + const draft = { + artifactKind: "skill-draft", + draftId: "draft-1", + groupId: "group-1", + operation: "CREATE", + content: "---\nname: focused-verification\ndescription: Verify focused changes.\n---\n\nRun checks.\n", + } as unknown as SkillDraft; + const validation = { + artifactKind: "skill-draft-validation", + validationId: "validation-1", + disposition: "accepted", + issues: [], + } as unknown as SkillDraftValidationReport; + const evaluation = { + artifactKind: "skill-evaluation", + evaluationId: "evaluation-1", + disposition: "accepted", + gates: { + suite: { passed: true, reasonCodes: [] }, + retrieval: { passed: true, reasonCodes: [] }, + behavior: { passed: true, reasonCodes: [] }, + }, + } as unknown as SkillEvaluationReport; + const group = { + groupId: "group-1", + state: "accepted", + proposalRevisions: [ + { + revisionId: "revision-1", + candidateId: candidate.candidateId, + sourceEvolutionJobId: "job-1", + }, + ], + draft: draftReference, + validation: validationReference, + evaluation: evaluationReference, + } as unknown as SkillCandidateGroup; + const input = { + ingestionHighWaterCursor: 1, + evidenceSources: [], + harness: { baseVersion: null }, + expectedHarnessBaseVersion: `sha256:${"a".repeat(64)}`, + skillCatalog: [], + tokenMeasurement: { estimatedTokens: 10 }, + } as unknown as EvolutionInputSnapshot; + const job = { + metadata: { scope }, + jobId: "job-1", + inputSnapshot: inputReference, + trigger: "manual", + stage: "completed", + outcome: "completed", + reasonCode: null, + failureMessage: null, + completion: null, + resolution: null, + checkpoints: [ + { + stage: "skill-curating", + outputArtifacts: [candidateReference], + state: { candidateIds: [candidate.candidateId] }, + completedAt: "2026-08-20T00:00:01.000Z", + }, + ], + attempt: 0, + revision: 1, + createdAt: "2026-08-20T00:00:00.000Z", + updatedAt: "2026-08-20T00:00:01.000Z", + terminalAt: "2026-08-20T00:00:01.000Z", + } as unknown as EvolutionJob; + const artifacts = new Map([ + [inputReference.artifactId, input], + [candidateReference.artifactId, candidate], + [draftReference.artifactId, draft], + [validationReference.artifactId, validation], + [evaluationReference.artifactId, evaluation], + ]); + const activate = vi.fn().mockResolvedValue({ + groupId: group.groupId, + batchId: "batch-1", + targetHarnessVersion: `sha256:${"b".repeat(64)}`, + receipt: {}, + }); + const service = new EvolutionControlService({ + scope, + jobs: { load: async () => job }, + artifacts: { + get: async (reference: EvolutionArtifactReference) => artifacts.get(reference.artifactId), + }, + harnesses: { + listVersions: async () => ({ + currentVersion: input.expectedHarnessBaseVersion, + versions: [ + { + manifest: { version: input.expectedHarnessBaseVersion, base: null }, + state: { memory: "", skills: [] }, + }, + ], + diagnostics: [], + }), + }, + ledger: { read: async () => [] }, + skillCandidates: { list: async () => [group] }, + skillActivation: { activate }, + observedHarnessVersion: () => input.expectedHarnessBaseVersion, + } as unknown as EvolutionControlServiceDependencies); + + expect(await service.getJob(job.jobId)).toMatchObject({ + skillCandidates: [ + { + candidateId: candidate.candidateId, + groupId: group.groupId, + proposedName: candidate.proposedName, + operation: "CREATE", + state: "accepted", + }, + ], + }); + const detail = await service.getSkillCandidate(job.jobId, candidate.candidateId); + expect(detail).toMatchObject({ + summary: { groupId: group.groupId, state: "accepted" }, + sourceEvolutionJobIds: [job.jobId], + proposalRevisionIds: ["revision-1"], + draft: { draftId: "draft-1" }, + validation: { disposition: "accepted" }, + evaluation: { disposition: "accepted" }, + targetHarnessVersion: null, + rollbackAvailable: false, + activationAvailable: true, + }); + + await expect(service.activateSkillCandidate(job.jobId, candidate.candidateId)).resolves.toMatchObject({ + batchId: "batch-1", + }); + expect(activate).toHaveBeenCalledWith(scope, group.groupId); + }); + it("reports durable latency, budget, coverage, conflict, and recovery observability", async () => { const fixture = await serviceFixture(); const started = await fixture.service.startManual(); From 0e1a4bde6e58148d70a927d9d2343cc8271901ea Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Fri, 28 Aug 2026 23:39:25 -0400 Subject: [PATCH 2/3] feat(coding-agent): add evolved skill operator controls --- .../src/core/evolution-runtime.ts | 4 + .../interactive/components/config-selector.ts | 16 ++ .../interactive/components/evolution-menu.ts | 207 +++++++++++++++++- .../components/settings-selector.ts | 13 ++ .../src/modes/interactive/interactive-mode.ts | 48 +++- .../coding-agent/test/evolution-menu.test.ts | 172 ++++++++++++++- .../test/interactive-mode-evolution.test.ts | 3 + .../test/settings-selector.test.ts | 63 +++++- packages/evolution/src/control-service.ts | 33 ++- .../evolution/test/control-service.test.ts | 49 ++++- 10 files changed, 587 insertions(+), 21 deletions(-) diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 70b04912f..9f05bc547 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -1109,6 +1109,10 @@ export class EvolutionRuntimeCoordinator implements AgentSessionLifecycleObserve return this.#control().activateSkillCandidate(jobId, candidateId); } + rollbackSkillCandidate(jobId: string, candidateId: string) { + return this.#control().rollbackSkillCandidate(jobId, candidateId); + } + startManual(): Promise { return this.#control().startManual(); } diff --git a/packages/coding-agent/src/modes/interactive/components/config-selector.ts b/packages/coding-agent/src/modes/interactive/components/config-selector.ts index 7b87136b0..147c4109c 100644 --- a/packages/coding-agent/src/modes/interactive/components/config-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/config-selector.ts @@ -454,6 +454,22 @@ class ResourceList implements Component, Focusable { lines.push(theme.fg("dim", ` (${currentItemIndex}/${itemCount})`)); } + const selected = this.filteredItems[this.selectedIndex]; + if (selected?.type === "item") { + lines.push(""); + lines.push(truncateToWidth(theme.fg("muted", ` Path: ${selected.item.path}`), width, "...")); + lines.push( + truncateToWidth( + theme.fg( + "muted", + ` Source: ${selected.item.metadata.source} · scope: ${this.getItemScope(selected.item)} · ${selected.item.enabled ? "enabled" : "disabled"}`, + ), + width, + "...", + ), + ); + } + return lines; } diff --git a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts index 2e4d6e5c7..76f426551 100644 --- a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts +++ b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts @@ -8,6 +8,7 @@ import { TruncatedText, UNSELECTED_LIST_PREFIX, } from "@earendil-works/diffex-tui"; +import { diffLines } from "diff"; import type { EvolutionHarnessVersionInfo, EvolutionJobDetail, @@ -16,6 +17,8 @@ import type { EvolutionProgressEvent, EvolutionProposalDetail, EvolutionRollbackAction, + EvolutionSkillActivationResult, + EvolutionSkillCandidateDetail, EvolutionStartResult, } from "#evolution"; import { theme } from "../theme/theme.ts"; @@ -39,6 +42,9 @@ export interface EvolutionMenuClient { listJobs(): Promise; jobDetail(jobId: string): Promise; proposalDetail(jobId: string, proposalId: string): Promise; + skillCandidateDetail(jobId: string, candidateId: string): Promise; + activateSkillCandidate(jobId: string, candidateId: string): Promise; + rollbackSkillCandidate(jobId: string, candidateId: string): Promise; startManual(): Promise; resumeNewest(): Promise; cancelJob(jobId: string): Promise; @@ -50,6 +56,13 @@ type MenuView = | { readonly kind: "jobs" } | { readonly kind: "job"; readonly jobId: string } | { readonly kind: "proposal"; readonly jobId: string; readonly proposalId: string } + | { readonly kind: "skill-candidate"; readonly jobId: string; readonly candidateId: string } + | { + readonly kind: "skill-candidate-review"; + readonly jobId: string; + readonly candidateId: string; + readonly section: "draft" | "evaluation" | "provenance" | "validation"; + } | { readonly kind: "proposal-evidence"; readonly jobId: string; @@ -58,8 +71,9 @@ type MenuView = } | { readonly kind: "confirm"; - readonly action: "cancel" | "resume" | "rollback" | "start"; + readonly action: "activate-skill" | "cancel" | "resume" | "rollback" | "rollback-skill" | "start"; readonly jobId?: string; + readonly candidateId?: string; }; interface MenuEntry { @@ -136,6 +150,23 @@ export function formatEvolutionEvidencePreview( return firstCharacters(content, 100); } +export function formatEvolutionSkillDraftDiff(detail: EvolutionSkillCandidateDetail): readonly string[] { + if (detail.draft === null) return ["Draft pending"]; + const before = detail.existingContents + .map((skill) => `# Existing skill: ${skill.name} (${skill.version})\n${skill.content}`) + .join("\n\n"); + const after = detail.draft.operation === "RETIRE" ? "" : detail.draft.content; + const changes = diffLines(before, after, { maxEditLength: 20_000 }); + if (changes === undefined) return ["Draft diff is too large to render"]; + return changes.flatMap((change) => { + const prefix = change.added ? "+" : change.removed ? "-" : " "; + return change.value + .replace(/\n$/u, "") + .split("\n") + .map((line) => `${prefix}${line}`); + }); +} + export class EvolutionMenuComponent extends Container implements Focusable { readonly #client: EvolutionMenuClient; readonly #onClose: () => void; @@ -151,6 +182,7 @@ export class EvolutionMenuComponent extends Container implements Focusable { #harnessVersionInfoLoaded = false; #job: EvolutionJobDetail | null = null; #proposal: EvolutionProposalDetail | null = null; + #skillCandidate: EvolutionSkillCandidateDetail | null = null; #entries: readonly MenuEntry[] = []; #selectedIndex = 0; #loading = true; @@ -205,6 +237,9 @@ export class EvolutionMenuComponent extends Container implements Focusable { if (view.kind === "overview" || view.kind === "confirm") return this.#overview !== null; if (view.kind === "jobs") return this.#jobsLoaded; if (view.kind === "job") return this.#job?.jobId === view.jobId; + if (view.kind === "skill-candidate" || view.kind === "skill-candidate-review") { + return this.#skillCandidate?.summary.candidateId === view.candidateId; + } return this.#proposal?.proposal.proposalId === view.proposalId; } @@ -267,6 +302,9 @@ export class EvolutionMenuComponent extends Container implements Focusable { if (view.kind === "proposal" || view.kind === "proposal-evidence") { this.#proposal = await this.#client.proposalDetail(view.jobId, view.proposalId); } + if (view.kind === "skill-candidate" || view.kind === "skill-candidate-review") { + this.#skillCandidate = await this.#client.skillCandidateDetail(view.jobId, view.candidateId); + } } catch (error) { this.#onStatus(error instanceof Error ? error.message : String(error), true); } finally { @@ -314,8 +352,12 @@ export class EvolutionMenuComponent extends Container implements Focusable { })(); } - #confirm(action: "cancel" | "resume" | "rollback" | "start", jobId?: string): void { - this.#push({ kind: "confirm", action, jobId }); + #confirm( + action: "activate-skill" | "cancel" | "resume" | "rollback" | "rollback-skill" | "start", + jobId?: string, + candidateId?: string, + ): void { + this.#push({ kind: "confirm", action, jobId, candidateId }); } #overviewEntries(overview: EvolutionOverview): readonly MenuEntry[] { @@ -394,6 +436,18 @@ export class EvolutionMenuComponent extends Container implements Focusable { "The durable checkpoint will become active background work.", ]; } + if (action === "activate-skill") { + return [ + `Activate ${this.#skillCandidate?.summary.proposedName ?? "the selected skill candidate"}?`, + "A complete immutable harness revision will be staged and selected. The next fresh prompt will observe it.", + ]; + } + if (action === "rollback-skill") { + return [ + `Roll back ${this.#skillCandidate?.summary.proposedName ?? "the selected evolved skill"}?`, + `The selected harness will return to ${this.#skillCandidate?.rollbackTargetHarnessVersion ?? "its immutable parent"}. The next fresh prompt will observe it.`, + ]; + } if (action === "cancel") { const job = [overview?.activeJob, overview?.newestResumableJob].find( (candidate) => candidate?.jobId === view.jobId, @@ -430,6 +484,20 @@ export class EvolutionMenuComponent extends Container implements Focusable { this.#run(() => this.#client.cancelJob(cancellationJobId), "Evolution round removed"); } if (action === "rollback") this.#run(() => this.#client.rollback(), "Harness revision rolled back"); + if (action === "activate-skill" && view.jobId !== undefined && view.candidateId !== undefined) { + const { candidateId, jobId } = view; + this.#run( + () => this.#client.activateSkillCandidate(jobId, candidateId), + (result) => `Evolved skill activated in ${result.targetHarnessVersion}`, + ); + } + if (action === "rollback-skill" && view.jobId !== undefined && view.candidateId !== undefined) { + const { candidateId, jobId } = view; + this.#run( + () => this.#client.rollbackSkillCandidate(jobId, candidateId), + "Evolved skill harness rolled back", + ); + } }; return [ { @@ -491,6 +559,16 @@ export class EvolutionMenuComponent extends Container implements Focusable { this.#overview?.currentHarnessVersion, job.baseHarnessInfo, )}`, + `Target harness: ${ + job.activatedHarnessVersion === null + ? "pending" + : formatHarnessVersionReference( + job.activatedHarnessVersion, + this.#harnessVersionInfo, + this.#overview?.currentHarnessVersion, + ) + }`, + `Rollback available: ${job.observability.rollbackAvailable ? "yes" : "no"}`, ]) { this.#addLine(line); } @@ -504,6 +582,99 @@ export class EvolutionMenuComponent extends Container implements Focusable { this.#addLine(`Outcome: ${formatEvolutionProposalOutcome(detail.proposal)}`); } + #skillCandidateEntries(detail: EvolutionSkillCandidateDetail, jobId: string): readonly MenuEntry[] { + const review = (section: Extract["section"]) => + this.#push({ kind: "skill-candidate-review", jobId, candidateId: detail.summary.candidateId, section }); + return [ + { label: "Provenance", enabled: true, select: () => review("provenance") }, + { + label: "Draft diff", + enabled: detail.draft !== null, + disabledStatus: "pending", + select: () => review("draft"), + }, + { + label: "Validation", + enabled: detail.validation !== null, + disabledStatus: "pending", + select: () => review("validation"), + }, + { + label: "Evaluation", + enabled: detail.evaluation !== null, + disabledStatus: "pending", + select: () => review("evaluation"), + }, + { + label: "Activate", + enabled: detail.activationAvailable, + disabledStatus: detail.targetHarnessVersion === null ? "unavailable" : "active", + select: () => this.#confirm("activate-skill", jobId, detail.summary.candidateId), + }, + { + label: "Rollback", + enabled: detail.rollbackAvailable, + disabledStatus: "unavailable", + select: () => this.#confirm("rollback-skill", jobId, detail.summary.candidateId), + }, + ]; + } + + #renderSkillCandidate(detail: EvolutionSkillCandidateDetail): void { + this.#addLine(`Skill candidate: ${detail.summary.proposedName}`, "accent"); + this.#addLine(`Operation: ${detail.summary.operation}`); + this.#addLine(`State: ${detail.summary.state}`); + this.#addLine(`Target harness: ${detail.targetHarnessVersion ?? "pending activation"}`); + this.#addLine(`Rollback available: ${detail.rollbackAvailable ? "yes" : "no"}`); + } + + #renderSkillCandidateReview( + detail: EvolutionSkillCandidateDetail, + section: Extract["section"], + ): void { + if (section === "draft") { + this.#addLine("Draft diff", "accent"); + const lines = formatEvolutionSkillDraftDiff(detail); + for (const line of lines.slice(0, 200)) { + this.#addLine(line, line.startsWith("+") ? "accent" : line.startsWith("-") ? "warning" : "text"); + } + if (lines.length > 200) this.#addLine(`... ${lines.length - 200} more lines`, "muted"); + return; + } + if (section === "provenance") { + this.#addLine("Provenance", "accent"); + for (const [label, values] of [ + ["Evolution jobs", detail.sourceEvolutionJobIds], + ["Candidate revisions", detail.proposalRevisionIds], + ["Lessons", detail.candidate.sourceLessonIds], + ["Evidence", [...detail.candidate.supportingEvidenceIds, ...detail.candidate.contradictingEvidenceIds]], + ["Traces", [...detail.candidate.supportingTraceIds, ...detail.candidate.contradictingTraceIds]], + ["Task lineages", detail.candidate.taskLineageIds], + ] as const) { + this.#addLine(`${label}: ${values.length === 0 ? "none" : values.join(", ")}`); + } + return; + } + if (section === "validation") { + this.#addLine("Deterministic validation", "accent"); + this.#addLine(`Result: ${detail.validation?.disposition ?? "pending"}`); + for (const issue of detail.validation?.issues ?? []) + this.#addLine(`${issue.code}: ${issue.message}`, "warning"); + return; + } + this.#addLine("Retrieval and behavior evaluation", "accent"); + this.#addLine(`Result: ${detail.evaluation?.disposition ?? "pending"}`); + for (const [name, gate] of Object.entries(detail.evaluation?.gates ?? {})) { + this.#addLine(`${name}: ${gate.passed ? "passed" : `failed (${gate.reasonCodes.join(", ")})`}`); + } + if (detail.evaluation !== null) { + this.#addLine(`Exact retrieval accuracy: ${detail.evaluation.metrics.exactRetrievalAccuracy}`); + this.#addLine(`Held-out improvement: ${detail.evaluation.metrics.heldOutIntendedImprovement}`); + this.#addLine(`Negative regressions: ${detail.evaluation.metrics.negativeRegressionCount}`); + this.#addLine(`Misapplications: ${detail.evaluation.metrics.misapplicationCount}`); + } + } + #proposalEvidenceEntries(detail: EvolutionProposalDetail, jobId: string, proposalId: string): readonly MenuEntry[] { return [ { @@ -586,14 +757,27 @@ export class EvolutionMenuComponent extends Container implements Focusable { } else if (view.kind === "job" && this.#job) { this.#renderJob(this.#job); this.addChild(new Spacer(1)); - this.#renderEntries( - this.#job.proposals.map((proposal) => ({ + this.#renderEntries([ + ...this.#job.proposals.map((proposal) => ({ label: formatEvolutionProposalLabel(proposal), enabled: true, - select: () => this.#push({ kind: "proposal", jobId: view.jobId, proposalId: proposal.proposalId }), + select: () => + this.#push({ kind: "proposal" as const, jobId: view.jobId, proposalId: proposal.proposalId }), })), - ); - if (this.#job.proposals.length === 0) this.#addLine("No proposals emitted", "muted"); + ...this.#job.skillCandidates.map((candidate) => ({ + label: `Skill ${candidate.proposedName} - ${candidate.state}`, + enabled: true, + select: () => + this.#push({ + kind: "skill-candidate" as const, + jobId: view.jobId, + candidateId: candidate.candidateId, + }), + })), + ]); + if (this.#job.proposals.length === 0 && this.#job.skillCandidates.length === 0) { + this.#addLine("No proposals or skill candidates emitted", "muted"); + } } else if (view.kind === "proposal" && this.#proposal) { this.#renderProposal(this.#proposal); this.addChild(new Spacer(1)); @@ -601,6 +785,13 @@ export class EvolutionMenuComponent extends Container implements Focusable { } else if (view.kind === "proposal-evidence" && this.#proposal) { this.#renderProposalEvidence(this.#proposal, view.evidenceKind); this.#entries = []; + } else if (view.kind === "skill-candidate" && this.#skillCandidate) { + this.#renderSkillCandidate(this.#skillCandidate); + this.addChild(new Spacer(1)); + this.#renderEntries(this.#skillCandidateEntries(this.#skillCandidate, view.jobId)); + } else if (view.kind === "skill-candidate-review" && this.#skillCandidate) { + this.#renderSkillCandidateReview(this.#skillCandidate, view.section); + this.#entries = []; } else if (view.kind === "confirm") { for (const line of this.#confirmationLines(view)) this.#addLine(line, "warning"); this.addChild(new Spacer(1)); diff --git a/packages/coding-agent/src/modes/interactive/components/settings-selector.ts b/packages/coding-agent/src/modes/interactive/components/settings-selector.ts index 2bc3eb588..860e12d70 100644 --- a/packages/coding-agent/src/modes/interactive/components/settings-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/settings-selector.ts @@ -121,6 +121,7 @@ export interface SettingsCallbacks { onTuiModeChange: (mode: TuiMode) => void; onFullscreenScrollbarChange: (mode: ScrollViewScrollbar) => void; onWarningsChange: (warnings: WarningSettings) => void; + createInstalledSkillsSettings?: (done: () => void) => Component; onCancel: () => void; } @@ -590,6 +591,18 @@ export class SettingsSelectorComponent extends Container { currentValue: config.treeFilterMode, values: ["default", "no-tools", "user-only", "labeled-only", "all"], }, + ...(callbacks.createInstalledSkillsSettings === undefined + ? [] + : [ + { + id: "installed-skills", + label: "Installed skills", + description: "Inspect and enable or disable workspace and global installed skills", + currentValue: "configure", + submenu: (_currentValue: string, done: (selectedValue?: string) => void) => + callbacks.createInstalledSkillsSettings?.(() => done()) ?? new Container(), + }, + ]), { id: "warnings", label: "Warnings", diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index b105eb1f1..2a37c866f 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -95,7 +95,7 @@ import { DefaultPackageManager } from "../../core/package-manager.ts"; import type { ResourceDiagnostic } from "../../core/resource-loader.ts"; import { formatMissingSessionCwdPrompt, MissingSessionCwdError } from "../../core/session-cwd.ts"; import { type SessionEntry, SessionManager, sessionEntryToContextMessages } from "../../core/session-manager.ts"; -import type { TuiMode } from "../../core/settings-manager.ts"; +import { SettingsManager, type TuiMode } from "../../core/settings-manager.ts"; import type { SkillReference } from "../../core/skills.ts"; import { BUILTIN_SLASH_COMMANDS } from "../../core/slash-commands.ts"; import type { SourceInfo } from "../../core/source-info.ts"; @@ -125,6 +125,7 @@ import { BashExecutionComponent } from "./components/bash-execution.ts"; import { BorderedLoader } from "./components/bordered-loader.ts"; import { BranchSummaryMessageComponent } from "./components/branch-summary-message.ts"; import { CompactionSummaryMessageComponent } from "./components/compaction-summary-message.ts"; +import { ConfigSelectorComponent, type ScopedResolvedPaths } from "./components/config-selector.ts"; import { CustomEditor } from "./components/custom-editor.ts"; import { CustomEntryComponent } from "./components/custom-entry.ts"; import { CustomMessageComponent } from "./components/custom-message.ts"; @@ -2917,8 +2918,8 @@ export class InteractiveMode { // Handle commands if (text === "/settings") { - this.showSettingsSelector(); this.editor.setText(""); + await this.showSettingsSelector(); return; } if (text === "/skills") { @@ -4484,7 +4485,29 @@ export class InteractiveMode { }); } - private showSettingsSelector(): void { + private async showSettingsSelector(): Promise { + const cwd = this.sessionManager.getCwd(); + const agentDir = this.runtimeHost.services.agentDir; + let installedSkillPaths: ScopedResolvedPaths; + try { + const globalSettings = SettingsManager.create(cwd, agentDir, { projectTrusted: false }); + const global = await new DefaultPackageManager({ cwd, agentDir, settingsManager: globalSettings }).resolve(); + const project = this.settingsManager.isProjectTrusted() + ? await new DefaultPackageManager({ cwd, agentDir, settingsManager: this.settingsManager }).resolve() + : global; + const skillsOnly = (resolved: typeof global) => ({ + extensions: [], + skills: resolved.skills, + prompts: [], + themes: [], + }); + installedSkillPaths = { global: skillsOnly(global), project: skillsOnly(project) }; + } catch (error) { + this.showError( + `Could not inspect installed skills: ${error instanceof Error ? error.message : String(error)}`, + ); + return; + } this.showSelector((done) => { let selector: SettingsSelectorComponent | undefined; selector = new SettingsSelectorComponent( @@ -4677,6 +4700,25 @@ export class InteractiveMode { onWarningsChange: (warnings) => { this.settingsManager.setWarnings(warnings); }, + createInstalledSkillsSettings: (finish) => + new ConfigSelectorComponent( + installedSkillPaths, + this.settingsManager, + cwd, + agentDir, + () => { + finish(); + this.showStatus("Installed skill settings saved; use /reload to apply them to this session"); + }, + () => { + finish(); + this.showStatus("Installed skill settings saved; use /reload to apply them to this session"); + }, + () => this.ui.requestRender(), + this.ui.terminal.rows, + "global", + this.settingsManager.isProjectTrusted(), + ), onCancel: () => { done(); this.ui.requestRender(); diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index e0d8f9ec4..31b4ca901 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -8,6 +8,7 @@ import type { EvolutionOverview, EvolutionProgressEvent, EvolutionProposalDetail, + EvolutionSkillCandidateDetail, } from "#evolution"; import { KeybindingsManager } from "../src/core/keybindings.ts"; import { @@ -17,6 +18,7 @@ import { formatEvolutionJobLabel, formatEvolutionProposalLabel, formatEvolutionProposalOutcome, + formatEvolutionSkillDraftDiff, formatEvolutionTimestamp, sanitizeEvolutionText, } from "../src/modes/interactive/components/evolution-menu.ts"; @@ -175,6 +177,54 @@ function proposalDetail(): EvolutionProposalDetail { }; } +function skillCandidateDetail(overrides: Partial = {}): EvolutionSkillCandidateDetail { + const detail = { + summary: { + candidateId: "candidate-1", + groupId: "group-1", + proposedName: "focused-verification", + operation: "CREATE", + state: "accepted", + }, + candidate: { + sourceLessonIds: ["lesson-1"], + supportingEvidenceIds: ["evidence-1"], + contradictingEvidenceIds: [], + supportingTraceIds: ["trace-1"], + contradictingTraceIds: [], + taskLineageIds: ["task-1"], + }, + sourceEvolutionJobIds: ["job-active"], + proposalRevisionIds: ["revision-1"], + existingContents: [], + draft: { + operation: "CREATE", + content: "---\nname: focused-verification\ndescription: Verify focused changes.\n---\n\nRun checks.\n", + }, + validation: { disposition: "accepted", issues: [] }, + evaluation: { + disposition: "accepted", + gates: { + suite: { passed: true, reasonCodes: [] }, + retrieval: { passed: true, reasonCodes: [] }, + behavior: { passed: true, reasonCodes: [] }, + }, + metrics: { + exactRetrievalAccuracy: 1, + heldOutIntendedImprovement: 0.5, + negativeRegressionCount: 0, + misapplicationCount: 0, + }, + }, + targetHarnessVersion: null, + rollbackTargetHarnessVersion: null, + rollbackAvailable: false, + activationAvailable: true, + ...overrides, + } as unknown as EvolutionSkillCandidateDetail; + return detail; +} + function evidence(evidenceId: string, text: string): EvolutionProposalDetail["supportingEvidence"][number] { return { evidenceId, @@ -198,6 +248,9 @@ function client(currentOverview: EvolutionOverview): { listJobs: ReturnType; jobDetailCall: ReturnType; proposalDetailCall: ReturnType; + skillCandidateDetailCall: ReturnType; + activateSkillCandidate: ReturnType; + rollbackSkillCandidate: ReturnType; getEnabled: ReturnType; setEnabled: ReturnType; } { @@ -214,6 +267,19 @@ function client(currentOverview: EvolutionOverview): { const listJobs = vi.fn().mockResolvedValue([]); const jobDetailCall = vi.fn().mockRejectedValue(new Error("unused")); const proposalDetailCall = vi.fn().mockRejectedValue(new Error("unused")); + const skillCandidateDetailCall = vi.fn().mockRejectedValue(new Error("unused")); + const activateSkillCandidate = vi.fn().mockResolvedValue({ + groupId: "group-1", + batchId: "batch-1", + targetHarnessVersion: `sha256:${"d".repeat(64)}`, + receipt: {}, + }); + const rollbackSkillCandidate = vi.fn().mockResolvedValue({ + jobId: "job-active", + fromHarnessVersion: `sha256:${"d".repeat(64)}`, + toHarnessVersion: rootVersion, + proposalIds: ["candidate-1"], + }); const getEnabled = vi.fn(() => enabled); const setEnabled = vi.fn(async (next: boolean) => { enabled = next; @@ -232,6 +298,9 @@ function client(currentOverview: EvolutionOverview): { listJobs, jobDetail: jobDetailCall, proposalDetail: proposalDetailCall, + skillCandidateDetail: skillCandidateDetailCall, + activateSkillCandidate, + rollbackSkillCandidate, startManual, resumeNewest: vi.fn().mockRejectedValue(new Error("unused")), cancelJob, @@ -247,6 +316,9 @@ function client(currentOverview: EvolutionOverview): { listJobs, jobDetailCall, proposalDetailCall, + skillCandidateDetailCall, + activateSkillCandidate, + rollbackSkillCandidate, getEnabled, setEnabled, }; @@ -467,6 +539,8 @@ describe("EvolutionMenuComponent", () => { expect(output).toContain("Evolution token: 12,345"); expect(output).toContain("Last checkpoint: Today at 9:05 am"); expect(output).toContain("Base harness: [current] Default Diffex Agent"); + expect(output).toContain("Target harness: pending"); + expect(output).toContain("Rollback available: no"); for (const removedLabel of [ "Job:", "Evidence high-water cursor:", @@ -479,8 +553,6 @@ describe("EvolutionMenuComponent", () => { "Provider cost:", "Proposal dispositions:", "Activation conflicts:", - "Rollback available:", - "Activated harness:", "Next prompt will refresh:", "Result:", "Proposals:", @@ -561,6 +633,102 @@ describe("EvolutionMenuComponent", () => { component.dispose(); }); + it("reviews candidate provenance, draft, validation, evaluation, and confirms activation", async () => { + const detail = skillCandidateDetail(); + const fixture = client(overview()); + fixture.listJobs.mockResolvedValue([job()]); + fixture.jobDetailCall.mockResolvedValue( + jobDetail({ + skillCandidates: [detail.summary], + }), + ); + fixture.skillCandidateDetailCall.mockResolvedValue(detail); + const status = vi.fn(); + const component = new EvolutionMenuComponent( + fixture.client, + () => {}, + status, + () => {}, + ); + await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Browse")); + + for (let index = 0; index < 4; index += 1) component.handleInput("\u001b[B"); + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain(formatEvolutionJobLabel(job())), + ); + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain("Skill focused-verification - accepted"), + ); + component.handleInput("\n"); + await vi.waitFor(() => + expect(fixture.skillCandidateDetailCall).toHaveBeenCalledWith("job-active", "candidate-1"), + ); + + let output = stripAnsi(component.render(180).join("\n")); + expect(output).toContain("Skill candidate: focused-verification"); + expect(output).toContain("Target harness: pending activation"); + expect(output).toContain("Provenance"); + expect(output).toContain("Draft diff"); + expect(output).toContain("Validation"); + expect(output).toContain("Evaluation"); + expect(output).toContain("Activate"); + + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain("Evolution jobs: job-active"), + ); + component.handleInput("\u001b"); + component.handleInput("\u001b[B"); + component.handleInput("\n"); + await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("+Run checks.")); + component.handleInput("\u001b"); + for (let index = 0; index < 3; index += 1) component.handleInput("\u001b[B"); + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain("Retrieval and behavior evaluation"), + ); + output = stripAnsi(component.render(180).join("\n")); + expect(output).toContain("retrieval: passed"); + expect(output).toContain("Negative regressions: 0"); + + component.handleInput("\u001b"); + for (let index = 0; index < 4; index += 1) component.handleInput("\u001b[B"); + component.handleInput("\n"); + await vi.waitFor(() => + expect(stripAnsi(component.render(180).join("\n"))).toContain("Activate focused-verification?"), + ); + expect(fixture.activateSkillCandidate).not.toHaveBeenCalled(); + component.handleInput("\n"); + await vi.waitFor(() => expect(fixture.activateSkillCandidate).toHaveBeenCalledWith("job-active", "candidate-1")); + await vi.waitFor(() => + expect(status).toHaveBeenCalledWith(expect.stringContaining("Evolved skill activated in sha256:"), false), + ); + component.dispose(); + }); + + it("formats create, update, and retirement draft diffs", () => { + expect(formatEvolutionSkillDraftDiff(skillCandidateDetail())).toContain("+Run checks."); + const existingContents = [{ skillId: "skill-1", name: "old", version: "v1", content: "old line\n" }]; + expect( + formatEvolutionSkillDraftDiff( + skillCandidateDetail({ + existingContents, + draft: { ...skillCandidateDetail().draft, operation: "UPDATE", content: "new line\n" }, + } as unknown as Partial), + ), + ).toEqual(expect.arrayContaining(["-old line", "+new line"])); + expect( + formatEvolutionSkillDraftDiff( + skillCandidateDetail({ + existingContents, + draft: { ...skillCandidateDetail().draft, operation: "RETIRE" }, + } as unknown as Partial), + ), + ).toContain("-old line"); + }); + it("formats non-add proposal outcomes as readable text", () => { const detail = proposalDetail(); if (detail.proposal.disposition !== "actionable") throw new Error("Expected an actionable proposal"); diff --git a/packages/coding-agent/test/interactive-mode-evolution.test.ts b/packages/coding-agent/test/interactive-mode-evolution.test.ts index b81d7d7a5..7f7bd0442 100644 --- a/packages/coding-agent/test/interactive-mode-evolution.test.ts +++ b/packages/coding-agent/test/interactive-mode-evolution.test.ts @@ -71,6 +71,9 @@ function client(): EvolutionMenuClient { listJobs: vi.fn().mockResolvedValue([]), jobDetail: vi.fn().mockRejectedValue(new Error("unused")), proposalDetail: vi.fn().mockRejectedValue(new Error("unused")), + skillCandidateDetail: vi.fn().mockRejectedValue(new Error("unused")), + activateSkillCandidate: vi.fn().mockRejectedValue(new Error("unused")), + rollbackSkillCandidate: vi.fn().mockRejectedValue(new Error("unused")), startManual: vi.fn().mockRejectedValue(new Error("unused")), resumeNewest: vi.fn().mockRejectedValue(new Error("unused")), cancelJob: vi.fn().mockRejectedValue(new Error("unused")), diff --git a/packages/coding-agent/test/settings-selector.test.ts b/packages/coding-agent/test/settings-selector.test.ts index ee59ac724..b50d7e515 100644 --- a/packages/coding-agent/test/settings-selector.test.ts +++ b/packages/coding-agent/test/settings-selector.test.ts @@ -1,12 +1,18 @@ -import { setKeybindings } from "@earendil-works/diffex-tui"; +import { setKeybindings, Text } from "@earendil-works/diffex-tui"; import { beforeAll, describe, expect, it, vi } from "vitest"; import { KeybindingsManager } from "../src/core/keybindings.ts"; +import { SettingsManager } from "../src/core/settings-manager.ts"; +import { + ConfigSelectorComponent, + type ScopedResolvedPaths, +} from "../src/modes/interactive/components/config-selector.ts"; import { type SettingsCallbacks, type SettingsConfig, SettingsSelectorComponent, } from "../src/modes/interactive/components/settings-selector.ts"; import { initTheme } from "../src/modes/interactive/theme/theme.ts"; +import { stripAnsi } from "../src/utils/ansi.ts"; describe("SettingsSelectorComponent", () => { beforeAll(() => { @@ -14,6 +20,61 @@ describe("SettingsSelectorComponent", () => { setKeybindings(new KeybindingsManager()); }); + it("opens installed skill inspection and controls", () => { + const createInstalledSkillsSettings = vi.fn(() => new Text("Installed skill controls")); + const selector = new SettingsSelectorComponent( + { + fullscreenScrollbar: "auto", + warnings: {}, + availableThinkingLevels: [], + availableThemes: [], + } as unknown as SettingsConfig, + { createInstalledSkillsSettings } as unknown as SettingsCallbacks, + ); + const settingsList = selector.getSettingsList(); + + for (const character of "Installed skills") settingsList.handleInput(character); + settingsList.handleInput("\r"); + + expect(createInstalledSkillsSettings).toHaveBeenCalledOnce(); + }); + + it("shows installed skill scope and path while persisting disablement", () => { + const settingsManager = SettingsManager.inMemory(); + const resolved = { + extensions: [], + skills: [ + { + path: "/tmp/skills/review/SKILL.md", + enabled: true, + metadata: { source: "auto", scope: "user", origin: "top-level", baseDir: "/tmp/skills" }, + }, + ], + prompts: [], + themes: [], + } satisfies ScopedResolvedPaths["global"]; + const selector = new ConfigSelectorComponent( + { global: resolved, project: resolved }, + settingsManager, + "/tmp/workspace", + "/tmp/agent", + () => {}, + () => {}, + () => {}, + 24, + "global", + false, + ); + const list = selector.getResourceList(); + + const output = stripAnsi(list.render(160).join("\n")); + expect(output).toContain("Path: /tmp/skills/review/SKILL.md"); + expect(output).toContain("Source: auto · scope: user · enabled"); + list.handleInput(" "); + + expect(settingsManager.getSkillPaths()).toContain("-review/SKILL.md"); + }); + it("cycles through fullscreen scrollbar modes", () => { const onChange = vi.fn(); const selector = new SettingsSelectorComponent( diff --git a/packages/evolution/src/control-service.ts b/packages/evolution/src/control-service.ts index 2d23a3810..0663c815c 100644 --- a/packages/evolution/src/control-service.ts +++ b/packages/evolution/src/control-service.ts @@ -131,6 +131,7 @@ export interface EvolutionSkillCandidateDetail { readonly validation: SkillDraftValidationReport | null; readonly evaluation: SkillEvaluationReport | null; readonly targetHarnessVersion: HarnessVersion | null; + readonly rollbackTargetHarnessVersion: HarnessVersion | null; readonly rollbackAvailable: boolean; readonly activationAvailable: boolean; } @@ -817,7 +818,10 @@ export class EvolutionControlService { (skill) => computeHarnessSkillId(skill.workspace, skill.path) === skillId, ), ) - : current.state.skills.some((skill) => skill.contentHash === draftDigest)); + : current.state.skills.some( + (skill) => + skill.path === `skills/${draft.proposedName}/SKILL.md` && skill.contentHash === draftDigest, + )); const rollbackAvailable = active && current?.manifest.base !== null && @@ -832,12 +836,39 @@ export class EvolutionControlService { validation, evaluation, targetHarnessVersion: active ? (versions.currentVersion ?? null) : null, + rollbackTargetHarnessVersion: active ? (current?.manifest.base ?? null) : null, rollbackAvailable, activationAvailable: group?.state === "accepted" && !active && this.#dependencies.skillActivation !== undefined, }; } + async rollbackSkillCandidate(jobId: string, candidateId: string): Promise { + const detail = await this.getSkillCandidate(jobId, candidateId); + if ( + !detail.rollbackAvailable || + detail.targetHarnessVersion === null || + detail.rollbackTargetHarnessVersion === null + ) { + throw new Error(`Skill candidate "${candidateId}" has no rollback available`); + } + const result = await this.#dependencies.harnesses.selectVersionIfCurrent( + detail.rollbackTargetHarnessVersion, + detail.targetHarnessVersion, + ); + if (result.status === "conflict") { + throw new Error( + `Harness revision changed before rollback: expected ${result.expectedCurrent}, found ${result.actualCurrent}`, + ); + } + return { + jobId, + fromHarnessVersion: detail.targetHarnessVersion, + toHarnessVersion: detail.rollbackTargetHarnessVersion, + proposalIds: [candidateId], + }; + } + async activateSkillCandidate(jobId: string, candidateId: string): Promise { const detail = await this.getSkillCandidate(jobId, candidateId); if ( diff --git a/packages/evolution/test/control-service.test.ts b/packages/evolution/test/control-service.test.ts index d4cad3579..15a62079a 100644 --- a/packages/evolution/test/control-service.test.ts +++ b/packages/evolution/test/control-service.test.ts @@ -6,6 +6,7 @@ import type { MaterializedInvocation } from "@earendil-works/diffex-harness-trac import { afterEach, describe, expect, it, vi } from "vitest"; import { computeEvolutionContentDigest, + computeEvolutionTextDigest, type EvolutionArtifactReference, EvolutionBackgroundWorker, EvolutionControlPlane, @@ -567,6 +568,7 @@ describe("evolution control service", () => { draftId: "draft-1", groupId: "group-1", operation: "CREATE", + proposedName: "focused-verification", content: "---\nname: focused-verification\ndescription: Verify focused changes.\n---\n\nRun checks.\n", } as unknown as SkillDraft; const validation = { @@ -639,11 +641,21 @@ describe("evolution control service", () => { [validationReference.artifactId, validation], [evaluationReference.artifactId, evaluation], ]); - const activate = vi.fn().mockResolvedValue({ - groupId: group.groupId, - batchId: "batch-1", - targetHarnessVersion: `sha256:${"b".repeat(64)}`, - receipt: {}, + const targetHarnessVersion = `sha256:${"b".repeat(64)}`; + let selectedHarnessVersion = input.expectedHarnessBaseVersion; + const activate = vi.fn().mockImplementation(async () => { + selectedHarnessVersion = targetHarnessVersion; + return { + groupId: group.groupId, + batchId: "batch-1", + targetHarnessVersion, + receipt: {}, + }; + }); + const selectVersionIfCurrent = vi.fn(async (version: string, expectedCurrent: string) => { + expect(selectedHarnessVersion).toBe(expectedCurrent); + selectedHarnessVersion = version; + return { status: "selected" as const, version }; }); const service = new EvolutionControlService({ scope, @@ -653,15 +665,29 @@ describe("evolution control service", () => { }, harnesses: { listVersions: async () => ({ - currentVersion: input.expectedHarnessBaseVersion, + currentVersion: selectedHarnessVersion, versions: [ { manifest: { version: input.expectedHarnessBaseVersion, base: null }, state: { memory: "", skills: [] }, }, + { + manifest: { version: targetHarnessVersion, base: input.expectedHarnessBaseVersion }, + state: { + memory: "", + skills: [ + { + path: "skills/focused-verification/SKILL.md", + workspace: { repositoryId: "diffex" }, + contentHash: computeEvolutionTextDigest(draft.content), + }, + ], + }, + }, ], diagnostics: [], }), + selectVersionIfCurrent, }, ledger: { read: async () => [] }, skillCandidates: { list: async () => [group] }, @@ -697,6 +723,17 @@ describe("evolution control service", () => { batchId: "batch-1", }); expect(activate).toHaveBeenCalledWith(scope, group.groupId); + expect(await service.getSkillCandidate(job.jobId, candidate.candidateId)).toMatchObject({ + targetHarnessVersion, + rollbackTargetHarnessVersion: input.expectedHarnessBaseVersion, + rollbackAvailable: true, + activationAvailable: false, + }); + await expect(service.rollbackSkillCandidate(job.jobId, candidate.candidateId)).resolves.toMatchObject({ + fromHarnessVersion: targetHarnessVersion, + toHarnessVersion: input.expectedHarnessBaseVersion, + }); + expect(selectVersionIfCurrent).toHaveBeenCalledWith(input.expectedHarnessBaseVersion, targetHarnessVersion); }); it("reports durable latency, budget, coverage, conflict, and recovery observability", async () => { From c2f1f187fe66413b27fd6d048c7872a2660cc2cf Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Fri, 28 Aug 2026 23:42:56 -0400 Subject: [PATCH 3/3] docs(coding-agent): document evolved skill operations --- packages/coding-agent/README.md | 8 +++- packages/coding-agent/docs/docs.json | 4 ++ packages/coding-agent/docs/evolution.md | 55 +++++++++++++++++++++ packages/coding-agent/docs/packages.md | 6 ++- packages/coding-agent/docs/rpc.md | 6 +++ packages/coding-agent/docs/sdk.md | 27 +++++++++++ packages/coding-agent/docs/security.md | 8 ++++ packages/coding-agent/docs/settings.md | 10 ++++ packages/coding-agent/docs/skills.md | 10 +++- packages/coding-agent/docs/usage.md | 8 +++- packages/evolution/COUNTER.md | 2 + packages/evolution/RECOVERY.md | 4 +- packages/harness-state/README.md | 64 +++++++++++++------------ 13 files changed, 174 insertions(+), 38 deletions(-) create mode 100644 packages/coding-agent/docs/evolution.md diff --git a/packages/coding-agent/README.md b/packages/coding-agent/README.md index 2a3a0af77..2240f3c8f 100644 --- a/packages/coding-agent/README.md +++ b/packages/coding-agent/README.md @@ -174,8 +174,10 @@ Type `/` in the editor to trigger commands. [Extensions](#extensions) can regist | [`/llama`](docs/llama-cpp.md) | Download, load, and unload llama.cpp router models | | `/model` | Switch models | | `/scoped-models` | Enable/disable models for Ctrl+P cycling | -| `/settings` | Thinking level, theme, message delivery, transport | +| `/settings` | Thinking level, theme, installed skills, message delivery, transport | | [`/skills`](docs/skills.md#skill-commands) | Search and reference one or more skills in the editor | +| [`/evolve`](docs/evolution.md) | Inspect and control harness evolution and generated skills | +| [`/version`](docs/evolution.md#controls) | Select the active harness revision | | `/resume` | Pick from previous sessions | | `/new` | Start a new session | | `/name ` | Set session display name | @@ -230,6 +232,10 @@ On Windows Terminal, `Alt+Enter` is fullscreen by default. Remap it in [docs/ter Configure delivery in [settings](docs/settings.md): `steeringMode` and `followUpMode` can be `"one-at-a-time"` (default, waits for response) or `"all"` (delivers all queued at once). `transport` selects provider transport preference (`"sse"`, `"websocket"`, or `"auto"`) for providers that support multiple transports. +### Harness Evolution + +`/evolve` shows evidence eligibility, jobs, memory proposals, generated skill candidates, provenance, draft diffs, validation, evaluation, target revisions, activation, and rollback. `/version` selects a complete immutable harness revision. Memory and evolved skills change together at the next prompt boundary; in-flight and admitted queued requests remain pinned. The footer shows the active harness and background activity. See [Harness Evolution](docs/evolution.md). + --- ## Sessions diff --git a/packages/coding-agent/docs/docs.json b/packages/coding-agent/docs/docs.json index 6a8c25a72..44f63af66 100644 --- a/packages/coding-agent/docs/docs.json +++ b/packages/coding-agent/docs/docs.json @@ -31,6 +31,10 @@ "title": "Settings", "path": "settings.md" }, + { + "title": "Harness Evolution", + "path": "evolution.md" + }, { "title": "Keybindings", "path": "keybindings.md" diff --git a/packages/coding-agent/docs/evolution.md b/packages/coding-agent/docs/evolution.md new file mode 100644 index 000000000..0cd710229 --- /dev/null +++ b/packages/coding-agent/docs/evolution.md @@ -0,0 +1,55 @@ +# Harness Evolution + +Diffex can derive workspace-scoped guidance from persisted invocation evidence and store it in immutable harness revisions. A revision can contain `EVOLVE.md` memory and evolved `skills//SKILL.md` artifacts. The selected revision changes both together. + +## Controls + +Use `/evolve` to: + +- inspect evidence eligibility and running or completed jobs; +- start, resume, cancel, or browse evolution work; +- inspect memory proposals and their evidence; +- inspect skill-candidate provenance, textual draft diffs, deterministic validation, and retrieval/behavior evaluation; +- activate a skill candidate only after its persisted evaluation is accepted; +- inspect target revisions and roll back an activated evolved skill to its immutable parent revision; +- enable or disable future tracing and evolution work. + +Skill activation always requires explicit confirmation. Automatic skill activation is not enabled. Candidates that have not passed every required gate remain read-only. + +Use `/version` to inspect compatible revisions for the current workspace and select one. Selection validates the complete bundle. A revision with corrupt, unsupported, or workspace-mismatched artifacts is not activated. + +## Request boundaries + +Harness selection is resolved at prompt admission. A fresh prompt observes the selected revision; in-flight work and already admitted queued messages keep their pinned memory and skill versions. If `/version` changes while an evolution job is running, the job continues from its frozen base and cannot replace the newer selection. + +The footer identifies the active harness and shows background evolution activity. `/skills` lists only evolved skills from the active revision whose workspace identity matches the current workspace. + +## Automatic eligibility + +When evolution is enabled, automatic work requires all of the following: + +- more than 1,000,000 estimated unprocessed evidence tokens; +- at least three eligible settled invocations; +- no blocking job; +- a 20-minute cooldown after the previous automatic run. + +Evolution model calls use the configured provider and may consume provider quota. Manual `/evolve` work uses the same frozen evidence and validation boundaries. + +## Validation and security + +Source traces, tool output, files, and generated drafts are treated as untrusted data. Deterministic validation checks structure, provenance, workspace scope, immutable versions, inventory, and bounded sensitive-data rules. Retrieval and behavioral evaluation are separate gates. + +These checks do not prove semantic safety and are not a sandbox. Review candidate provenance, draft diffs, validation issues, and evaluation results before activation. See [Security](security.md). + +## Storage and recovery + +Evolution state is stored beneath the effective agent directory: + +- `traces/` contains immutable invocation evidence intervals; +- `evolution/` contains jobs, artifacts, candidate history, observations, receipts, and recovery state; +- `evolution-failures/` contains failure reports; +- `harness-state/` contains immutable revisions and the selected pointer. + +Interrupted jobs and candidate work resume from durable checkpoints. Writer retries use stable idempotency keys, leases, exact base revisions, and compare-and-swap selection. Rollback selects an existing immutable parent; it does not mutate historical revisions. + +Set `evolution.enabled` to `false` in global settings or use the toggle in `/evolve` to stop future tracing and background/manual evolution controls. Disabling evolution does not unselect the current harness or remove already active evolved skills. diff --git a/packages/coding-agent/docs/packages.md b/packages/coding-agent/docs/packages.md index 094ecb781..6e5bb0ea5 100644 --- a/packages/coding-agent/docs/packages.md +++ b/packages/coding-agent/docs/packages.md @@ -219,11 +219,13 @@ Filter what a package loads using the object form in settings: ## Enable and Disable Resources -Use `diffex config` to enable or disable extensions, skills, prompt templates, and themes from installed packages and local directories. `diffex config` starts in global settings (`~/.diffex/agent/settings.json`); press Tab to switch between global and project-local modes. Use `diffex config -l` to start in project overrides (`.diffex/settings.json`) with inherited global resources dimmed. +Use `diffex config` to enable or disable extensions, skills, prompt templates, and themes from installed packages and local directories. `diffex config` starts in global settings (`~/.diffex/agent/settings.json`); press Tab to switch between global and project-local modes. Use `diffex config -l` to start in project overrides (`.diffex/settings.json`) with inherited global resources dimmed. Interactive sessions also expose installed-skill inspection and toggles under **Installed skills** in `/settings`; use `/reload` after changing them. ## Scope and Deduplication -Packages can appear in both global and project settings. If the same package appears in both, the project entry wins unless the project entry has `autoload: false`, in which case it is applied as a delta over the global entry. Identity is determined by: +Packages can appear in both global and project settings. If the same package appears in both, the project entry wins unless the project entry has `autoload: false`, in which case it is applied as a delta over the global entry. Project-installed skills take precedence over user and package skills with the same name. An installed skill also retains the unsuffixed name when it collides with an evolved skill. See [Skills](skills.md) for complete catalog and collision rules. + +Identity is determined by: - npm: package name - git: repository URL without ref diff --git a/packages/coding-agent/docs/rpc.md b/packages/coding-agent/docs/rpc.md index ebb9f5d73..028d0486e 100644 --- a/packages/coding-agent/docs/rpc.md +++ b/packages/coding-agent/docs/rpc.md @@ -833,6 +833,12 @@ Each command has: **Note**: Built-in TUI commands (`/settings`, `/hotkeys`, etc.) are not included. They are handled only in interactive mode and would not execute if sent via `prompt`. +## Evolution and Harness Revisions + +Persisted CLI RPC sessions participate in invocation tracing and background evolution when `evolution.enabled` is enabled. The selected harness revision is refreshed at prompt admission, and explicit skill references use the same versioned expansion path as other modes. + +`/evolve` and `/version` are interactive-only controls. RPC currently has no commands to list, select, activate, or roll back harness revisions, and `get_state` does not expose the active revision. An RPC process therefore follows the globally selected compatible harness unless its embedded session was explicitly pinned through SDK composition. + ## Events Events are streamed to stdout as JSON lines during agent operation. Events do not generally include an `id` field; `bash_execution_update` includes the `id` of its originating `bash` command when one was provided. diff --git a/packages/coding-agent/docs/sdk.md b/packages/coding-agent/docs/sdk.md index 1052ddbe3..1feb74031 100644 --- a/packages/coding-agent/docs/sdk.md +++ b/packages/coding-agent/docs/sdk.md @@ -770,6 +770,33 @@ const { session } = await createAgentSession({ resourceLoader: loader }); > See [examples/sdk/04-skills.ts](../examples/sdk/04-skills.ts) +### Harness Revisions and Evolved Skills + +`createAgentSession()` loads the selected compatible harness by default. Its memory and workspace-matching evolved skills are pinned together for each admitted request. The SDK does not start invocation tracing or evolution workers; those are composed by the CLI runtime. + +```typescript +const active = session.activeHarnessVersion; +const bundle = session.harnessVersionBundle; +const versions = await session.listHarnessVersions(); + +await session.switchHarnessVersion(versions.versions[0]!.manifest.version); +const references = session.createSkillReferences(); +``` + +Use explicit harness options when composing a custom runtime: + +```typescript +const { session } = await createAgentSession({ + harnessStateStore, + harnessVersionBundle, + pinHarnessVersion: true, +}); +``` + +`pinHarnessVersion: true` prevents later prompt admissions from following changes to the selected global pointer. Child sessions pin the parent's memory and evolved-skill snapshot. Custom `ResourceLoader` implementations must implement `setEvolvedSkills()`; `getSkills()` must return the unified installed/evolved catalog. + +See [Harness Evolution](evolution.md) and [Skills](skills.md). + ### Context Files ```typescript diff --git a/packages/coding-agent/docs/security.md b/packages/coding-agent/docs/security.md index aba4dadc1..e51ffef8f 100644 --- a/packages/coding-agent/docs/security.md +++ b/packages/coding-agent/docs/security.md @@ -52,6 +52,14 @@ Common patterns are documented in [Containerization](containerization.md): If you bind-mount a host workspace read/write, writes from inside the container or VM can still modify host files. Use read-only mounts or copy files into and out of the sandbox when you need stronger protection from unintended writes. +## Evolution Data and Generated Skills + +Invocation traces, evolution artifacts and failures, candidate history, lifecycle observations, writer receipts, and harness revisions persist beneath the effective agent directory. They can contain session evidence, generated instructions, skill content, paths, and runtime metadata. Evolution model calls may consume provider quota. + +Evolved skills are generated instructions and can direct the model to use tools or referenced assets. Structural, provenance, scope, security, retrieval, and behavioral gates reduce risk but do not prove semantic safety and are not a sandbox. Project trust does not sandbox an already selected harness revision. + +Review `/evolve` provenance, draft diffs, validation, and evaluation before activation. Use `/version` and rollback controls to select immutable revisions, and use OS or container isolation for unattended work. See [Harness Evolution](evolution.md). + ## Reporting Security Issues To report a security issue, follow the repository [Security Policy](https://github.com/diffexai/diffex/blob/main/SECURITY.md). Do not open a public issue for security-sensitive reports. diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index 4ea88222a..5a3f13198 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -108,6 +108,14 @@ Set `DIFFEX_SKIP_VERSION_CHECK=1` to disable the upstream compatibility release } ``` +### Evolution + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `evolution.enabled` | boolean | `true` | Global-only. Enables future invocation tracing and manual/background evolution controls. Disabling it does not unselect the active harness or remove active evolved skills. | + +Automatic work requires more than 1,000,000 estimated unprocessed evidence tokens, at least three eligible settled invocations, and a 20-minute cooldown after the previous automatic run. Evolution model calls may consume provider quota. See [Harness Evolution](evolution.md). + ### Compaction | Setting | Type | Default | Description | @@ -234,6 +242,8 @@ When multiple sources specify a session directory, precedence is `--session-dir` These settings define where to load extensions, skills, prompts, and themes from. +Use **Installed skills** in `/settings` to inspect global and workspace skill paths, sources, scopes, and enabled state. Toggle entries to persist global or project overrides, then use `/reload` to apply them to the current session. Evolved skills are read-only outside `/evolve` and `/version`. + Paths in `~/.diffex/agent/settings.json` resolve relative to `~/.diffex/agent`. Paths in `.diffex/settings.json` resolve relative to `.diffex`. Absolute paths and `~` are supported. | Setting | Type | Default | Description | diff --git a/packages/coding-agent/docs/skills.md b/packages/coding-agent/docs/skills.md index 9b6ceb1aa..2775410f8 100644 --- a/packages/coding-agent/docs/skills.md +++ b/packages/coding-agent/docs/skills.md @@ -72,9 +72,15 @@ For project-level Claude Code skills, add to `.diffex/settings.json`: This is progressive disclosure: only names, descriptions, and locations are always in context; full instructions load on-demand. The TUI, system prompt, SDK, and evolution snapshots all read from the same catalog. Trace observations do not infer that a skill was used or caused an outcome. +### Evolved Skills + +Evolved skills are immutable, workspace-scoped artifacts in a harness revision. Diffex exposes them only when the active revision and workspace identity match. Selecting another revision changes `EVOLVE.md` memory and the evolved-skill set together at the next prompt boundary; in-flight and already admitted queued requests remain pinned. + +An evolved reference binds its skill ID, content version, digest, workspace, and harness revision. A reference from an inactive revision is rejected. `/reload` refreshes installed skills while preserving the evolved overlay from the active harness. Use `/evolve` to inspect provenance, validation, evaluation, activation, and rollback. See [Harness Evolution](evolution.md). + ## Skill Commands -Use `/skills` in interactive mode to choose installed workspace, installed global, or evolved skills, then search and select one or more skills within that category. Selected skills appear in the editor as purple skill-name tokens while retaining versioned references internally, so you can add the task before submitting. Each reference loads that exact reviewed skill content for one request, including queued steering and follow-up messages. Skills not selected remain progressively disclosed by name, description, and location. +Use `/skills` in interactive mode to choose installed workspace, installed global, or evolved skills, then search and select one or more skills within that category. Selected skills appear in the editor as purple skill-name tokens while retaining versioned references internally, so you can add the task before submitting. Each reference loads that exact captured skill content for one request, including queued steering and follow-up messages. Skills not selected remain progressively disclosed by name, description, and location. To install a package containing skills from an interactive terminal, run `diffex install ` and choose `Install globally` or `Install for this workspace`. The prompt shows both destinations and the trust implications. Workspace installation requires project trust and never falls back to global installation. Non-interactive installs retain the global default; pass `-l` to select workspace scope explicitly. @@ -174,7 +180,7 @@ Unknown frontmatter fields are ignored. **Exception:** Skills with missing description are not loaded. -Name collisions (same name from different locations) warn and keep the first skill found. +Installed skill name precedence is project, user, then package order. Collisions warn and retain the higher-precedence installed skill. When an evolved skill has the same name, the installed skill retains the unsuffixed name and the evolved skill is exposed as `-Evolved`; if that alias also collides, the evolved skill is omitted with a diagnostic. ## Example diff --git a/packages/coding-agent/docs/usage.md b/packages/coding-agent/docs/usage.md index 29d772769..f497523aa 100644 --- a/packages/coding-agent/docs/usage.md +++ b/packages/coding-agent/docs/usage.md @@ -40,8 +40,10 @@ Type `/` in the editor to open command completion. Extensions can register custo | [`/llama`](llama-cpp.md) | Download, load, and unload llama.cpp router models | | `/model` | Switch models | | `/scoped-models` | Enable/disable models for Ctrl+P cycling | -| `/settings` | Thinking level, theme, message delivery, transport | +| `/settings` | Thinking level, theme, installed skills, message delivery, transport | | `/skills` | Browse installed workspace, installed global, and evolved skills | +| [`/evolve`](evolution.md) | Inspect and control harness evolution and generated skills | +| [`/version`](evolution.md#controls) | Select the active harness revision | | `/resume` | Pick from previous sessions | | `/new` | Start a new session | | `/name ` | Set session display name | @@ -125,6 +127,10 @@ On Windows Terminal, Alt+Enter is fullscreen by default. Remap it as described i Configure delivery in [Settings](settings.md) with `steeringMode` and `followUpMode`. +## Harness Evolution + +`/evolve` shows evidence eligibility, jobs, memory proposals, generated skill candidates, provenance, draft diffs, validation, evaluation, target revisions, activation, and rollback. `/version` selects a complete immutable harness revision. Memory and evolved skills change together at the next prompt boundary; in-flight and admitted queued requests remain pinned. An active evolution job continues from its frozen base if the selected revision changes. The footer shows the active harness and background activity. See [Harness Evolution](evolution.md). + ## Sessions Sessions are saved automatically to `~/.diffex/agent/sessions/`, organized by working directory. diff --git a/packages/evolution/COUNTER.md b/packages/evolution/COUNTER.md index 50c1d23d9..0715de9cf 100644 --- a/packages/evolution/COUNTER.md +++ b/packages/evolution/COUNTER.md @@ -60,3 +60,5 @@ Each observation binds one exact skill ID, version, content digest, harness revi Final observations are unique per workspace, exact skill version, and top-level task lineage. Direct user attribution and deterministic test, tool, replay, or environment outcomes receive more report weight than model self-assessment or inference. Candidate observations remain auditable but do not contribute to final outcome weights or recommendations. Lifecycle reports consider recommendations only for the exact evolved skill versions active in the supplied workspace snapshot. Outcomes for the same immutable skill version remain applicable when that version reappears in a later harness revision, while the observation retains its original harness attribution. Different skill versions and installed skills remain available for audit but cannot target the evolved-skill writer. Reports can recommend narrowing triggers, revising a procedure, merging active evolved duplicates, deactivating a harmful skill, or retiring obsolete guidance. A recommendation is not a mutation: it requires a new Skill Curator candidate and the existing deterministic validation, retrieval evaluation, and behavioral evaluation gates before the skill writer can act. + +The observation/report APIs and filesystem persistence are implemented. The coding-agent runtime does not yet derive final lifecycle observations from trace and deterministic outcome evidence automatically. diff --git a/packages/evolution/RECOVERY.md b/packages/evolution/RECOVERY.md index 888ad1d5a..f0f97e289 100644 --- a/packages/evolution/RECOVERY.md +++ b/packages/evolution/RECOVERY.md @@ -26,7 +26,7 @@ Recovery never treats partial output as completed work and never feeds evolution | Successful rollback selection | The recorded immutable parent is still present and the evolved revision is current. | Select the parent with compare-and-swap and refresh on the next prompt boundary. | Existing in-flight prompts stay pinned to their admitted revision. | | Unsupported skill or orchestration target | The target registry has no enabled writer route. | Preserve the finding as deferred evidence with no executable payload. | Disabled target kinds cannot reach filesystem mutation or activation. | | Interrupted skill-candidate consolidation | A completed evolution job has checkpointed candidates but no completed candidate-work event. | Resume from append-only registry events with the same source-job idempotency key; repair missing scores or transitions before completion. | Candidate proposal revisions remain immutable, retries do not add recurrence, and partial registry appends cannot activate a skill. | -| Interrupted skill synthesis or draft validation | An evidence-qualified group has no draft, a draft-record event lacks its `drafted` transition, or a validation-record event lacks its terminal transition. | Resume with the revision-bound idempotency keys; reuse recorded immutable artifacts and repair the missing transition before completing candidate work. | Only evidence-qualified groups invoke synthesis, and only deterministically accepted drafts reach `structurally-valid`; no draft has a writer or activation path. | +| Interrupted skill synthesis, validation, evaluation, write, or activation | A qualified group is missing the next immutable artifact or transition, or an accepted candidate has a durable write without a matching activation. | Resume with revision-bound idempotency keys; reuse recorded artifacts and receipts, repair the missing transition, and revalidate the exact accepted group before writing or activation. | Only evidence-qualified groups invoke synthesis, only deterministically valid drafts enter evaluation, only accepted evaluations reach the writer, and retries cannot stage or activate a skill twice. | ## Operational observability @@ -40,4 +40,4 @@ These operational records remain outside materialized product-runtime evidence, Deterministic validation proves provenance, recurrence, immutable closure, score bounds, structural eligibility, workspace and version freshness, and bounded sensitive-data checks. Candidate priority scoring separately orders evaluation work; it cannot skip later gates, accept a candidate, or activate a skill. It does not prove semantic usefulness, that natural-language evidence entails a proposal, or the absence of all semantically unsafe guidance. -Post-activation outcome evaluation and automatic regression rollback are intentionally outside this recovery contract. +Skill lifecycle observations and reports remain outside harness content and can recommend a new candidate cycle after activation. Automatic regression rollback is intentionally outside this recovery contract. diff --git a/packages/harness-state/README.md b/packages/harness-state/README.md index d4600fe47..58d02e04f 100644 --- a/packages/harness-state/README.md +++ b/packages/harness-state/README.md @@ -4,7 +4,7 @@ Persistent, content-addressed harness revisions for Diffex. ## Storage -`HarnessStateStore` stores global revisions beneath an agent directory: +`HarnessStateStore` stores revisions beneath an agent directory: ```text /harness-state/ @@ -12,50 +12,54 @@ Persistent, content-addressed harness revisions for Diffex. `-- versions/ `-- sha256-/ |-- manifest.json - `-- EVOLVE.md + |-- EVOLVE.md + `-- skills/ + `-- / + `-- SKILL.md ``` -Revision identifiers use canonical `sha256:` strings in APIs and data. -Directory names replace the colon with a dash for Windows portability. +Revision identifiers use canonical `sha256:` strings in APIs and data. Directory names replace the colon with a dash for Windows portability. -Each manifest contains `schemaVersion`, `version`, `base`, content-addressed harness metadata including the next evolution lesson number, and a deterministic artifact inventory. -The current manifest schema version is 2. -The revision hash covers the schema version, parent revision, harness metadata, and sorted artifact descriptors, but not the `version` field itself. -Each artifact retains its own content hash over its exact bytes. -Including the parent revision makes repeated evolution with identical artifact content produce a distinct successor. +The schema-v2 manifest contains the revision, immutable parent, evolution-memory metadata, and a closed sorted artifact inventory. The revision hash covers the schema version, parent, metadata, and artifact descriptors; each artifact also records a hash over its exact bytes. Including the parent means identical content on different lineages produces different revisions. -Only one `evolution-memory` artifact at `EVOLVE.md` is supported for activation today. -The manifest schema can represent future artifact kinds so incompatible revisions can be inspected and reported, but activation rejects every kind the runtime cannot fully understand. +Supported artifacts are one `evolution-memory` at `EVOLVE.md` and validated workspace-scoped `skill` artifacts at canonical `skills//SKILL.md` paths. The manifest can describe future kinds for inspection, but activation rejects a bundle containing a kind the runtime cannot fully understand. + +## State and inheritance + +`HarnessStateInput` requires `memory` and accepts optional `skills`: + +- omitting `skills` while saving from a base inherits that base's exact skill set; +- passing `skills: []` explicitly clears the skill set; +- passing a skill list replaces the complete inherited set. + +Memory and skills are therefore saved, validated, selected, and rolled back as one revision. The store persists artifacts; it does not synthesize, evaluate, or decide when to activate skills. Session-level request pinning belongs to the coding-agent runtime. ## Validation -Activation validates the complete bundle before returning any state. -It rejects malformed or noncanonical manifests, revision and content hash mismatches, undeclared or missing files, duplicate and nonportable paths, incorrect casing, symbolic links, non-regular files, malformed UTF-8, leading BOMs, and unsupported artifact kinds. -`EVOLVE.md` may be exactly empty for the root revision. -Non-empty whitespace-only memory remains invalid. +Loading or selecting validates the complete bundle. Validation rejects malformed or noncanonical manifests, revision and content hash mismatches, undeclared or missing files, duplicate and nonportable paths, incorrect casing, symbolic links, non-regular files, malformed UTF-8, leading BOMs, unsupported artifact kinds, malformed skill frontmatter, and missing or mismatched workspace metadata. + +`EVOLVE.md` may be exactly empty for the root revision. Non-empty whitespace-only memory is invalid. ## API - `computeHarnessVersion(input)` computes a canonical revision identifier. -- `serializeHarnessVersion(input)` exposes the canonical serialization used by the hash. +- `serializeHarnessVersion(input)` exposes the canonical hash serialization. - `harnessVersionDirectoryName(version)` creates the portable directory name. -- `saveHarnessState(state, destinationPath, base, metadata)` writes an immutable, idempotent revision bundle. +- `computeHarnessSkillId(workspace, path)` derives stable workspace/path identity for a harness skill. +- `validateHarnessSkillContent(path, content)` validates canonical Agent Skills frontmatter and content. +- `saveHarnessState(state, destinationPath, base, metadata)` writes an immutable revision bundle. - `loadHarnessState(bundlePath)` validates and loads one complete revision bundle. -- `new HarnessStateStore(agentDir)` manages the global store beneath an effective agent directory. +- `new HarnessStateStore(agentDir)` manages the global store. -The store provides these operations: +The store provides: -- `initialize()` creates and selects the empty root if no pointer exists. +- `initialize()` creates and selects the empty root when no pointer exists. - `loadCurrent()` strictly validates and loads the selected revision. -- `listVersions()` returns compatible bundles and per-path diagnostics for corrupt or incompatible entries. - Diagnostics retain validated manifest and EVOLVE state when an incompatible future artifact is the only activation blocker. -- `saveVersion(state, base, metadata?)` creates an immutable revision without selecting it and inherits omitted metadata from its base. +- `loadVersion(version)` loads one exact immutable revision without selecting it. +- `listVersions()` returns compatible bundles and per-path diagnostics for corrupt or incompatible entries. Diagnostics retain validated manifest and memory state when a future artifact is the only activation blocker. +- `saveVersion(state, base, metadata?)` creates an immutable revision without selecting it and inherits omitted metadata and skills from its base. - `selectVersion(version)` validates the complete target and atomically replaces the pointer. -- `advance(state, expectedBase, metadata?)` commits and selects a successor only when the selected revision still matches `expectedBase`, inheriting omitted metadata. - -Initialization, selection, and advancement share a cross-process store lock. -Version writes stage beside their destination and are idempotent only when an existing revision validates to the same canonical content. -Pointer replacement uses a staged file and atomic rename. -An expected-base race returns a discriminated `conflict` result and does not create or select a silent branch. +- `selectVersionIfCurrent(version, expectedCurrent)` selects only when the pointer still matches the expected revision. Skill activation and rollback use this compare-and-swap primitive. +- `advance(state, expectedBase, metadata?)` commits and selects a successor only when the selected revision still matches `expectedBase`. -The package does not implement skill generation, source mutation, orchestration loading, build execution, restart handling, rollback, garbage collection, aliases, project-local stores, or session-level revision pinning. +Initialization, selection, and advancement share a cross-process store lock. Version writes stage beside their destination and are idempotent only when an existing revision validates to the same canonical content. Pointer replacement uses a staged file and atomic rename. Expected-base races return discriminated conflict results and never create or select a silent branch.