From 5e0284c038a5839aacc0816dd316a7437009f404 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 11:39:10 -0400 Subject: [PATCH 01/10] feat(agent): version evolved skill artifacts --- package-lock.json | 3 +- .../install-lock/package-lock.json | 3 +- packages/coding-agent/npm-shrinkwrap.json | 3 +- .../test/suite/runtime-telemetry.test.ts | 2 +- .../evolution/src/evolution-memory-writer.ts | 14 +- .../test/evolution-memory-writer.test.ts | 60 +++ packages/harness-state/package.json | 3 +- packages/harness-state/src/index.ts | 348 +++++++++++++++--- .../harness-state/test/harness-state.test.ts | 186 +++++++++- 9 files changed, 557 insertions(+), 65 deletions(-) diff --git a/package-lock.json b/package-lock.json index f8553f15c..121a08121 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6037,7 +6037,8 @@ "license": "MIT", "dependencies": { "proper-lockfile": "4.1.2", - "typebox": "1.3.7" + "typebox": "1.3.7", + "yaml": "2.9.0" }, "devDependencies": { "@types/proper-lockfile": "4.1.4", diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index d0ea613e0..e8d64790c 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -546,7 +546,8 @@ "license": "MIT", "dependencies": { "proper-lockfile": "4.1.2", - "typebox": "1.3.7" + "typebox": "1.3.7", + "yaml": "2.9.0" }, "engines": { "node": ">=22.19.0" diff --git a/packages/coding-agent/npm-shrinkwrap.json b/packages/coding-agent/npm-shrinkwrap.json index c4c5f1700..dd187506e 100644 --- a/packages/coding-agent/npm-shrinkwrap.json +++ b/packages/coding-agent/npm-shrinkwrap.json @@ -536,7 +536,8 @@ "license": "MIT", "dependencies": { "proper-lockfile": "4.1.2", - "typebox": "1.3.7" + "typebox": "1.3.7", + "yaml": "2.9.0" }, "engines": { "node": ">=22.19.0" diff --git a/packages/coding-agent/test/suite/runtime-telemetry.test.ts b/packages/coding-agent/test/suite/runtime-telemetry.test.ts index 99c114667..b5279fe35 100644 --- a/packages/coding-agent/test/suite/runtime-telemetry.test.ts +++ b/packages/coding-agent/test/suite/runtime-telemetry.test.ts @@ -178,7 +178,7 @@ describe("coding-agent runtime telemetry", () => { const store = new HarnessStateStore(bootstrap.tempDir); const version = `sha256:${"1".repeat(64)}`; const bundle: HarnessVersionBundle = { - state: { memory: "captured memory" }, + state: { memory: "captured memory", skills: [] }, manifest: { schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, version, diff --git a/packages/evolution/src/evolution-memory-writer.ts b/packages/evolution/src/evolution-memory-writer.ts index 13fbf9bf2..ad5c147f6 100644 --- a/packages/evolution/src/evolution-memory-writer.ts +++ b/packages/evolution/src/evolution-memory-writer.ts @@ -4,6 +4,7 @@ import { basename, dirname, join, resolve } from "node:path"; import { computeHarnessVersion, HARNESS_VERSION_SCHEMA_VERSION, + type HarnessArtifactManifest, type HarnessStateStore, type HarnessVersion, type HarnessVersionBundle, @@ -298,12 +299,16 @@ function resultingHarnessVersion( memory: string, base: HarnessVersion, metadata: HarnessVersionMetadata, + baseArtifacts: readonly HarnessArtifactManifest[], ): HarnessVersion { return computeHarnessVersion({ schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, base, metadata, - artifacts: [{ kind: "evolution-memory", path: "EVOLVE.md", contentHash: memoryArtifactContentHash(memory) }], + artifacts: [ + { kind: "evolution-memory", path: "EVOLVE.md", contentHash: memoryArtifactContentHash(memory) }, + ...baseArtifacts.filter((artifact) => artifact.kind !== "evolution-memory"), + ], }); } @@ -828,7 +833,12 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { throw new Error("EVOLVE.md candidate item index is inconsistent after applying mutations"); } const harnessMetadata: HarnessVersionMetadata = { evolutionMemory: { nextLessonNumber } }; - const nextVersion = resultingHarnessVersion(memory, batch.expectedBaseVersion, harnessMetadata); + const nextVersion = resultingHarnessVersion( + memory, + batch.expectedBaseVersion, + harnessMetadata, + base.manifest.artifacts, + ); items = items.map((item) => { if (addedIds.has(item.memoryItemId)) return { ...item, creationRevision: nextVersion }; if (removedIds.has(item.memoryItemId)) { diff --git a/packages/evolution/test/evolution-memory-writer.test.ts b/packages/evolution/test/evolution-memory-writer.test.ts index 58e759c92..b87326eb4 100644 --- a/packages/evolution/test/evolution-memory-writer.test.ts +++ b/packages/evolution/test/evolution-memory-writer.test.ts @@ -548,6 +548,66 @@ describe("evolution memory writer", () => { ); }); + it("preserves evolved skill artifacts while activating a memory mutation", async () => { + const root = await mkdtemp(join(tmpdir(), "diffex-evolution-writer-skills-")); + temporaryDirectories.push(root); + const harnessState = new HarnessStateStore(join(root, "agent")); + const empty = await harnessState.initialize(); + const skillContent = + "---\nname: review-workflow\ndescription: Review changes with focused checks.\n---\n\n# Review workflow\n"; + const initial = await harnessState.advance( + { + memory: "Baseline memory.\n", + skills: [ + { + path: "skills/review-workflow/SKILL.md", + content: skillContent, + workspace: { repositoryId: "global", directoryId: "directory-1" }, + }, + ], + }, + empty.manifest.version, + ); + if (initial.status !== "advanced") throw new Error("Expected skill harness advance"); + const writer = new EvolutionMemoryWriter({ + version: "1.0.0", + harnessState, + statePath: join(root, "writer-state"), + }); + const snapshot = await writer.captureSnapshot(); + const selectedBatch = batch(snapshot, [ + { + mutationId: "mutation-preserve-skill", + proposalId: "proposal-preserve-skill", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: null, anchor: null }, + content: "Keep skill and memory activation atomic.", + intendedScope: ["repository"], + }, + }, + ]); + + const writeResult = await writer.write(selectedBatch, { + idempotencyKey: "write-preserve-skill", + lease: writerLease, + }); + const activation = await writer.activate(selectedBatch, writeResult, { + idempotencyKey: "activate-preserve-skill", + lease: writerLease, + }); + const activated = await harnessState.loadCurrent(); + + expect(activated.manifest.version).toBe(activation.resultingHarnessVersion); + expect(activated.state.memory).toContain("Keep skill and memory activation atomic."); + expect(activated.state.skills).toHaveLength(1); + expect(activated.state.skills[0]?.content).toBe(skillContent); + expect(activated.manifest.artifacts.filter((artifact) => artifact.kind === "skill")).toEqual( + initial.bundle.manifest.artifacts.filter((artifact) => artifact.kind === "skill"), + ); + }); + it("rejects a partial batch without publishing any mutation", async () => { const { harnessState, initial, initialMemory, snapshot, writer } = await fixture(); const selectedBatch = batch(snapshot, [ diff --git a/packages/harness-state/package.json b/packages/harness-state/package.json index ccf97bcc6..a9086fc44 100644 --- a/packages/harness-state/package.json +++ b/packages/harness-state/package.json @@ -38,7 +38,8 @@ }, "dependencies": { "proper-lockfile": "4.1.2", - "typebox": "1.3.7" + "typebox": "1.3.7", + "yaml": "2.9.0" }, "devDependencies": { "@types/proper-lockfile": "4.1.4", diff --git a/packages/harness-state/src/index.ts b/packages/harness-state/src/index.ts index 1458ae026..3d2f317de 100644 --- a/packages/harness-state/src/index.ts +++ b/packages/harness-state/src/index.ts @@ -6,6 +6,7 @@ import { TextDecoder, TextEncoder } from "node:util"; import lockfile from "proper-lockfile"; import Type from "typebox"; import { Check } from "typebox/value"; +import { parse } from "yaml"; const SHA256_PATTERN = "^sha256:[0-9a-f]{64}$"; const VERSION_DIRECTORY_PATTERN = /^sha256-([0-9a-f]{64})$/; @@ -16,7 +17,10 @@ const STORE_DIRECTORY_NAME = "harness-state"; const VERSIONS_DIRECTORY_NAME = "versions"; const LOCK_NAME = ".harness-state.lock"; const LOCK_STALE_AFTER_MS = 30_000; -const SUPPORTED_ARTIFACT_KIND = "evolution-memory"; +const EVOLUTION_MEMORY_ARTIFACT_KIND = "evolution-memory"; +const SKILL_ARTIFACT_KIND = "skill"; +const MAX_SKILL_NAME_LENGTH = 64; +const MAX_SKILL_DESCRIPTION_LENGTH = 1024; const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); const UTF8_ENCODER = new TextEncoder(); @@ -28,14 +32,94 @@ export type HarnessVersion = string; const NonEmptyEvolutionMemorySchema = Type.String({ pattern: "^(?!\\uFEFF)(?=[\\s\\S]*\\S)(?![\\s\\S]*[\\uD800-\\uDFFF])[\\s\\S]+$", }); +const EvolutionMemorySchema = Type.Union([Type.Literal(""), NonEmptyEvolutionMemorySchema]); +const SkillContentSchema = Type.String({ + pattern: "^(?!\\uFEFF)(?![\\s\\S]*[\\uD800-\\uDFFF])[\\s\\S]+$", +}); +export const HarnessWorkspaceIdentitySchema = Type.Readonly( + Type.Object( + { + repositoryId: Type.String({ minLength: 1 }), + directoryId: Type.Optional(Type.String({ minLength: 1 })), + }, + { additionalProperties: false }, + ), +); +export interface HarnessWorkspaceIdentity { + readonly repositoryId: string; + readonly directoryId?: string; +} + +export const HarnessSkillInputSchema = Type.Readonly( + Type.Object( + { + path: Type.String({ minLength: 1 }), + content: SkillContentSchema, + workspace: HarnessWorkspaceIdentitySchema, + }, + { additionalProperties: false }, + ), +); +export interface HarnessSkillInput { + readonly path: string; + readonly content: string; + readonly workspace: HarnessWorkspaceIdentity; +} + +export const HarnessSkillSchema = Type.Readonly( + Type.Object( + { + path: Type.String({ minLength: 1 }), + filePath: Type.String({ minLength: 1 }), + baseDir: Type.String({ minLength: 1 }), + name: Type.String({ minLength: 1 }), + description: Type.String({ minLength: 1 }), + content: SkillContentSchema, + contentHash: HarnessVersionSchema, + workspace: HarnessWorkspaceIdentitySchema, + disableModelInvocation: Type.Boolean(), + }, + { additionalProperties: false }, + ), +); +export interface HarnessSkill { + readonly path: string; + readonly filePath: string; + readonly baseDir: string; + readonly name: string; + readonly description: string; + readonly content: string; + readonly contentHash: HarnessVersion; + readonly workspace: HarnessWorkspaceIdentity; + readonly disableModelInvocation: boolean; +} + export const HarnessStateSchema = Type.Readonly( Type.Object( - { memory: Type.Union([Type.Literal(""), NonEmptyEvolutionMemorySchema]) }, + { + memory: EvolutionMemorySchema, + skills: Type.Readonly(Type.Array(HarnessSkillSchema)), + }, { additionalProperties: false }, ), ); export interface HarnessState { readonly memory: string; + readonly skills: readonly HarnessSkill[]; +} + +export const HarnessStateInputSchema = Type.Readonly( + Type.Object( + { + memory: EvolutionMemorySchema, + skills: Type.Optional(Type.Readonly(Type.Array(HarnessSkillInputSchema))), + }, + { additionalProperties: false }, + ), +); +export interface HarnessStateInput { + readonly memory: string; + readonly skills?: readonly HarnessSkillInput[]; } export const HarnessVersionMetadataSchema = Type.Readonly( @@ -63,6 +147,7 @@ export const HarnessArtifactManifestSchema = Type.Readonly( kind: Type.String({ minLength: 1 }), path: Type.String({ minLength: 1 }), contentHash: HarnessVersionSchema, + workspace: Type.Optional(HarnessWorkspaceIdentitySchema), }, { additionalProperties: false }, ), @@ -71,6 +156,7 @@ export interface HarnessArtifactManifest { readonly kind: string; readonly path: string; readonly contentHash: HarnessVersion; + readonly workspace?: HarnessWorkspaceIdentity; } export const HarnessVersionManifestSchema = Type.Readonly( @@ -175,10 +261,30 @@ function compareArtifacts(left: HarnessArtifactManifest, right: HarnessArtifactM return ( compareText(left.kind, right.kind) || compareText(left.path, right.path) || - compareText(left.contentHash, right.contentHash) + compareText(left.contentHash, right.contentHash) || + compareText(left.workspace?.repositoryId ?? "", right.workspace?.repositoryId ?? "") || + compareText(left.workspace?.directoryId ?? "", right.workspace?.directoryId ?? "") ); } +function canonicalArtifact(artifact: HarnessArtifactManifest): HarnessArtifactManifest { + return { + kind: artifact.kind, + path: artifact.path, + contentHash: artifact.contentHash, + ...(artifact.workspace === undefined + ? {} + : { + workspace: { + repositoryId: artifact.workspace.repositoryId, + ...(artifact.workspace.directoryId === undefined + ? {} + : { directoryId: artifact.workspace.directoryId }), + }, + }), + }; +} + export function serializeHarnessVersion(input: HarnessVersionInput): string { if ( input.schemaVersion !== HARNESS_VERSION_SCHEMA_VERSION || @@ -195,11 +301,7 @@ export function serializeHarnessVersion(input: HarnessVersionInput): string { if (!Check(HarnessVersionMetadataSchema, input.metadata)) { throw new Error("Invalid harness version metadata"); } - const artifacts = [...input.artifacts].sort(compareArtifacts).map(({ kind, path, contentHash }) => ({ - kind, - path, - contentHash, - })); + const artifacts = [...input.artifacts].sort(compareArtifacts).map(canonicalArtifact); return JSON.stringify({ schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, base: input.base, @@ -263,6 +365,45 @@ function parseManifest(bytes: Uint8Array, manifestPath: string): HarnessVersionM return parsed as HarnessVersionManifest; } +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +interface ParsedSkillFrontmatter { + readonly name: string; + readonly description: string; + readonly disableModelInvocation: boolean; +} + +function parseSkillFrontmatter(content: string, artifactPath: string): ParsedSkillFrontmatter { + const match = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/.exec(content); + if (match === null) throw new Error(`Harness skill "${artifactPath}" must start with YAML frontmatter`); + let parsed: unknown; + try { + parsed = parse(match[1] ?? ""); + } catch (cause) { + throw new Error(`Harness skill "${artifactPath}" has invalid YAML frontmatter`, { cause }); + } + if (!isRecord(parsed)) throw new Error(`Harness skill "${artifactPath}" frontmatter must be a mapping`); + const name = parsed.name; + const description = parsed.description; + const disableModelInvocation = parsed["disable-model-invocation"]; + if (typeof name !== "string" || name.length > MAX_SKILL_NAME_LENGTH || !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(name)) { + throw new Error(`Harness skill "${artifactPath}" has an invalid name`); + } + if ( + typeof description !== "string" || + description.trim() === "" || + description.length > MAX_SKILL_DESCRIPTION_LENGTH + ) { + throw new Error(`Harness skill "${artifactPath}" has an invalid description`); + } + if (disableModelInvocation !== undefined && typeof disableModelInvocation !== "boolean") { + throw new Error(`Harness skill "${artifactPath}" has an invalid disable-model-invocation value`); + } + return { name, description, disableModelInvocation: disableModelInvocation === true }; +} + function validateArtifactPath(path: string): void { const segments = path.split("/"); if ( @@ -347,19 +488,35 @@ function validateManifestInventory(manifest: HarnessVersionManifest): void { } } -function validateSupportedArtifacts(manifest: HarnessVersionManifest): void { - const evolutionArtifacts = manifest.artifacts.filter((artifact) => artifact.kind === SUPPORTED_ARTIFACT_KIND); - const unsupported = manifest.artifacts.find((artifact) => artifact.kind !== SUPPORTED_ARTIFACT_KIND); +interface SupportedArtifactInventory { + readonly memory: HarnessArtifactManifest; + readonly skills: readonly HarnessArtifactManifest[]; +} + +function validateSupportedArtifacts(manifest: HarnessVersionManifest): SupportedArtifactInventory { + const memoryArtifacts = manifest.artifacts.filter((artifact) => artifact.kind === EVOLUTION_MEMORY_ARTIFACT_KIND); + const skillArtifacts = manifest.artifacts.filter((artifact) => artifact.kind === SKILL_ARTIFACT_KIND); + const unsupported = manifest.artifacts.find( + (artifact) => artifact.kind !== EVOLUTION_MEMORY_ARTIFACT_KIND && artifact.kind !== SKILL_ARTIFACT_KIND, + ); if (unsupported !== undefined) { throw new Error(`Unsupported harness artifact kind "${unsupported.kind}"`); } if ( - evolutionArtifacts.length !== 1 || - evolutionArtifacts[0]?.path !== EVOLUTION_MEMORY_NAME || - manifest.artifacts.length !== 1 + memoryArtifacts.length !== 1 || + memoryArtifacts[0]?.path !== EVOLUTION_MEMORY_NAME || + memoryArtifacts[0]?.workspace !== undefined ) { - throw new Error(`Harness version must declare exactly one ${SUPPORTED_ARTIFACT_KIND} artifact at EVOLVE.md`); + throw new Error( + `Harness version must declare exactly one ${EVOLUTION_MEMORY_ARTIFACT_KIND} artifact at EVOLVE.md`, + ); + } + for (const artifact of skillArtifacts) { + if (artifact.workspace === undefined || !/^skills\/[a-z0-9]+(?:-[a-z0-9]+)*\/SKILL\.md$/.test(artifact.path)) { + throw new Error(`Harness skill artifact "${artifact.path}" must have a canonical path and workspace identity`); + } } + return { memory: memoryArtifacts[0] as HarnessArtifactManifest, skills: skillArtifacts }; } export async function loadHarnessState(bundlePath: string): Promise { @@ -399,6 +556,7 @@ export async function loadHarnessState(bundlePath: string): Promise(); for (const artifact of manifest.artifacts) { const artifactPath = join(bundlePath, ...artifact.path.split("/")); const artifactStats = await lstat(artifactPath); @@ -409,13 +567,46 @@ export async function loadHarnessState(bundlePath: string): Promise(); + for (const artifact of supported.skills) { + const artifactPath = join(bundlePath, ...artifact.path.split("/")); + const bytes = artifactBytes.get(artifact.path); + if (bytes === undefined) throw new Error(`Harness skill is missing at "${artifactPath}"`); + const content = decodeUtf8(bytes, artifactPath); + const frontmatter = parseSkillFrontmatter(content, artifact.path); + if (artifact.path !== `skills/${frontmatter.name}/SKILL.md`) { + throw new Error(`Harness skill "${artifact.path}" name does not match its canonical path`); + } + if (skillNames.has(frontmatter.name)) { + throw new Error(`Harness skill name collision for "${frontmatter.name}"`); + } + skillNames.add(frontmatter.name); + skills.push({ + path: artifact.path, + filePath: artifactPath, + baseDir: dirname(artifactPath), + name: frontmatter.name, + description: frontmatter.description, + content, + contentHash: artifact.contentHash, + workspace: artifact.workspace as HarnessWorkspaceIdentity, + disableModelInvocation: frontmatter.disableModelInvocation, + }); + } + if (!Check(EvolutionMemorySchema, memory)) { + throw new Error(`Invalid harness evolution memory at "${memoryPath}"`); + } + const state = { memory, skills }; + if (!Check(HarnessStateSchema, state)) throw new Error(`Invalid harness state at "${bundlePath}"`); return { state, manifest }; } @@ -428,7 +619,7 @@ async function assertDirectoryParent(destinationPath: string): Promise { } function bundlesMatch(left: HarnessVersionBundle, right: HarnessVersionBundle): boolean { - return left.manifest.version === right.manifest.version && left.state.memory === right.state.memory; + return left.manifest.version === right.manifest.version; } async function loadIdempotentDestination( @@ -448,45 +639,99 @@ async function inspectIncompatibleBundle(bundlePath: string): Promise<{ }> { const manifestPath = join(bundlePath, MANIFEST_NAME); const manifest = parseManifest(await readFile(manifestPath), manifestPath); + validateManifestInventory(manifest); const evolutionArtifacts = manifest.artifacts.filter( - (artifact) => artifact.kind === SUPPORTED_ARTIFACT_KIND && artifact.path === EVOLUTION_MEMORY_NAME, + (artifact) => artifact.kind === EVOLUTION_MEMORY_ARTIFACT_KIND && artifact.path === EVOLUTION_MEMORY_NAME, ); if (evolutionArtifacts.length !== 1) return { manifest, state: null }; const memoryPath = join(bundlePath, EVOLUTION_MEMORY_NAME); const memoryBytes = await readFile(memoryPath); if (computeContentHash(memoryBytes) !== evolutionArtifacts[0]?.contentHash) return { manifest, state: null }; - const state = { memory: decodeUtf8(memoryBytes, memoryPath) }; + const state = { memory: decodeUtf8(memoryBytes, memoryPath), skills: [] }; return { manifest, state: Check(HarnessStateSchema, state) ? state : null }; } +interface PreparedHarnessState { + readonly state: HarnessState; + readonly artifacts: readonly HarnessArtifactManifest[]; + readonly files: ReadonlyMap; +} + +function prepareHarnessState(input: HarnessStateInput, bundlePath: string): PreparedHarnessState { + const memoryBytes = UTF8_ENCODER.encode(input.memory); + const memoryArtifact: HarnessArtifactManifest = { + kind: EVOLUTION_MEMORY_ARTIFACT_KIND, + path: EVOLUTION_MEMORY_NAME, + contentHash: computeContentHash(memoryBytes), + }; + const files = new Map([[EVOLUTION_MEMORY_NAME, memoryBytes]]); + const skills: HarnessSkill[] = []; + const skillArtifacts: HarnessArtifactManifest[] = []; + const skillNames = new Set(); + for (const inputSkill of [...(input.skills ?? [])].sort((left, right) => compareText(left.path, right.path))) { + validateArtifactPath(inputSkill.path); + const contentBytes = UTF8_ENCODER.encode(inputSkill.content); + const frontmatter = parseSkillFrontmatter(inputSkill.content, inputSkill.path); + if (inputSkill.path !== `skills/${frontmatter.name}/SKILL.md`) { + throw new Error(`Harness skill "${inputSkill.path}" name does not match its canonical path`); + } + if (skillNames.has(frontmatter.name)) { + throw new Error(`Harness skill name collision for "${frontmatter.name}"`); + } + skillNames.add(frontmatter.name); + const filePath = join(bundlePath, ...inputSkill.path.split("/")); + const contentHash = computeContentHash(contentBytes); + skills.push({ + path: inputSkill.path, + filePath, + baseDir: dirname(filePath), + name: frontmatter.name, + description: frontmatter.description, + content: inputSkill.content, + contentHash, + workspace: inputSkill.workspace, + disableModelInvocation: frontmatter.disableModelInvocation, + }); + skillArtifacts.push({ + kind: SKILL_ARTIFACT_KIND, + path: inputSkill.path, + contentHash, + workspace: inputSkill.workspace, + }); + files.set(inputSkill.path, contentBytes); + } + const artifacts = [memoryArtifact, ...skillArtifacts].sort(compareArtifacts); + return { state: { memory: input.memory, skills }, artifacts, files }; +} + export async function saveHarnessState( state: unknown, destinationPath: string, base: HarnessVersion | null, metadata: HarnessVersionMetadata, ): Promise { - if (!Check(HarnessStateSchema, state)) throw new Error("Invalid harness state to save"); + if (!Check(HarnessStateInputSchema, state)) throw new Error("Invalid harness state to save"); if (!Check(Type.Union([Type.Null(), HarnessVersionSchema]), base)) throw new Error("Invalid harness base version"); if (!Check(HarnessVersionMetadataSchema, metadata)) { throw new Error("Invalid harness version metadata"); } - const memoryBytes = UTF8_ENCODER.encode(state.memory); - const artifact: HarnessArtifactManifest = { - kind: SUPPORTED_ARTIFACT_KIND, - path: EVOLUTION_MEMORY_NAME, - contentHash: computeContentHash(memoryBytes), + const absoluteDestinationPath = resolve(destinationPath); + const prepared = prepareHarnessState(state as HarnessStateInput, absoluteDestinationPath); + const versionInput = { + schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, + base, + metadata, + artifacts: prepared.artifacts, }; - const versionInput = { schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, base, metadata, artifacts: [artifact] }; const version = computeHarnessVersion(versionInput); const manifest: HarnessVersionManifest = { schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, version, base, metadata, - artifacts: [artifact], + artifacts: prepared.artifacts, }; - const bundle: HarnessVersionBundle = { state, manifest }; - const absoluteDestinationPath = resolve(destinationPath); + const bundle: HarnessVersionBundle = { state: prepared.state, manifest }; await assertDirectoryParent(absoluteDestinationPath); if (await pathExists(absoluteDestinationPath)) return loadIdempotentDestination(absoluteDestinationPath, bundle); @@ -499,7 +744,11 @@ export async function saveHarnessState( encoding: "utf8", flag: "wx", }); - await writeFile(join(temporaryPath, EVOLUTION_MEMORY_NAME), memoryBytes, { flag: "wx" }); + for (const [path, bytes] of prepared.files) { + const filePath = join(temporaryPath, ...path.split("/")); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, bytes, { flag: "wx" }); + } try { await rename(temporaryPath, absoluteDestinationPath); temporaryPath = undefined; @@ -722,29 +971,36 @@ export class HarnessStateStore { } async saveVersion( - state: HarnessState, + state: HarnessStateInput, base: HarnessVersion | null, metadata?: HarnessVersionMetadata, ): Promise { await this.#ensureDirectories(); - if (!Check(HarnessStateSchema, state)) throw new Error("Invalid harness state to save"); + if (!Check(HarnessStateInputSchema, state)) throw new Error("Invalid harness state to save"); if (!Check(Type.Union([Type.Null(), HarnessVersionSchema]), base)) throw new Error("Invalid harness base version"); - const artifact: HarnessArtifactManifest = { - kind: SUPPORTED_ARTIFACT_KIND, - path: EVOLUTION_MEMORY_NAME, - contentHash: computeContentHash(UTF8_ENCODER.encode(state.memory)), - }; - const baseMetadata = base === null ? undefined : (await this.#loadVersion(base)).manifest.metadata; - const inheritedMetadata = metadata ?? baseMetadata; + const baseBundle = base === null ? undefined : await this.#loadVersion(base); + const inheritedMetadata = metadata ?? baseBundle?.manifest.metadata; if (inheritedMetadata === undefined) throw new Error("Root harness versions require metadata"); + const resolvedState: HarnessStateInput = + state.skills !== undefined || baseBundle === undefined + ? state + : { + memory: state.memory, + skills: baseBundle.state.skills.map((skill) => ({ + path: skill.path, + content: skill.content, + workspace: skill.workspace, + })), + }; + const prepared = prepareHarnessState(resolvedState, this.#versionsPath); const version = computeHarnessVersion({ schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, base, metadata: inheritedMetadata, - artifacts: [artifact], + artifacts: prepared.artifacts, }); - return saveHarnessState(state, this.#versionPath(version), base, inheritedMetadata); + return saveHarnessState(resolvedState, this.#versionPath(version), base, inheritedMetadata); } async selectVersion(version: HarnessVersion): Promise { @@ -780,13 +1036,13 @@ export class HarnessStateStore { } async advance( - state: HarnessState, + state: HarnessStateInput, expectedBase: HarnessVersion, metadata?: HarnessVersionMetadata, ): Promise { return this.#withLock(async () => { await this.#ensureDirectories(); - if (!Check(HarnessStateSchema, state)) throw new Error("Invalid harness state to advance"); + if (!Check(HarnessStateInputSchema, state)) throw new Error("Invalid harness state to advance"); if (!Check(HarnessVersionSchema, expectedBase)) throw new Error("Invalid expected harness base version"); const current = await this.#loadPointer(); if (current.version !== expectedBase) { diff --git a/packages/harness-state/test/harness-state.test.ts b/packages/harness-state/test/harness-state.test.ts index ca214ae96..065e4e536 100644 --- a/packages/harness-state/test/harness-state.test.ts +++ b/packages/harness-state/test/harness-state.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { lstat, mkdir, mkdtemp, readdir, readFile, rename, rm, symlink, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { Check } from "typebox/value"; import { afterEach, describe, expect, test } from "vitest"; import { @@ -9,7 +9,11 @@ import { HARNESS_VERSION_SCHEMA_VERSION, type HarnessArtifactManifest, HarnessArtifactManifestSchema, + type HarnessSkill, + HarnessSkillInputSchema, + HarnessSkillSchema, type HarnessState, + HarnessStateInputSchema, HarnessStateSchema, HarnessStateStore, type HarnessVersion, @@ -20,6 +24,8 @@ import { type HarnessVersionMetadata, HarnessVersionMetadataSchema, HarnessVersionPointerSchema, + type HarnessWorkspaceIdentity, + HarnessWorkspaceIdentitySchema, harnessVersionDirectoryName, INITIAL_HARNESS_VERSION_METADATA, loadHarnessState, @@ -75,10 +81,21 @@ async function writeRawBundle( await mkdir(bundlePath, { recursive: true }); await writeFile(join(bundlePath, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); for (const [path, content] of Object.entries(files)) { - await writeFile(join(bundlePath, path), content); + const filePath = join(bundlePath, path); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, content); } } +const WORKSPACE: HarnessWorkspaceIdentity = { + repositoryId: "global", + directoryId: `sha256:${"1".repeat(64)}`, +}; + +function skillContent(name: string, description = `Use ${name} for focused work.`): string { + return `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`; +} + async function pathExists(path: string): Promise { try { await lstat(path); @@ -91,12 +108,15 @@ async function pathExists(path: string): Promise { function assertReadonlyTypes( state: HarnessState, + skill: HarnessSkill, artifact: HarnessArtifactManifest, manifest: HarnessVersionManifest, bundle: HarnessVersionBundle, ): void { // @ts-expect-error HarnessState fields are readonly. state.memory = "changed"; + // @ts-expect-error HarnessSkill fields are readonly. + skill.name = "changed"; // @ts-expect-error HarnessArtifactManifest fields are readonly. artifact.path = "changed"; // @ts-expect-error HarnessVersionManifest fields are readonly. @@ -171,6 +191,14 @@ describe("canonical harness versions", () => { artifacts: [{ ...original, path: "nested/EVOLVE.md" }], }), ).not.toBe(version); + expect( + computeHarnessVersion({ + schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, + base: null, + metadata: INITIAL_HARNESS_VERSION_METADATA, + artifacts: [{ ...original, workspace: WORKSPACE }], + }), + ).not.toBe(version); expect( computeHarnessVersion({ schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, @@ -184,15 +212,25 @@ describe("canonical harness versions", () => { test("exports strict schemas", () => { const artifact = createArtifact(""); const manifest = createManifest([artifact]); - const bundle = { state: { memory: "" }, manifest }; - - expect(Check(HarnessStateSchema, { memory: "" })).toBe(true); - expect(Check(HarnessStateSchema, { memory: " \t\n" })).toBe(false); - expect(Check(HarnessStateSchema, { memory: "\uFEFFmemory" })).toBe(false); - expect(Check(HarnessStateSchema, { memory: "\uD800" })).toBe(false); - expect(Check(HarnessStateSchema, { memory: "\uDC00" })).toBe(false); - expect(Check(HarnessStateSchema, { memory: "\uD800\uDC00" })).toBe(true); - expect(Check(HarnessStateSchema, { memory: "memory", extra: true })).toBe(false); + const bundle = { state: { memory: "", skills: [] }, manifest }; + + expect(Check(HarnessStateInputSchema, { memory: "" })).toBe(true); + expect(Check(HarnessStateSchema, { memory: "", skills: [] })).toBe(true); + expect(Check(HarnessStateSchema, { memory: " \t\n", skills: [] })).toBe(false); + expect(Check(HarnessStateSchema, { memory: "\uFEFFmemory", skills: [] })).toBe(false); + expect(Check(HarnessStateSchema, { memory: "\uD800", skills: [] })).toBe(false); + expect(Check(HarnessStateSchema, { memory: "\uDC00", skills: [] })).toBe(false); + expect(Check(HarnessStateSchema, { memory: "\uD800\uDC00", skills: [] })).toBe(true); + expect(Check(HarnessStateSchema, { memory: "memory", skills: [], extra: true })).toBe(false); + expect(Check(HarnessWorkspaceIdentitySchema, WORKSPACE)).toBe(true); + expect( + Check(HarnessSkillInputSchema, { + path: "skills/example/SKILL.md", + content: skillContent("example"), + workspace: WORKSPACE, + }), + ).toBe(true); + expect(Check(HarnessSkillSchema, { path: "skills/example/SKILL.md" })).toBe(false); expect(Check(HarnessArtifactManifestSchema, artifact)).toBe(true); expect(Check(HarnessArtifactManifestSchema, { ...artifact, extra: true })).toBe(false); expect(Check(HarnessVersionMetadataSchema, LESSON_METADATA)).toBe(true); @@ -226,6 +264,51 @@ describe("bundle persistence and validation", () => { expect(await loadHarnessState(destination)).toEqual(saved); }); + test("persists validated workspace-scoped skills in the atomic revision", async () => { + const root = await createTemporaryRoot(); + const destination = join(root, "bundle"); + const content = skillContent("review-workflow"); + const saved = await saveHarnessState( + { + memory: "Use the selected harness.\n", + skills: [ + { + path: "skills/review-workflow/SKILL.md", + content, + workspace: WORKSPACE, + }, + ], + }, + destination, + EXTERNAL_VERSION, + INITIAL_HARNESS_VERSION_METADATA, + ); + + expect(saved.state.skills).toEqual([ + expect.objectContaining({ + path: "skills/review-workflow/SKILL.md", + name: "review-workflow", + description: "Use review-workflow for focused work.", + content, + contentHash: hash(content), + workspace: WORKSPACE, + disableModelInvocation: false, + }), + ]); + expect(saved.state.skills[0]?.filePath).toBe(join(destination, "skills", "review-workflow", "SKILL.md")); + expect(saved.manifest.artifacts).toEqual([ + createArtifact("Use the selected harness.\n"), + { + kind: "skill", + path: "skills/review-workflow/SKILL.md", + contentHash: hash(content), + workspace: WORKSPACE, + }, + ]); + expect(await readFile(join(destination, "skills", "review-workflow", "SKILL.md"), "utf8")).toBe(content); + expect(await loadHarnessState(destination)).toEqual(saved); + }); + test("is idempotent only for the same immutable revision", async () => { const root = await createTemporaryRoot(); const destination = join(root, "bundle"); @@ -364,6 +447,54 @@ describe("bundle persistence and validation", () => { await expect(loadHarnessState(destination)).rejects.toThrow(/case-insensitive filesystem/i); }); + test.each([ + ["noncanonical path", "skills/example/README.md", skillContent("example"), /canonical path/i], + ["path and name mismatch", "skills/example/SKILL.md", skillContent("different"), /name.*canonical path/i], + ["missing frontmatter", "skills/example/SKILL.md", "# Example\n", /YAML frontmatter/i], + ["missing description", "skills/example/SKILL.md", "---\nname: example\n---\n", /description/i], + ] as const)("rejects a skill with %s", async (_label, path, content, expected) => { + const root = await createTemporaryRoot(); + await expect( + saveHarnessState( + { memory: "", skills: [{ path, content, workspace: WORKSPACE }] }, + join(root, "bundle"), + null, + INITIAL_HARNESS_VERSION_METADATA, + ), + ).rejects.toThrow(expected); + }); + + test("rejects skill artifacts without workspace metadata", async () => { + const root = await createTemporaryRoot(); + const destination = join(root, "bundle"); + const content = skillContent("example"); + const artifacts = [createArtifact(""), createArtifact(content, "skill", "skills/example/SKILL.md")]; + await writeRawBundle(destination, createManifest(artifacts), { + "EVOLVE.md": "", + "skills/example/SKILL.md": content, + }); + + await expect(loadHarnessState(destination)).rejects.toThrow(/workspace identity/i); + }); + + test.each([ + ["malformed UTF-8", new Uint8Array([0xc3, 0x28]), /UTF-8/i], + ["BOM", new Uint8Array([0xef, 0xbb, 0xbf, 0x61]), /BOM/i], + ] as const)("rejects %s skill bytes", async (_label, bytes, expected) => { + const root = await createTemporaryRoot(); + const destination = join(root, "bundle"); + const skillArtifact = { + ...createArtifact(bytes, "skill", "skills/example/SKILL.md"), + workspace: WORKSPACE, + }; + await writeRawBundle(destination, createManifest([createArtifact(""), skillArtifact]), { + "EVOLVE.md": "", + "skills/example/SKILL.md": bytes, + }); + + await expect(loadHarnessState(destination)).rejects.toThrow(expected); + }); + test("rejects duplicate paths and unsupported artifact kinds", async () => { const root = await createTemporaryRoot(); const duplicatePath = join(root, "duplicate"); @@ -493,6 +624,37 @@ describe("HarnessStateStore", () => { expect((await store.loadCurrent()).manifest.version).toBe(second.bundle.manifest.version); }); + test("inherits evolved skills across memory-only revisions and clears them only explicitly", async () => { + const agentDir = await createTemporaryRoot(); + const store = new HarnessStateStore(agentDir); + const root = await store.initialize(); + const content = skillContent("inherited-skill"); + const withSkill = await store.advance( + { + memory: "First memory.", + skills: [ + { + path: "skills/inherited-skill/SKILL.md", + content, + workspace: WORKSPACE, + }, + ], + }, + root.manifest.version, + ); + if (withSkill.status !== "advanced") throw new Error("Expected skill advance to succeed"); + + const inherited = await store.advance({ memory: "Second memory." }, withSkill.bundle.manifest.version); + if (inherited.status !== "advanced") throw new Error("Expected inherited advance to succeed"); + expect(inherited.bundle.state.skills).toHaveLength(1); + expect(inherited.bundle.state.skills[0]?.content).toBe(content); + expect(inherited.bundle.manifest.artifacts.some((artifact) => artifact.kind === "skill")).toBe(true); + + const cleared = await store.advance({ memory: "Third memory.", skills: [] }, inherited.bundle.manifest.version); + if (cleared.status !== "advanced") throw new Error("Expected cleared advance to succeed"); + expect(cleared.bundle.state.skills).toEqual([]); + }); + test("inherits durable harness metadata across ordinary advances", async () => { const agentDir = await createTemporaryRoot(); const store = new HarnessStateStore(agentDir); @@ -689,7 +851,7 @@ describe("HarnessStateStore", () => { const diagnostic = (await store.listVersions()).diagnostics.find((entry) => entry.version === manifest.version); expect(diagnostic).toMatchObject({ manifest, - state: { memory: "preview memory" }, + state: { memory: "preview memory", skills: [] }, }); expect(diagnostic?.message).toMatch(/unsupported harness artifact kind/i); }); From 12e6ca6be64ba7092dbdcdf02edc8676d976a66a Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 11:45:26 -0400 Subject: [PATCH 02/10] fix(coding-agent): refresh evolved harness footer metadata --- packages/coding-agent/CHANGELOG.md | 1 + .../components/evolution-footer-status.ts | 11 ++-- .../test/evolution-footer-status.test.ts | 61 +++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index b63185600..dbf4e013d 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -15,6 +15,7 @@ ### Fixed +- Fixed the footer showing an unknown harness after self-evolution switched to the new version. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)). diff --git a/packages/coding-agent/src/modes/interactive/components/evolution-footer-status.ts b/packages/coding-agent/src/modes/interactive/components/evolution-footer-status.ts index 63e8cd0ab..9f229a2d2 100644 --- a/packages/coding-agent/src/modes/interactive/components/evolution-footer-status.ts +++ b/packages/coding-agent/src/modes/interactive/components/evolution-footer-status.ts @@ -44,7 +44,8 @@ export class EvolutionFooterStatus { #selectedHarnessVersion: string | null = null; #selectedHarnessVersionLoaded = false; #activateSelectedHarness = false; - #reloadVersionInfo = true; + #versionInfoGeneration = 0; + #loadedVersionInfoGeneration = -1; #refreshPromise: Promise | null = null; #refreshAgain = false; #pollTimeout: ReturnType | null = null; @@ -100,7 +101,9 @@ export class EvolutionFooterStatus { async #refresh(): Promise { try { - const shouldLoadVersionInfo = this.#reloadVersionInfo || !this.#versionInfoLoaded; + const versionInfoGeneration = this.#versionInfoGeneration; + const shouldLoadVersionInfo = + versionInfoGeneration !== this.#loadedVersionInfoGeneration || !this.#versionInfoLoaded; const [overview, versionInfo] = await Promise.all([ this.#client.overview(), shouldLoadVersionInfo ? this.#client.listHarnessVersionInfo() : Promise.resolve(this.#versionInfo), @@ -109,7 +112,7 @@ export class EvolutionFooterStatus { this.#versionInfo = versionInfo; this.#versionInfoLoaded = true; - this.#reloadVersionInfo = false; + if (shouldLoadVersionInfo) this.#loadedVersionInfoGeneration = versionInfoGeneration; this.#setRunning(overview.activeJob !== null); const selectedVersion = overview.currentHarnessVersion; @@ -139,7 +142,7 @@ export class EvolutionFooterStatus { if (event.type === "completed" || event.type === "failed") this.#setRunning(false); if (event.type === "completed") { this.#activateSelectedHarness = true; - this.#reloadVersionInfo = true; + this.#versionInfoGeneration += 1; } this.#requestRender(); void this.refresh(); diff --git a/packages/coding-agent/test/evolution-footer-status.test.ts b/packages/coding-agent/test/evolution-footer-status.test.ts index 0302394ac..27cf87f6d 100644 --- a/packages/coding-agent/test/evolution-footer-status.test.ts +++ b/packages/coding-agent/test/evolution-footer-status.test.ts @@ -180,4 +180,65 @@ describe("EvolutionFooterStatus", () => { expect(status.getSnapshot().activeVersionInfo?.version).toBe(evolvedVersion); status.dispose(); }); + + it("reloads harness metadata when completion races with an in-flight refresh", async () => { + let listener: ((event: EvolutionProgressEvent) => void) | undefined; + let resolveOverview: ((value: EvolutionOverview) => void) | undefined; + let activeVersion: HarnessVersion = rootVersion; + const evolvedOverview = overview(evolvedVersion); + const evolvedInfo: EvolutionHarnessVersionInfo = { + version: evolvedVersion, + kind: "evolved", + evolvedAt: "2026-08-20T12:48:00.000Z", + evolvedIn: "/projects/diffex", + }; + const overviewClient = vi + .fn<() => Promise>() + .mockResolvedValueOnce(overview(rootVersion)) + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveOverview = resolve; + }), + ) + .mockResolvedValue(evolvedOverview); + const listHarnessVersionInfo = vi + .fn<() => Promise>() + .mockResolvedValueOnce([{ version: rootVersion, kind: "default", evolvedAt: null, evolvedIn: null }]) + .mockResolvedValue([evolvedInfo]); + const selected = vi.fn(async (version: string) => { + activeVersion = version as HarnessVersion; + }); + const status = new EvolutionFooterStatus({ + client: { + subscribe: (next) => { + listener = next; + return () => {}; + }, + overview: overviewClient, + listHarnessVersionInfo, + }, + getActiveHarnessVersion: () => activeVersion, + requestRender: () => {}, + onSelectedHarnessChange: selected, + }); + await vi.waitFor(() => expect(status.getSnapshot().loaded).toBe(true)); + + const staleRefresh = status.refresh(); + await vi.waitFor(() => expect(overviewClient).toHaveBeenCalledTimes(2)); + listener?.({ + jobId: "job-active", + emittedAt: "2026-08-20T12:48:00.000Z", + type: "completed", + stage: "completed", + }); + if (resolveOverview === undefined) throw new Error("Expected an in-flight overview request"); + resolveOverview(evolvedOverview); + await staleRefresh; + + await vi.waitFor(() => expect(status.getSnapshot().activeVersionInfo).toEqual(evolvedInfo)); + expect(selected).toHaveBeenCalledWith(evolvedVersion); + expect(listHarnessVersionInfo).toHaveBeenCalledTimes(2); + status.dispose(); + }); }); From 687bc8be025ae0bd02a9b660b6313fcd29d52870 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 11:49:20 -0400 Subject: [PATCH 03/10] feat(coding-agent): activate versioned evolved skills --- packages/coding-agent/CHANGELOG.md | 1 + .../examples/sdk/12-full-control.ts | 1 + .../src/core/agent-session-services.ts | 2 + .../coding-agent/src/core/agent-session.ts | 62 ++++++++- .../src/core/evolution-runtime.ts | 16 +-- .../src/core/harness-state-runtime.ts | 29 ++++- .../coding-agent/src/core/resource-loader.ts | 59 ++++++++- packages/coding-agent/src/core/sdk.ts | 7 +- packages/coding-agent/src/core/skills.ts | 52 ++++++++ .../test/evolved-skill-catalog.test.ts | 122 ++++++++++++++++++ .../test/harness-state-runtime.test.ts | 39 +++++- .../test/sdk-codex-cache-probe-tool-loop.ts | 1 + packages/coding-agent/test/sdk-skills.test.ts | 2 + .../agent-session-harness-version.test.ts | 111 +++++++++++++++- packages/coding-agent/test/suite/harness.ts | 3 + .../7193-event-bus-lifecycle.test.ts | 1 + .../test/suite/subagent-services.test.ts | 61 +++++++++ packages/coding-agent/test/utilities.ts | 7 +- packages/harness-state/src/index.ts | 8 +- 19 files changed, 558 insertions(+), 26 deletions(-) create mode 100644 packages/coding-agent/test/evolved-skill-catalog.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index dbf4e013d..cd5a62f87 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Added atomic harness revisions for workspace-scoped evolved skills. - Added `pi auth check` provider/model auth preflight with optional credential output ([#7152](https://github.com/earendil-works/pi/issues/7152)). - Added durable, branch-scoped sub-agent rosters and child histories for saved sessions, with interrupted restoration and explicit wake without automatic work, replay, or result delivery. - Added bounded child-to-parent escalation reports for root-controlled sub-agents, with hidden delivery that can resume coordinated work without consuming terminal results. diff --git a/packages/coding-agent/examples/sdk/12-full-control.ts b/packages/coding-agent/examples/sdk/12-full-control.ts index 3493b46c0..5b1d4875f 100644 --- a/packages/coding-agent/examples/sdk/12-full-control.ts +++ b/packages/coding-agent/examples/sdk/12-full-control.ts @@ -36,6 +36,7 @@ const cwd = process.cwd(); const resourceLoader: ResourceLoader = { getExtensions: () => ({ extensions: [], errors: [], runtime: createExtensionRuntime() }), getSkills: () => ({ skills: [], diagnostics: [] }), + setEvolvedSkills: () => {}, getPrompts: () => ({ prompts: [], diagnostics: [] }), getThemes: () => ({ themes: [], diagnostics: [] }), getAgentsFiles: () => ({ agentsFiles: [] }), diff --git a/packages/coding-agent/src/core/agent-session-services.ts b/packages/coding-agent/src/core/agent-session-services.ts index b4dde5850..740cd5278 100644 --- a/packages/coding-agent/src/core/agent-session-services.ts +++ b/packages/coding-agent/src/core/agent-session-services.ts @@ -71,6 +71,7 @@ export interface CreateAgentSessionFromServicesOptions { customTools?: ToolDefinition[]; subagents?: CreateAgentSessionOptions["subagents"]; lifecycleObserver?: AgentSessionLifecycleObserver; + pinHarnessVersion?: boolean; } /** @@ -246,6 +247,7 @@ export async function createInternalAgentSessionFromServices( resourceLoader: options.services.resourceLoader, harnessStateStore: options.services.harnessStateStore, harnessVersionBundle: options.services.harnessVersionBundle, + pinHarnessVersion: options.pinHarnessVersion, telemetryContext: options.services.telemetryContext, sessionManager: options.sessionManager, model: options.model, diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index eeb625ed3..3214e5c09 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -54,6 +54,7 @@ import type { HarnessVersion, HarnessVersionBundle, HarnessVersionList, + HarnessWorkspaceIdentity, } from "@earendil-works/diffex-harness-state"; import { getThemeByName, theme } from "../modes/interactive/theme/theme.ts"; import { resolvePath } from "../utils/paths.ts"; @@ -124,6 +125,7 @@ import { } from "./session-manager.ts"; import type { SettingsManager } from "./settings-manager.ts"; import { + createEvolvedSkillCatalog, createSkillReferenceSnapshot, formatSkillReferenceSnapshot, type Skill, @@ -313,6 +315,10 @@ export interface AgentSessionConfig { harnessStateStore?: HarnessStateStore; /** Harness revision activated for this session. */ harnessVersionBundle?: HarnessVersionBundle; + /** Stable identity used to scope evolved harness skills. */ + workspaceIdentity?: HarnessWorkspaceIdentity; + /** Keep the supplied harness revision fixed instead of following the global selected pointer. */ + pinHarnessVersion?: boolean; /** Parent context for passive runtime telemetry. */ telemetryContext?: TelemetryContext; /** Process-owned observer for durable lifecycle boundaries. */ @@ -515,10 +521,13 @@ export class AgentSession { private _systemPromptOverride?: string; private readonly _subagentController: SubagentController | undefined; private readonly _harnessStateStore: HarnessStateStore | undefined; + private readonly _workspaceIdentity: HarnessWorkspaceIdentity | undefined; + private readonly _pinHarnessVersion: boolean; private _harnessVersionBundle: HarnessVersionBundle | undefined; private readonly _telemetryContext: TelemetryContext; private readonly _lifecycleObserver: AgentSessionLifecycleObserver | undefined; private _harnessSwitchInProgress = false; + private _promptAdmissionInProgress = false; private _subagentCoordination: SubagentCoordination | undefined; private _pendingSubagentEscalationEnvelope: PendingSubagentEscalationEnvelope | undefined; private readonly _pendingIdleSubagentEscalations = new Map(); @@ -542,7 +551,14 @@ export class AgentSession { this._sessionStartEvent = config.sessionStartEvent ?? { type: "session_start", reason: "startup" }; this._subagentController = config.subagentController; this._harnessStateStore = config.harnessStateStore; + this._workspaceIdentity = config.workspaceIdentity; + this._pinHarnessVersion = config.pinHarnessVersion ?? false; this._harnessVersionBundle = config.harnessVersionBundle; + this._resourceLoader.setEvolvedSkills( + config.harnessVersionBundle === undefined || config.workspaceIdentity === undefined + ? { skills: [], diagnostics: [] } + : createEvolvedSkillCatalog(config.harnessVersionBundle, config.workspaceIdentity), + ); this._telemetryContext = config.telemetryContext ?? NOOP_TELEMETRY_CONTEXT; this._lifecycleObserver = config.lifecycleObserver; @@ -1307,6 +1323,9 @@ export class AgentSession { if (this._harnessSwitchInProgress) { throw new Error("Cannot switch harness versions while another switch is in progress."); } + if (this._promptAdmissionInProgress) { + throw new Error("Cannot switch harness versions while a prompt is being admitted."); + } if (this.isStreaming) { throw new Error("Cannot switch harness versions while a response is streaming."); } @@ -1342,6 +1361,11 @@ export class AgentSession { ): Promise { await this._subagentController?.reset(); const activatedBundle = selectBundle ? await selectBundle() : bundle; + this._resourceLoader.setEvolvedSkills( + this._workspaceIdentity === undefined + ? { skills: [], diagnostics: [] } + : createEvolvedSkillCatalog(activatedBundle, this._workspaceIdentity), + ); this._harnessVersionBundle = activatedBundle; this._baseSystemPrompt = this._rebuildSystemPrompt(this.getActiveToolNames()); this._systemPromptOverride = undefined; @@ -1350,7 +1374,9 @@ export class AgentSession { } private async _refreshSelectedHarnessAtPromptBoundary(): Promise { - if (!this._harnessStateStore || hasFixedHarnessSubagentSession(this.sessionManager)) return; + if (!this._harnessStateStore || this._pinHarnessVersion || hasFixedHarnessSubagentSession(this.sessionManager)) { + return; + } const selected = await this._harnessStateStore.loadCurrent(); if (selected.manifest.version === this.activeHarnessVersion) return; @@ -1924,6 +1950,26 @@ export class AgentSession { options: PromptOptions | undefined, releaseUnsignaledPreflight: () => void, operationSpan: CodingAgentTelemetrySpan<"diffex.coding_agent.operation">, + ): Promise { + const reservesAdmission = !this.isStreaming; + if (reservesAdmission) { + if (this._promptAdmissionInProgress) { + throw new Error("Cannot submit a prompt while another prompt is being admitted."); + } + this._promptAdmissionInProgress = true; + } + try { + await this._promptAfterReservation(text, options, releaseUnsignaledPreflight, operationSpan); + } finally { + if (reservesAdmission) this._promptAdmissionInProgress = false; + } + } + + private async _promptAfterReservation( + text: string, + options: PromptOptions | undefined, + releaseUnsignaledPreflight: () => void, + operationSpan: CodingAgentTelemetrySpan<"diffex.coding_agent.operation">, ): Promise { if (this._harnessSwitchInProgress) { throw new Error("Cannot submit a prompt while a harness version switch is in progress."); @@ -1969,6 +2015,9 @@ export class AgentSession { ); } + if (!this.isStreaming) await this._refreshSelectedHarnessAtPromptBoundary(); + signal?.throwIfAborted(); + // Emit input event for extension interception (before skill/template expansion) let currentText = text; let currentImages = options?.images; @@ -2020,9 +2069,6 @@ export class AgentSession { return; } - await this._refreshSelectedHarnessAtPromptBoundary(); - signal?.throwIfAborted(); - // Validate model if (!this.model) { throw new Error(formatNoModelSelectedMessage()); @@ -2205,6 +2251,14 @@ export class AgentSession { if (!snapshot) { throw new Error(`Skill reference ${skillId}@${version} is not available in this session.`); } + if ( + snapshot.sourceKind === "evolved" && + snapshot.harnessRevision !== this._harnessVersionBundle?.manifest.version + ) { + throw new Error( + `Evolved skill reference ${skillId}@${version} is not part of the active harness revision.`, + ); + } if (!seen.has(key)) { seen.add(key); references.push(snapshot); diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 5bfe29713..7073e8298 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -1,7 +1,7 @@ import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { mkdir, readdir, readFile, realpath, rename, stat, unlink, writeFile } from "node:fs/promises"; -import { basename, join, resolve } from "node:path"; +import { mkdir, readdir, readFile, rename, stat, unlink, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; import { promisify } from "node:util"; import { type AssistantMessage, contentText, type Message, type Model, type Tool } from "@earendil-works/diffex-ai"; import type { HarnessStateStore, HarnessVersion } from "@earendil-works/diffex-harness-state"; @@ -52,6 +52,7 @@ import { VERSION } from "../config.ts"; import type { AgentSession } from "./agent-session.ts"; import type { AgentSessionLifecycleObserver, AgentSessionOperationSettlement } from "./agent-session-lifecycle.ts"; import { FileEvolutionFailureReporter } from "./evolution-failure-reporter.ts"; +import { resolveHarnessWorkspace } from "./harness-state-runtime.ts"; import { INVOCATION_TRACE_LOCK_STALE_MS, type InvocationTrace } from "./invocation-trace.ts"; import type { ModelRuntime } from "./model-runtime.ts"; import type { ResourceLoader } from "./resource-loader.ts"; @@ -76,13 +77,10 @@ export async function resolveEvolutionDirectoryIdentity(cwd: string): Promise<{ readonly directoryId: string; readonly rootPath: string; }> { - const rootPath = await realpath(cwd).catch(() => resolve(cwd)); - const directory = await stat(rootPath, { bigint: true }); - const identity = - directory.ino === 0n - ? { kind: "canonical-path-v1", rootPath } - : { kind: "filesystem-directory-v1", device: directory.dev.toString(), inode: directory.ino.toString() }; - return { directoryId: computeEvolutionContentDigest(identity), rootPath }; + const resolved = await resolveHarnessWorkspace(cwd); + const directoryId = resolved.workspace.directoryId; + if (directoryId === undefined) throw new Error("Resolved workspace is missing its directory identity"); + return { directoryId, rootPath: resolved.rootPath }; } async function gitRepositoryContainsRevision(rootPath: string, revision: string): Promise { diff --git a/packages/coding-agent/src/core/harness-state-runtime.ts b/packages/coding-agent/src/core/harness-state-runtime.ts index d9875223a..831d7c232 100644 --- a/packages/coding-agent/src/core/harness-state-runtime.ts +++ b/packages/coding-agent/src/core/harness-state-runtime.ts @@ -1,4 +1,31 @@ -import { HarnessStateStore, type HarnessVersionBundle } from "@earendil-works/diffex-harness-state"; +import { createHash } from "node:crypto"; +import { realpath, stat } from "node:fs/promises"; +import { resolve } from "node:path"; +import { + HarnessStateStore, + type HarnessVersionBundle, + type HarnessWorkspaceIdentity, +} from "@earendil-works/diffex-harness-state"; + +export interface ResolvedHarnessWorkspace { + readonly workspace: HarnessWorkspaceIdentity; + readonly rootPath: string; +} + +export async function resolveHarnessWorkspace(cwd: string): Promise { + const rootPath = await realpath(cwd).catch(() => resolve(cwd)); + const directory = await stat(rootPath, { bigint: true }); + const identity = + directory.ino === 0n + ? JSON.stringify({ kind: "canonical-path-v1", rootPath }) + : JSON.stringify({ + device: directory.dev.toString(), + inode: directory.ino.toString(), + kind: "filesystem-directory-v1", + }); + const directoryId = `sha256:${createHash("sha256").update(identity).digest("hex")}`; + return { workspace: { repositoryId: "global", directoryId }, rootPath }; +} export interface LoadedHarnessState { store: HarnessStateStore; diff --git a/packages/coding-agent/src/core/resource-loader.ts b/packages/coding-agent/src/core/resource-loader.ts index dc1962be2..2ff345f32 100644 --- a/packages/coding-agent/src/core/resource-loader.ts +++ b/packages/coding-agent/src/core/resource-loader.ts @@ -38,6 +38,7 @@ export interface ResourceLoaderReloadOptions { export interface ResourceLoader { getExtensions(): LoadExtensionsResult; getSkills(): SkillCatalog; + setEvolvedSkills(catalog: SkillCatalog): void; getPrompts(): { prompts: PromptTemplate[]; diagnostics: ResourceDiagnostic[] }; getThemes(): { themes: Theme[]; diagnostics: ResourceDiagnostic[] }; getAgentsFiles(): { agentsFiles: Array<{ path: string; content: string }> }; @@ -225,6 +226,8 @@ export class DefaultResourceLoader implements ResourceLoader { private appendSystemPromptOverride?: (base: string[]) => string[]; private extensionsResult: LoadExtensionsResult; + private installedSkillCatalog: SkillCatalog; + private evolvedSkillCatalog: SkillCatalog; private skillCatalog: SkillCatalog; private disabledSkillDiagnostics: ResourceDiagnostic[]; private prompts: PromptTemplate[]; @@ -277,6 +280,8 @@ export class DefaultResourceLoader implements ResourceLoader { this.appendSystemPromptOverride = options.appendSystemPromptOverride; this.extensionsResult = { extensions: [], errors: [], runtime: createExtensionRuntime() }; + this.installedSkillCatalog = { skills: [], diagnostics: [] }; + this.evolvedSkillCatalog = { skills: [], diagnostics: [] }; this.skillCatalog = { skills: [], diagnostics: [] }; this.disabledSkillDiagnostics = []; this.prompts = []; @@ -304,6 +309,14 @@ export class DefaultResourceLoader implements ResourceLoader { return this.skillCatalog; } + setEvolvedSkills(catalog: SkillCatalog): void { + this.evolvedSkillCatalog = { + skills: [...catalog.skills], + diagnostics: [...catalog.diagnostics], + }; + this.rebuildSkillCatalog(); + } + getPrompts(): { prompts: PromptTemplate[]; diagnostics: ResourceDiagnostic[] } { return { prompts: this.prompts, diagnostics: this.promptDiagnostics }; } @@ -684,6 +697,49 @@ export class DefaultResourceLoader implements ResourceLoader { }); } + private rebuildSkillCatalog(): void { + const installedNames = new Set(this.installedSkillCatalog.skills.map((skill) => skill.name)); + const exposedNames = new Set(installedNames); + const evolvedSkills: SkillCatalog["skills"] = []; + const collisionDiagnostics: ResourceDiagnostic[] = []; + for (const skill of this.evolvedSkillCatalog.skills) { + const name = installedNames.has(skill.name) ? `${skill.name}-Evolved` : skill.name; + if (name !== skill.name) { + collisionDiagnostics.push({ + type: "collision", + message: `name "${skill.name}" collision; evolved skill exposed as "${name}"`, + path: skill.filePath, + collision: { + resourceType: "skill", + name: skill.name, + winnerPath: + this.installedSkillCatalog.skills.find((installed) => installed.name === skill.name)?.filePath ?? + "", + loserPath: skill.filePath, + }, + }); + } + if (exposedNames.has(name)) { + collisionDiagnostics.push({ + type: "collision", + message: `name "${name}" collision prevents the evolved skill from being exposed`, + path: skill.filePath, + }); + continue; + } + exposedNames.add(name); + evolvedSkills.push(name === skill.name ? skill : { ...skill, name }); + } + this.skillCatalog = { + skills: [...evolvedSkills, ...this.installedSkillCatalog.skills], + diagnostics: [ + ...this.installedSkillCatalog.diagnostics, + ...this.evolvedSkillCatalog.diagnostics, + ...collisionDiagnostics, + ], + }; + } + private updateSkillsFromPaths(skillPaths: string[], metadataByPath?: Map): void { let skillsResult: SkillCatalog; if (this.noSkills && skillPaths.length === 0) { @@ -701,7 +757,7 @@ export class DefaultResourceLoader implements ResourceLoader { diagnostics: [...this.disabledSkillDiagnostics, ...skillsResult.diagnostics], }; const resolvedSkills = this.skillsOverride ? this.skillsOverride(skillsResult) : skillsResult; - this.skillCatalog = { + this.installedSkillCatalog = { skills: resolvedSkills.skills.map((skill) => { const sourceInfo = this.findSourceInfoForPath(skill.filePath, this.extensionSkillSourceInfos, metadataByPath) ?? @@ -724,6 +780,7 @@ export class DefaultResourceLoader implements ResourceLoader { }), diagnostics: resolvedSkills.diagnostics, }; + this.rebuildSkillCatalog(); } private updatePromptsFromPaths(promptPaths: string[], metadataByPath?: Map): void { diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 8b5804873..6038e953b 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -16,7 +16,7 @@ import type { AgentSessionLifecycleObserver } from "./agent-session-lifecycle.ts import { formatNoModelsAvailableMessage } from "./auth-guidance.ts"; import { DEFAULT_THINKING_LEVEL } from "./defaults.ts"; import type { ExtensionRunner, LoadExtensionsResult, SessionStartEvent, ToolDefinition } from "./extensions/index.ts"; -import { loadGlobalHarnessState } from "./harness-state-runtime.ts"; +import { loadGlobalHarnessState, resolveHarnessWorkspace } from "./harness-state-runtime.ts"; import { convertToLlm } from "./messages.ts"; import { findInitialModel } from "./model-resolver.ts"; import { ModelRuntime } from "./model-runtime.ts"; @@ -125,6 +125,8 @@ export interface CreateAgentSessionOptions { harnessStateStore?: HarnessStateStore; /** Activated harness revision from the preloaded store. */ harnessVersionBundle?: HarnessVersionBundle; + /** Keep the supplied harness revision fixed instead of following the global selection. */ + pinHarnessVersion?: boolean; /** Optional parent for passive runtime telemetry. */ telemetryContext?: TelemetryContext; /** Optional process-owned observer for durable lifecycle boundaries. */ @@ -410,6 +412,7 @@ export async function createInternalAgentSession( internalCustomTools: readonly ToolDefinition[] = [], ): Promise { const cwd = resolvePath(options.cwd ?? options.sessionManager?.getCwd() ?? process.cwd()); + const workspace = await resolveHarnessWorkspace(cwd); const agentDir = options.agentDir ? resolvePath(options.agentDir) : getDefaultAgentDir(); const loadedHarnessState = options.harnessStateStore ? { store: options.harnessStateStore, bundle: options.harnessVersionBundle, warning: undefined } @@ -654,6 +657,8 @@ export async function createInternalAgentSession( subagentController, harnessStateStore: loadedHarnessState.store, harnessVersionBundle: loadedHarnessState.bundle, + workspaceIdentity: workspace.workspace, + pinHarnessVersion: options.pinHarnessVersion, telemetryContext, lifecycleObserver: options.lifecycleObserver, }); diff --git a/packages/coding-agent/src/core/skills.ts b/packages/coding-agent/src/core/skills.ts index 5787869fc..2298a1a4d 100644 --- a/packages/coding-agent/src/core/skills.ts +++ b/packages/coding-agent/src/core/skills.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import type { HarnessVersionBundle, HarnessWorkspaceIdentity } from "@earendil-works/diffex-harness-state"; import { existsSync, readdirSync, readFileSync, statSync } from "fs"; import ignore from "ignore"; import { basename, dirname, join, relative, resolve, sep } from "path"; @@ -181,6 +182,57 @@ export function createSkill(options: CreateSkillOptions): Skill { }; } +function workspaceIdentitiesMatch(left: HarnessWorkspaceIdentity, right: HarnessWorkspaceIdentity): boolean { + return left.repositoryId === right.repositoryId && left.directoryId === right.directoryId; +} + +export function createEvolvedSkillCatalog( + bundle: HarnessVersionBundle, + workspace: HarnessWorkspaceIdentity, +): SkillCatalog { + const skills: Skill[] = []; + const diagnostics: ResourceDiagnostic[] = []; + for (const artifact of bundle.state.skills) { + if (!workspaceIdentitiesMatch(artifact.workspace, workspace)) { + diagnostics.push({ + type: "warning", + message: `evolved skill scope does not match the active workspace for harness ${bundle.manifest.version}`, + path: artifact.filePath, + }); + continue; + } + skills.push( + createSkill({ + name: artifact.name, + description: artifact.description, + filePath: artifact.filePath, + baseDir: artifact.baseDir, + sourceInfo: createSyntheticSourceInfo(artifact.filePath, { + source: `harness:${bundle.manifest.version}`, + scope: "project", + baseDir: artifact.baseDir, + }), + content: artifact.content, + sourceKind: "evolved", + scope: "workspace", + skillId: `skill:${sha256( + JSON.stringify([ + "evolved", + artifact.workspace.repositoryId, + artifact.workspace.directoryId ?? null, + artifact.path, + ]), + )}`, + contentDigest: artifact.contentHash, + version: artifact.contentHash, + harnessRevision: bundle.manifest.version, + disableModelInvocation: artifact.disableModelInvocation, + }), + ); + } + return { skills, diagnostics }; +} + export function createSkillReferenceSnapshot(skill: Skill): SkillReferenceSnapshot { const reference = ``; return { diff --git a/packages/coding-agent/test/evolved-skill-catalog.test.ts b/packages/coding-agent/test/evolved-skill-catalog.test.ts new file mode 100644 index 000000000..84edca407 --- /dev/null +++ b/packages/coding-agent/test/evolved-skill-catalog.test.ts @@ -0,0 +1,122 @@ +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { HarnessStateStore } from "@earendil-works/diffex-harness-state"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveHarnessWorkspace } from "../src/core/harness-state-runtime.ts"; +import { DefaultResourceLoader } from "../src/core/resource-loader.ts"; +import { createEvolvedSkillCatalog, createSkill } from "../src/core/skills.ts"; +import { createSyntheticSourceInfo } from "../src/core/source-info.ts"; + +const temporaryDirectories: string[] = []; + +function createTemporaryDirectory(label: string): string { + const path = join(tmpdir(), `diffex-${label}-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(path, { recursive: true }); + temporaryDirectories.push(path); + return path; +} + +afterEach(() => { + for (const path of temporaryDirectories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +function skillContent(name: string, description: string): string { + return `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`; +} + +describe("evolved skill catalogs", () => { + it("includes only artifacts for the exact active workspace", async () => { + const cwd = createTemporaryDirectory("evolved-scope"); + const agentDir = createTemporaryDirectory("evolved-agent"); + const active = await resolveHarnessWorkspace(cwd); + const store = new HarnessStateStore(agentDir); + const root = await store.initialize(); + const bundle = await store.saveVersion( + { + memory: "Scoped memory.\n", + skills: [ + { + path: "skills/scoped-review/SKILL.md", + content: skillContent("scoped-review", "Review changes in this workspace."), + workspace: active.workspace, + }, + ], + }, + root.manifest.version, + ); + + const matching = createEvolvedSkillCatalog(bundle, active.workspace); + expect(matching.diagnostics).toEqual([]); + expect(matching.skills).toEqual([ + expect.objectContaining({ + name: "scoped-review", + sourceKind: "evolved", + scope: "workspace", + version: bundle.state.skills[0]?.contentHash, + harnessRevision: bundle.manifest.version, + filePath: bundle.state.skills[0]?.filePath, + }), + ]); + + const mismatched = createEvolvedSkillCatalog(bundle, { + repositoryId: active.workspace.repositoryId, + directoryId: "different-directory", + }); + expect(mismatched.skills).toEqual([]); + expect(mismatched.diagnostics[0]).toMatchObject({ + type: "warning", + path: bundle.state.skills[0]?.filePath, + }); + expect(mismatched.diagnostics[0]?.message).toContain("does not match the active workspace"); + }); + + it("keeps the harness overlay across reloads and disambiguates installed names", async () => { + const cwd = createTemporaryDirectory("evolved-overlay"); + const agentDir = createTemporaryDirectory("evolved-overlay-agent"); + const active = await resolveHarnessWorkspace(cwd); + const store = new HarnessStateStore(agentDir); + const root = await store.initialize(); + const bundle = await store.saveVersion( + { + memory: "Overlay memory.\n", + skills: [ + { + path: "skills/shared-name/SKILL.md", + content: skillContent("shared-name", "Use the evolved workflow."), + workspace: active.workspace, + }, + ], + }, + root.manifest.version, + ); + const installedPath = join(agentDir, "installed", "shared-name", "SKILL.md"); + mkdirSync(dirname(installedPath), { recursive: true }); + writeFileSync(installedPath, skillContent("shared-name", "Use the installed workflow.")); + const installed = createSkill({ + name: "shared-name", + description: "Use the installed workflow.", + filePath: installedPath, + baseDir: dirname(installedPath), + sourceInfo: createSyntheticSourceInfo(installedPath, { source: "test-installed" }), + content: skillContent("shared-name", "Use the installed workflow."), + }); + const loader = new DefaultResourceLoader({ + cwd, + agentDir, + noExtensions: true, + noPromptTemplates: true, + noThemes: true, + noContextFiles: true, + skillsOverride: () => ({ skills: [installed], diagnostics: [] }), + }); + await loader.reload(); + loader.setEvolvedSkills(createEvolvedSkillCatalog(bundle, active.workspace)); + + expect(loader.getSkills().skills.map((skill) => skill.name)).toEqual(["shared-name-Evolved", "shared-name"]); + expect(loader.getSkills().diagnostics.some((diagnostic) => diagnostic.type === "collision")).toBe(true); + await loader.reload(); + expect(loader.getSkills().skills.map((skill) => skill.name)).toEqual(["shared-name-Evolved", "shared-name"]); + expect(loader.getSkills().skills[0]?.harnessRevision).toBe(bundle.manifest.version); + }); +}); diff --git a/packages/coding-agent/test/harness-state-runtime.test.ts b/packages/coding-agent/test/harness-state-runtime.test.ts index b4c42b050..1381852c7 100644 --- a/packages/coding-agent/test/harness-state-runtime.test.ts +++ b/packages/coding-agent/test/harness-state-runtime.test.ts @@ -106,15 +106,42 @@ describe("global harness state startup", () => { expect(readFileSync(pointerPath, "utf8")).toBe(selectedPointer); }); - it("warns and falls back when the selected revision uses an unsupported artifact", async () => { + it("loads a selected revision with evolved skill artifacts", async () => { + const agentDir = createAgentDir(); + const store = new HarnessStateStore(agentDir); + const root = await store.initialize(); + const skill = "---\nname: future\ndescription: Use the future workflow.\n---\n\n# Future workflow\n"; + const selected = await store.saveVersion( + { + memory: "Skill-compatible memory.\n", + skills: [ + { + path: "skills/future/SKILL.md", + content: skill, + workspace: { repositoryId: "global", directoryId: "directory-1" }, + }, + ], + }, + root.manifest.version, + ); + await store.selectVersion(selected.manifest.version); + + const loaded = await loadGlobalHarnessState(agentDir); + + expect(loaded.warning).toBeUndefined(); + expect(loaded.bundle).toEqual(selected); + expect(loaded.bundle?.state.skills[0]?.content).toBe(skill); + }); + + it("warns and preserves the pointer for an unsupported future artifact", async () => { const agentDir = createAgentDir(); const store = new HarnessStateStore(agentDir); const root = await store.initialize(); const memory = "Future-compatible memory.\n"; - const skill = "Future skill.\n"; + const future = "Future artifact.\n"; const artifacts = [ { kind: "evolution-memory", path: "EVOLVE.md", contentHash: contentHash(memory) }, - { kind: "skill", path: "skills/future/SKILL.md", contentHash: contentHash(skill) }, + { kind: "future-skill-asset", path: "future.txt", contentHash: contentHash(future) }, ]; const version = computeHarnessVersion({ schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, @@ -123,7 +150,7 @@ describe("global harness state startup", () => { artifacts, }); const bundlePath = join(agentDir, "harness-state", "versions", harnessVersionDirectoryName(version)); - mkdirSync(join(bundlePath, "skills", "future"), { recursive: true }); + mkdirSync(bundlePath, { recursive: true }); writeFileSync( join(bundlePath, "manifest.json"), `${JSON.stringify( @@ -139,7 +166,7 @@ describe("global harness state startup", () => { )}\n`, ); writeFileSync(join(bundlePath, "EVOLVE.md"), memory); - writeFileSync(join(bundlePath, "skills", "future", "SKILL.md"), skill); + writeFileSync(join(bundlePath, "future.txt"), future); const pointerPath = join(agentDir, "harness-state", "current.json"); writeFileSync(pointerPath, `${JSON.stringify({ version }, null, 2)}\n`); const incompatiblePointer = readFileSync(pointerPath, "utf8"); @@ -147,7 +174,7 @@ describe("global harness state startup", () => { const loaded = await loadGlobalHarnessState(agentDir); expect(loaded.bundle).toBeUndefined(); - expect(loaded.warning).toContain('Unsupported harness artifact kind "skill"'); + expect(loaded.warning).toContain('Unsupported harness artifact kind "future-skill-asset"'); expect(readFileSync(pointerPath, "utf8")).toBe(incompatiblePointer); }); }); diff --git a/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts b/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts index 50acdcafb..31f664b44 100644 --- a/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts +++ b/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts @@ -174,6 +174,7 @@ function createMinimalResourceLoader(systemPrompt: string): ResourceLoader { return { getExtensions: () => ({ extensions: [], errors: [], runtime: createExtensionRuntime() }), getSkills: () => ({ skills: [], diagnostics: [] }), + setEvolvedSkills: () => {}, getPrompts: () => ({ prompts: [], diagnostics: [] }), getThemes: () => ({ themes: [], diagnostics: [] }), getAgentsFiles: () => ({ agentsFiles: [] }), diff --git a/packages/coding-agent/test/sdk-skills.test.ts b/packages/coding-agent/test/sdk-skills.test.ts index 94839cb94..72bac5a49 100644 --- a/packages/coding-agent/test/sdk-skills.test.ts +++ b/packages/coding-agent/test/sdk-skills.test.ts @@ -55,6 +55,7 @@ This is a test skill. const resourceLoader: ResourceLoader = { getExtensions: () => ({ extensions: [], errors: [], runtime: createExtensionRuntime() }), getSkills: () => ({ skills: [], diagnostics: [] }), + setEvolvedSkills: () => {}, getPrompts: () => ({ prompts: [], diagnostics: [] }), getThemes: () => ({ themes: [], diagnostics: [] }), getAgentsFiles: () => ({ agentsFiles: [] }), @@ -90,6 +91,7 @@ This is a test skill. const resourceLoader: ResourceLoader = { getExtensions: () => ({ extensions: [], errors: [], runtime: createExtensionRuntime() }), getSkills: () => ({ skills: [customSkill], diagnostics: [] }), + setEvolvedSkills: () => {}, getPrompts: () => ({ prompts: [], diagnostics: [] }), getThemes: () => ({ themes: [], diagnostics: [] }), getAgentsFiles: () => ({ agentsFiles: [] }), diff --git a/packages/coding-agent/test/suite/agent-session-harness-version.test.ts b/packages/coding-agent/test/suite/agent-session-harness-version.test.ts index 9c3d8aa8d..e7a997ebf 100644 --- a/packages/coding-agent/test/suite/agent-session-harness-version.test.ts +++ b/packages/coding-agent/test/suite/agent-session-harness-version.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { fauxAssistantMessage } from "@earendil-works/diffex-ai"; import { HarnessStateStore, type HarnessVersion } from "@earendil-works/diffex-harness-state"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveHarnessWorkspace } from "../../src/core/harness-state-runtime.ts"; import { createHarness, type Harness } from "./harness.ts"; describe("AgentSession harness revisions", () => { @@ -72,6 +73,98 @@ describe("AgentSession harness revisions", () => { ); }); + it("refreshes memory and evolved skills from one selected revision", async () => { + const { harness, store, rootVersion } = await createVersionedHarness(); + const workspace = await resolveHarnessWorkspace(harness.tempDir); + const skillContent = + "---\nname: atomic-review\ndescription: Review the active harness atomically.\n---\n\n# Atomic review\n"; + const revision = await store.saveVersion( + { + memory: "Use the matching evolved skill revision.", + skills: [ + { + path: "skills/atomic-review/SKILL.md", + content: skillContent, + workspace: workspace.workspace, + }, + ], + }, + rootVersion, + ); + await store.selectVersion(revision.manifest.version); + let providerPrompt = ""; + harness.setResponses([ + (context) => { + providerPrompt = context.systemPrompt ?? ""; + return fauxAssistantMessage("activated"); + }, + ]); + + await harness.session.prompt("observe the atomic revision"); + + expect(harness.session.activeHarnessVersion).toBe(revision.manifest.version); + expect(providerPrompt).toContain( + `\nUse the matching evolved skill revision.\n`, + ); + expect(providerPrompt).toContain("atomic-review"); + expect(providerPrompt).toContain(revision.state.skills[0]?.filePath); + expect(harness.session.createSkillReferences()).toEqual([ + expect.objectContaining({ + name: "atomic-review", + sourceKind: "evolved", + harnessRevision: revision.manifest.version, + version: revision.state.skills[0]?.contentHash, + }), + ]); + + await harness.session.switchHarnessVersion(rootVersion); + expect(harness.session.systemPrompt).not.toContain("Use the matching evolved skill revision."); + expect(harness.session.systemPrompt).not.toContain("atomic-review"); + expect(harness.session.createSkillReferences()).toEqual([]); + }); + + it("rejects an evolved reference after the selected harness changes", async () => { + const { harness, store, rootVersion } = await createVersionedHarness(); + const workspace = await resolveHarnessWorkspace(harness.tempDir); + const first = await store.saveVersion( + { + memory: "First revision.", + skills: [ + { + path: "skills/versioned-review/SKILL.md", + content: "---\nname: versioned-review\ndescription: Use the first review revision.\n---\n\n# First\n", + workspace: workspace.workspace, + }, + ], + }, + rootVersion, + ); + await harness.session.switchHarnessVersion(first.manifest.version); + const reference = harness.session.createSkillReferences()[0]; + if (reference === undefined) throw new Error("Expected an evolved skill reference"); + const second = await store.saveVersion( + { + memory: "Second revision.", + skills: [ + { + path: "skills/versioned-review/SKILL.md", + content: + "---\nname: versioned-review\ndescription: Use the second review revision.\n---\n\n# Second\n", + workspace: workspace.workspace, + }, + ], + }, + first.manifest.version, + ); + await store.selectVersion(second.manifest.version); + + await expect(harness.session.prompt(`${reference.reference}\n\nreview this`)).rejects.toThrow( + "is not part of the active harness revision", + ); + expect(harness.session.activeHarnessVersion).toBe(second.manifest.version); + expect(harness.getPendingResponseCount()).toBe(0); + }); + it("switches repeatedly between valid revisions", async () => { const { harness, store, rootVersion } = await createVersionedHarness(); const branch = await store.saveVersion({ memory: "Use the alternate harness." }, rootVersion); @@ -136,7 +229,21 @@ describe("AgentSession harness revisions", () => { const inFlight = harness.session.prompt("request admitted before activation"); await firstStarted; - const advance = await store.advance({ memory: "Use memory selected during the request." }, rootVersion); + const workspace = await resolveHarnessWorkspace(harness.tempDir); + const advance = await store.advance( + { + memory: "Use memory selected during the request.", + skills: [ + { + path: "skills/later-review/SKILL.md", + content: + "---\nname: later-review\ndescription: Review only after activation.\n---\n\n# Later review\n", + workspace: workspace.workspace, + }, + ], + }, + rootVersion, + ); expect(advance.status).toBe("advanced"); if (advance.status !== "advanced") throw new Error("Expected harness advance to succeed"); expect(harness.session.activeHarnessVersion).toBe(rootVersion); @@ -146,7 +253,9 @@ describe("AgentSession harness revisions", () => { await harness.session.prompt("request admitted after activation"); expect(providerPrompts[0]).not.toContain("Use memory selected during the request."); + expect(providerPrompts[0]).not.toContain("later-review"); expect(providerPrompts[1]).toContain("Use memory selected during the request."); + expect(providerPrompts[1]).toContain("later-review"); expect(harness.session.activeHarnessVersion).toBe(advance.bundle.manifest.version); }); diff --git a/packages/coding-agent/test/suite/harness.ts b/packages/coding-agent/test/suite/harness.ts index d59b31e5e..9412691a8 100644 --- a/packages/coding-agent/test/suite/harness.ts +++ b/packages/coding-agent/test/suite/harness.ts @@ -20,6 +20,7 @@ import { AgentSession, type AgentSessionEvent } from "../../src/core/agent-sessi import type { AgentSessionLifecycleObserver } from "../../src/core/agent-session-lifecycle.ts"; import { AuthStorage } from "../../src/core/auth-storage.ts"; import type { ExtensionRunner, SessionStartEvent } from "../../src/core/extensions/index.ts"; +import { resolveHarnessWorkspace } from "../../src/core/harness-state-runtime.ts"; import { convertToLlm } from "../../src/core/messages.ts"; import { SessionManager } from "../../src/core/session-manager.ts"; import type { Settings } from "../../src/core/settings-manager.ts"; @@ -188,6 +189,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise { const resourceLoader: ResourceLoader = { getExtensions: () => extensionsResult, getSkills: () => ({ skills: [], diagnostics: [] }), + setEvolvedSkills: () => {}, getPrompts: () => ({ prompts: [], diagnostics: [] }), getThemes: () => ({ themes: [], diagnostics: [] }), getAgentsFiles: () => ({ agentsFiles: [] }), diff --git a/packages/coding-agent/test/suite/subagent-services.test.ts b/packages/coding-agent/test/suite/subagent-services.test.ts index e306a4ca0..a9c5abc24 100644 --- a/packages/coding-agent/test/suite/subagent-services.test.ts +++ b/packages/coding-agent/test/suite/subagent-services.test.ts @@ -6,6 +6,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { AgentSession } from "../../src/core/agent-session.ts"; import { createAgentSessionFromServices, createAgentSessionServices } from "../../src/core/agent-session-services.ts"; import type { ToolDefinition } from "../../src/core/extensions/index.ts"; +import { resolveHarnessWorkspace } from "../../src/core/harness-state-runtime.ts"; import { SessionManager } from "../../src/core/session-manager.ts"; import { captureSubagentSessionConfig, @@ -406,6 +407,66 @@ describe("subagent sessions", () => { expect(child.getActiveToolNames()).toEqual([]); }); + it("pins child memory and skills to the parent harness snapshot", async () => { + const { harness, fixture, root } = await setupRoot(); + const store = root.harnessStateStore; + const rootVersion = root.activeHarnessVersion; + if (store === undefined || rootVersion === undefined) throw new Error("Expected an active harness store"); + const workspace = await resolveHarnessWorkspace(fixture.projectDir); + const parentRevision = await store.saveVersion( + { + memory: "Parent-pinned memory.", + skills: [ + { + path: "skills/child-review/SKILL.md", + content: + "---\nname: child-review\ndescription: Use the parent-pinned child workflow.\n---\n\n# Parent child review\n", + workspace: workspace.workspace, + }, + ], + }, + rootVersion, + ); + await root.switchHarnessVersion(parentRevision.manifest.version); + const captured = captureSubagentSessionConfig(root, fixture.agentDir); + const selected = await store.advance( + { + memory: "New globally selected memory.", + skills: [ + { + path: "skills/new-review/SKILL.md", + content: + "---\nname: new-review\ndescription: Use only after a fresh parent request.\n---\n\n# New review\n", + workspace: workspace.workspace, + }, + ], + }, + parentRevision.manifest.version, + ); + if (selected.status !== "advanced") throw new Error("Expected a selected successor harness"); + const child = await createSubagentSession(captured); + sessions.push(child); + let providerPrompt = ""; + harness.setResponses([ + (context) => { + providerPrompt = context.systemPrompt ?? ""; + return fauxAssistantMessage("pinned"); + }, + ]); + + await child.prompt("inspect the pinned harness"); + + expect(child.activeHarnessVersion).toBe(parentRevision.manifest.version); + expect(providerPrompt).toContain("Parent-pinned memory."); + expect(providerPrompt).not.toContain("New globally selected memory."); + expect(child.createSkillReferences()).toContainEqual( + expect.objectContaining({ + name: "child-review", + harnessRevision: parentRevision.manifest.version, + }), + ); + }); + it("keeps root and sibling transcripts independent", async () => { const { harness, fixture, root } = await setupRoot(); const first = await createChild(root, fixture.agentDir); diff --git a/packages/coding-agent/test/utilities.ts b/packages/coding-agent/test/utilities.ts index bdaec4e1f..1213d28a1 100644 --- a/packages/coding-agent/test/utilities.ts +++ b/packages/coding-agent/test/utilities.ts @@ -23,6 +23,7 @@ import { createExtensionRuntime, loadExtensionFromFactory } from "../src/core/ex import type { ResourceLoader } from "../src/core/resource-loader.ts"; import { SessionManager } from "../src/core/session-manager.ts"; import { SettingsManager } from "../src/core/settings-manager.ts"; +import type { SkillCatalog } from "../src/core/skills.ts"; import { createCodingTools } from "../src/index.ts"; /** @@ -217,9 +218,13 @@ export function createTestResourceLoader(options: CreateTestResourceLoaderOption runtime: createExtensionRuntime(), }; + let skillCatalog: SkillCatalog = { skills: [], diagnostics: [] }; return { getExtensions: () => extensionsResult, - getSkills: () => ({ skills: [], diagnostics: [] }), + getSkills: () => skillCatalog, + setEvolvedSkills: (catalog) => { + skillCatalog = catalog; + }, getPrompts: () => ({ prompts: [], diagnostics: [] }), getThemes: () => ({ themes: [], diagnostics: [] }), getAgentsFiles: () => ({ agentsFiles: [] }), diff --git a/packages/harness-state/src/index.ts b/packages/harness-state/src/index.ts index 3d2f317de..3c24b946e 100644 --- a/packages/harness-state/src/index.ts +++ b/packages/harness-state/src/index.ts @@ -681,6 +681,10 @@ function prepareHarnessState(input: HarnessStateInput, bundlePath: string): Prep skillNames.add(frontmatter.name); const filePath = join(bundlePath, ...inputSkill.path.split("/")); const contentHash = computeContentHash(contentBytes); + const workspace: HarnessWorkspaceIdentity = { + repositoryId: inputSkill.workspace.repositoryId, + ...(inputSkill.workspace.directoryId === undefined ? {} : { directoryId: inputSkill.workspace.directoryId }), + }; skills.push({ path: inputSkill.path, filePath, @@ -689,14 +693,14 @@ function prepareHarnessState(input: HarnessStateInput, bundlePath: string): Prep description: frontmatter.description, content: inputSkill.content, contentHash, - workspace: inputSkill.workspace, + workspace, disableModelInvocation: frontmatter.disableModelInvocation, }); skillArtifacts.push({ kind: SKILL_ARTIFACT_KIND, path: inputSkill.path, contentHash, - workspace: inputSkill.workspace, + workspace, }); files.set(inputSkill.path, contentBytes); } From 8c1db6704f8d4e553c192089c62357b8df84c5bc Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 12:16:28 -0400 Subject: [PATCH 04/10] feat(coding-agent): trace versioned skill observations --- .../coding-agent/src/core/agent-session.ts | 210 +++++++++++++++--- packages/coding-agent/src/core/skills.ts | 25 ++- packages/coding-agent/src/core/telemetry.ts | 91 +++++++- packages/coding-agent/src/core/tools/read.ts | 4 + .../test/invocation-tracing.test.ts | 63 +++++- .../test/suite/agent-session-prompt.test.ts | 28 ++- packages/coding-agent/test/tools.test.ts | 2 +- 7 files changed, 381 insertions(+), 42 deletions(-) diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 3214e5c09..8f81cd79a 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -57,7 +57,7 @@ import type { HarnessWorkspaceIdentity, } from "@earendil-works/diffex-harness-state"; import { getThemeByName, theme } from "../modes/interactive/theme/theme.ts"; -import { resolvePath } from "../utils/paths.ts"; +import { canonicalizePath, resolvePath } from "../utils/paths.ts"; import { sleep } from "../utils/sleep.ts"; import { normalizeToolResultImages } from "../utils/tool-result-images.ts"; import type { AgentSessionLifecycleObserver } from "./agent-session-lifecycle.ts"; @@ -128,6 +128,7 @@ import { createEvolvedSkillCatalog, createSkillReferenceSnapshot, formatSkillReferenceSnapshot, + isSkillListedInPrompt, type Skill, type SkillReference, type SkillReferenceSnapshot, @@ -206,10 +207,16 @@ export function parseSkillBlock(text: string): ParsedSkillBlock | null { interface SkillExpansion { text: string; references: SkillReferenceSnapshot[]; + forceLoaded: SkillReferenceSnapshot[]; +} + +interface ActiveSkillTrace { + operationSpan: CodingAgentTelemetrySpan<"diffex.coding_agent.operation">; + skills: readonly SkillReferenceSnapshot[]; } const SKILL_REFERENCE_PATTERN = - //g; + //g; /** Session-specific events that extend the core AgentEvent */ export type AgentSessionEvent = @@ -468,6 +475,7 @@ export class AgentSession { private _pendingNextTurnMessages: CustomMessage[] = []; private _pendingCurrentPlanRestoration = false; private readonly _skillReferenceSnapshots = new Map(); + private _activeSkillTrace: ActiveSkillTrace | undefined; // Compaction state private _compactionAbortController: AbortController | undefined = undefined; @@ -718,6 +726,7 @@ export class AgentSession { const normalizedContent = await normalizeToolResultImages(content, { autoResizeImages: this.settingsManager.getImageAutoResize(), }); + this._recordSkillRead(toolCall.name, args, result.details, hookResult?.isError ?? isError); if (!hookResult && normalizedContent === content) { return undefined; @@ -1472,6 +1481,84 @@ export class AgentSession { return snapshot; } + private _snapshotSkillCatalog(): SkillReferenceSnapshot[] { + return this._resourceLoader.getSkills().skills.map((skill) => this._snapshotSkill(skill)); + } + + private _skillObservationAttributes(skill: SkillReferenceSnapshot) { + return { + "diffex.skill.id": skill.skillId, + "diffex.skill.version": skill.version, + "diffex.skill.content_digest": skill.contentDigest, + "diffex.skill.source_kind": skill.sourceKind, + "diffex.skill.source": skill.sourceInfo.source, + "diffex.skill.scope": skill.scope, + ...(skill.harnessRevision === undefined ? {} : { "diffex.skill.harness_revision": skill.harnessRevision }), + ...(this._workspaceIdentity === undefined + ? {} + : { + "diffex.workspace.repository_id": this._workspaceIdentity.repositoryId, + ...(this._workspaceIdentity.directoryId === undefined + ? {} + : { "diffex.workspace.directory_id": this._workspaceIdentity.directoryId }), + }), + ...(this.activeHarnessVersion === undefined ? {} : { "diffex.harness.version": this.activeHarnessVersion }), + }; + } + + private _recordSkillAdmission( + operationSpan: CodingAgentTelemetrySpan<"diffex.coding_agent.operation">, + available: readonly SkillReferenceSnapshot[], + references: readonly SkillReferenceSnapshot[], + forceLoaded: readonly SkillReferenceSnapshot[], + systemPrompt: string, + ): readonly SkillReferenceSnapshot[] { + for (const skill of available) { + operationSpan.addEvent("diffex.skill.available", this._skillObservationAttributes(skill)); + if (isSkillListedInPrompt(systemPrompt, skill)) { + operationSpan.addEvent("diffex.skill.listed", this._skillObservationAttributes(skill)); + } + } + this._recordSkillReferences(operationSpan, references); + for (const skill of references) { + operationSpan.addEvent("diffex.skill.explicitly_referenced", this._skillObservationAttributes(skill)); + } + for (const skill of forceLoaded) { + operationSpan.addEvent("diffex.skill.force_loaded", this._skillObservationAttributes(skill)); + } + + const snapshots = new Map(); + for (const skill of [...available, ...references]) { + snapshots.set(this._skillReferenceKey(skill.skillId, skill.version), skill); + } + return [...snapshots.values()]; + } + + private _recordSkillRead(toolName: string, args: unknown, details: unknown, isError: boolean): void { + const activeTrace = this._activeSkillTrace; + if ( + isError || + toolName !== "read" || + !activeTrace || + typeof args !== "object" || + args === null || + typeof details !== "object" || + details === null + ) { + return; + } + const path = "path" in args && typeof args.path === "string" ? args.path : undefined; + const contentDigest = + "contentDigest" in details && typeof details.contentDigest === "string" ? details.contentDigest : undefined; + if (!path || !contentDigest) return; + const resolvedPath = canonicalizePath(resolvePath(path, this._cwd)); + for (const skill of activeTrace.skills) { + if (skill.contentDigest !== contentDigest) continue; + if (canonicalizePath(resolvePath(skill.location, this._cwd)) !== resolvedPath) continue; + activeTrace.operationSpan.addEvent("diffex.skill.read", this._skillObservationAttributes(skill)); + } + } + /** Snapshot every currently available skill for explicit selection. */ createSkillReferences(): SkillReference[] { const references: SkillReference[] = []; @@ -1978,6 +2065,7 @@ export class AgentSession { const preflightResult = options?.preflightResult; const signal = options?.signal; let messages: AgentMessage[] | undefined; + let activeSkillTrace: ActiveSkillTrace | undefined; let consumedNextTurnMessages: CustomMessage[] | undefined; let preflightAccepted = false; let pendingCompaction: CompactionAppendTransaction | undefined; @@ -2042,14 +2130,16 @@ export class AgentSession { } // Expand prompt templates and explicit skill references. + const availableSkills = this._snapshotSkillCatalog(); let expandedText = currentText; let referencedSkills: SkillReferenceSnapshot[] = []; + let forceLoadedSkills: SkillReferenceSnapshot[] = []; if (expandPromptTemplates) { expandedText = expandPromptTemplate(expandedText, [...this.promptTemplates]); const explicitReferences = this._expandSkillReferences(expandedText); expandedText = explicitReferences.text; referencedSkills = explicitReferences.references; - this._recordSkillReferences(operationSpan, referencedSkills); + forceLoadedSkills = explicitReferences.forceLoaded; } // If streaming, queue via steer() or followUp() based on option @@ -2059,10 +2149,17 @@ export class AgentSession { "Agent is already processing. Specify streamingBehavior ('steer' or 'followUp') to queue the message.", ); } + this._recordSkillAdmission( + operationSpan, + availableSkills, + referencedSkills, + forceLoadedSkills, + this.agent.state.systemPrompt, + ); if (options.streamingBehavior === "followUp") { - await this._queueFollowUp(expandedText, currentImages, referencedSkills); + await this._queueFollowUp(expandedText, currentImages, referencedSkills, forceLoadedSkills, false); } else { - await this._queueSteer(expandedText, currentImages, referencedSkills); + await this._queueSteer(expandedText, currentImages, referencedSkills, forceLoadedSkills, false); } preflightAccepted = true; preflightResult?.(true); @@ -2154,6 +2251,16 @@ export class AgentSession { this._systemPromptOverride = undefined; this.agent.state.systemPrompt = this._appendSubagentDelegatedMessageTrustGuidance(this._baseSystemPrompt); } + activeSkillTrace = { + operationSpan, + skills: this._recordSkillAdmission( + operationSpan, + availableSkills, + referencedSkills, + forceLoadedSkills, + this.agent.state.systemPrompt, + ), + }; } catch (error) { let failure = error; if (!preflightAccepted) { @@ -2200,7 +2307,12 @@ export class AgentSession { consumedNextTurnMessages = undefined; releaseUnsignaledPreflight(); this._releaseIdleSubagentEscalationsForSelectedBranch(); - await this._runAgentPrompt(messages); + this._activeSkillTrace = activeSkillTrace; + try { + await this._runAgentPrompt(messages); + } finally { + if (this._activeSkillTrace === activeSkillTrace) this._activeSkillTrace = undefined; + } } /** @@ -2244,33 +2356,39 @@ export class AgentSession { private _expandSkillReferences(text: string): SkillExpansion { const references: SkillReferenceSnapshot[] = []; + const forceLoaded: SkillReferenceSnapshot[] = []; const seen = new Set(); - const remaining = text.replace(SKILL_REFERENCE_PATTERN, (_match, skillId: string, version: string) => { - const key = this._skillReferenceKey(skillId, version); - const snapshot = this._skillReferenceSnapshots.get(key); - if (!snapshot) { - throw new Error(`Skill reference ${skillId}@${version} is not available in this session.`); - } - if ( - snapshot.sourceKind === "evolved" && - snapshot.harnessRevision !== this._harnessVersionBundle?.manifest.version - ) { - throw new Error( - `Evolved skill reference ${skillId}@${version} is not part of the active harness revision.`, - ); - } - if (!seen.has(key)) { - seen.add(key); - references.push(snapshot); - } - return ""; - }); - if (references.length === 0) return { text, references }; + const remaining = text.replace( + SKILL_REFERENCE_PATTERN, + (_match, skillId: string, version: string, forceLoadMarker: string | undefined) => { + const key = this._skillReferenceKey(skillId, version); + const snapshot = this._skillReferenceSnapshots.get(key); + if (!snapshot) { + throw new Error(`Skill reference ${skillId}@${version} is not available in this session.`); + } + if ( + snapshot.sourceKind === "evolved" && + snapshot.harnessRevision !== this._harnessVersionBundle?.manifest.version + ) { + throw new Error( + `Evolved skill reference ${skillId}@${version} is not part of the active harness revision.`, + ); + } + if (!seen.has(key)) { + seen.add(key); + references.push(snapshot); + if (forceLoadMarker !== undefined) forceLoaded.push(snapshot); + } + return ""; + }, + ); + if (references.length === 0) return { text, references, forceLoaded }; const userText = remaining.trim(); const skillBlocks = references.map(formatSkillReferenceSnapshot).join("\n\n"); return { text: userText ? `${skillBlocks}\n\n${userText}` : skillBlocks, references, + forceLoaded, }; } @@ -2290,7 +2408,12 @@ export class AgentSession { // Expand prompt templates and explicit skill references. const explicitReferences = this._expandSkillReferences(expandPromptTemplate(text, [...this.promptTemplates])); - await this._queueSteer(explicitReferences.text, images, explicitReferences.references); + await this._queueSteer( + explicitReferences.text, + images, + explicitReferences.references, + explicitReferences.forceLoaded, + ); } /** @@ -2308,7 +2431,12 @@ export class AgentSession { // Expand prompt templates and explicit skill references. const explicitReferences = this._expandSkillReferences(expandPromptTemplate(text, [...this.promptTemplates])); - await this._queueFollowUp(explicitReferences.text, images, explicitReferences.references); + await this._queueFollowUp( + explicitReferences.text, + images, + explicitReferences.references, + explicitReferences.forceLoaded, + ); } /** @@ -2318,9 +2446,19 @@ export class AgentSession { text: string, images?: ImageContent[], references: readonly SkillReferenceSnapshot[] = [], + forceLoaded: readonly SkillReferenceSnapshot[] = [], + recordAdmission = true, ): Promise { await this._traceOperation("queue", (operationSpan) => { - this._recordSkillReferences(operationSpan, references); + if (recordAdmission) { + this._recordSkillAdmission( + operationSpan, + this._snapshotSkillCatalog(), + references, + forceLoaded, + this.agent.state.systemPrompt, + ); + } return this._enqueueSteer(text, images); }); } @@ -2346,9 +2484,19 @@ export class AgentSession { text: string, images?: ImageContent[], references: readonly SkillReferenceSnapshot[] = [], + forceLoaded: readonly SkillReferenceSnapshot[] = [], + recordAdmission = true, ): Promise { await this._traceOperation("queue", (operationSpan) => { - this._recordSkillReferences(operationSpan, references); + if (recordAdmission) { + this._recordSkillAdmission( + operationSpan, + this._snapshotSkillCatalog(), + references, + forceLoaded, + this.agent.state.systemPrompt, + ); + } return this._enqueueFollowUp(text, images); }); } diff --git a/packages/coding-agent/src/core/skills.ts b/packages/coding-agent/src/core/skills.ts index 2298a1a4d..28d9d0bf6 100644 --- a/packages/coding-agent/src/core/skills.ts +++ b/packages/coding-agent/src/core/skills.ts @@ -234,7 +234,7 @@ export function createEvolvedSkillCatalog( } export function createSkillReferenceSnapshot(skill: Skill): SkillReferenceSnapshot { - const reference = ``; + const reference = ``; return { skillId: skill.skillId, version: skill.version, @@ -524,6 +524,23 @@ function loadSkillFromFile( * Skills with disableModelInvocation=true are excluded from the prompt * and can only be referenced explicitly through the skill selector. */ +function formatSkillPromptEntry(skill: Pick): string { + return [ + " ", + ` ${escapeXml(skill.name)}`, + ` ${escapeXml(skill.description)}`, + ` ${escapeXml(skill.filePath)}`, + " ", + ].join("\n"); +} + +/** Whether the exact catalog entry is progressively disclosed in a concrete model prompt. */ +export function isSkillListedInPrompt(prompt: string, skill: SkillReferenceSnapshot): boolean { + return prompt.includes( + formatSkillPromptEntry({ name: skill.name, description: skill.description, filePath: skill.location }), + ); +} + export function formatSkillsForPrompt(skills: Skill[]): string { const visibleSkills = skills.filter((s) => !s.disableModelInvocation); @@ -540,11 +557,7 @@ export function formatSkillsForPrompt(skills: Skill[]): string { ]; for (const skill of visibleSkills) { - lines.push(" "); - lines.push(` ${escapeXml(skill.name)}`); - lines.push(` ${escapeXml(skill.description)}`); - lines.push(` ${escapeXml(skill.filePath)}`); - lines.push(" "); + lines.push(formatSkillPromptEntry(skill)); } lines.push(""); diff --git a/packages/coding-agent/src/core/telemetry.ts b/packages/coding-agent/src/core/telemetry.ts index ed3982f0f..44c34cf9a 100644 --- a/packages/coding-agent/src/core/telemetry.ts +++ b/packages/coding-agent/src/core/telemetry.ts @@ -22,8 +22,71 @@ export function isInstallTelemetryEnabled( return telemetryEnv !== undefined ? isTruthyEnvFlag(telemetryEnv) : settingsManager.getEnableInstallTelemetry(); } +const SKILL_OBSERVATION_ATTRIBUTES = { + "diffex.skill.id": { + type: "string", + required: true, + cardinality: "high", + description: "Stable skill source identity", + }, + "diffex.skill.version": { + type: "string", + required: true, + cardinality: "high", + description: "Immutable skill version admitted for the request", + }, + "diffex.skill.content_digest": { + type: "string", + required: true, + cardinality: "high", + description: "Digest of the admitted skill content", + }, + "diffex.skill.source_kind": { + type: "string", + required: true, + values: ["installed", "evolved"], + description: "Skill source kind", + }, + "diffex.skill.source": { + type: "string", + required: true, + cardinality: "high", + description: "Catalog source that supplied the skill", + }, + "diffex.skill.scope": { + type: "string", + required: true, + values: ["global", "workspace"], + description: "Skill installation or evolution scope", + }, + "diffex.skill.harness_revision": { + type: "string", + required: false, + cardinality: "high", + description: "Harness revision that owns an evolved skill artifact", + }, + "diffex.workspace.repository_id": { + type: "string", + required: false, + cardinality: "high", + description: "Active workspace repository identity", + }, + "diffex.workspace.directory_id": { + type: "string", + required: false, + cardinality: "high", + description: "Active workspace directory identity", + }, + "diffex.harness.version": { + type: "string", + required: false, + cardinality: "high", + description: "Harness revision pinned to the request", + }, +} as const; + export const CODING_AGENT_TELEMETRY_SCHEMA = { - version: 3, + version: 4, spans: { "diffex.coding_agent.diagnostic": { description: "One runtime diagnostic surfaced to the process user", @@ -87,6 +150,32 @@ export const CODING_AGENT_TELEMETRY_SCHEMA = { description: "Exact skill id and version pairs explicitly loaded for the operation", }, }, + events: { + "diffex.skill.available": { + description: "One immutable skill version available when the request was admitted", + attributes: SKILL_OBSERVATION_ATTRIBUTES, + }, + "diffex.skill.listed": { + description: "One skill version listed in the system prompt sent to the model", + attributes: SKILL_OBSERVATION_ATTRIBUTES, + }, + "diffex.skill.selected": { + description: "One skill version returned by an automatic selector", + attributes: SKILL_OBSERVATION_ATTRIBUTES, + }, + "diffex.skill.explicitly_referenced": { + description: "One skill version explicitly referenced in the admitted user request", + attributes: SKILL_OBSERVATION_ATTRIBUTES, + }, + "diffex.skill.read": { + description: "One skill version successfully read through the model-callable read tool", + attributes: SKILL_OBSERVATION_ATTRIBUTES, + }, + "diffex.skill.force_loaded": { + description: "One skill version force-loaded from a versioned skill-selector reference", + attributes: SKILL_OBSERVATION_ATTRIBUTES, + }, + }, status: { default: "ok", errorWhen: "The operation fails or is aborted" }, }, "diffex.coding_agent.retry": { diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index 05b769571..90adcafff 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { basename, dirname, isAbsolute, relative, resolve as resolvePath, sep } from "node:path"; import type { AgentTool } from "@earendil-works/diffex-agent-core"; import type { Api, ImageContent, Model, TextContent } from "@earendil-works/diffex-ai"; @@ -31,6 +32,7 @@ export const readToolSystemPromptContribution = { export type ReadToolInput = Static; export interface ReadToolDetails { + contentDigest?: string; truncation?: TruncationResult; } @@ -270,6 +272,7 @@ export function createReadToolDefinition( // Read text content. const buffer = await ops.readFile(absolutePath); const textContent = buffer.toString("utf-8"); + const contentDigest = `sha256:${createHash("sha256").update(textContent).digest("hex")}`; const allLines = textContent.split("\n"); const totalFileLines = allLines.length; // Apply offset if specified. Convert from 1-indexed input to 0-indexed array access. @@ -317,6 +320,7 @@ export function createReadToolDefinition( // No truncation and no remaining user-limited content. outputText = truncation.content; } + details = { ...details, contentDigest }; content = [{ type: "text", text: outputText }]; } diff --git a/packages/coding-agent/test/invocation-tracing.test.ts b/packages/coding-agent/test/invocation-tracing.test.ts index 9f2957358..b10aae0d9 100644 --- a/packages/coding-agent/test/invocation-tracing.test.ts +++ b/packages/coding-agent/test/invocation-tracing.test.ts @@ -9,6 +9,7 @@ import { type TraceFooterRecord, type TraceRecord, type TraceSessionAttachRecord, + type TraceSpanEventRecord, } from "@earendil-works/diffex-harness-trace"; import lockfile from "proper-lockfile"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -21,7 +22,7 @@ import { SessionManager } from "../src/core/session-manager.ts"; const cliPath = resolve(__dirname, "../src/cli.ts"); const tsconfigPath = resolve(__dirname, "../../../tsconfig.json"); const FAUX_EXTENSION_SOURCE = ` -import { fauxAssistantMessage, registerFauxProvider } from "@earendil-works/diffex-ai/compat"; +import { fauxAssistantMessage, fauxToolCall, registerFauxProvider } from "@earendil-works/diffex-ai/compat"; const faux = registerFauxProvider(); faux.setResponses([ @@ -30,7 +31,11 @@ faux.setResponses([ stopReason: "error", errorMessage: "intentional tracing test failure", }) - : fauxAssistantMessage("first response"), + : process.env.DIFFEX_TRACE_SKILL_PATH + ? fauxAssistantMessage(fauxToolCall("read", { path: process.env.DIFFEX_TRACE_SKILL_PATH }), { + stopReason: "toolUse", + }) + : fauxAssistantMessage("first response"), fauxAssistantMessage("replacement response"), ]); @@ -477,6 +482,60 @@ describe("CLI invocation tracing", () => { expect(footer(resumedRecords)?.terminationStatus).toBe("completed"); }); + it("records immutable skill availability, listing, and read observations", async () => { + const directories = setup(); + const skillDirectory = join(directories.projectDir, "traced-skill"); + const skillPath = join(skillDirectory, "SKILL.md"); + mkdirSync(skillDirectory); + writeFileSync( + skillPath, + "---\nname: traced-skill\ndescription: Trace skill observations\n---\n\n# Traced skill\n", + ); + + const result = await runCli( + directories, + [...commonArgs(directories), "--skill", skillPath, "-p", "inspect the skill"], + { DIFFEX_TRACE_SKILL_PATH: skillPath }, + ); + + expect(result).toMatchObject({ code: 0, signal: null, stdout: "replacement response\n", stderr: "" }); + const records = await readTrace(onlyTracePath(directories.agentDir)); + const operationStart = records.find( + (record) => + record.type === "span_start" && + record.name === "diffex.coding_agent.operation" && + record.attributes["diffex.operation.kind"] === "prompt", + ); + expect(operationStart).toMatchObject({ + type: "span_start", + attributes: { "diffex.task.lineage.id": expect.any(String) }, + }); + if (operationStart?.type !== "span_start") throw new Error("Prompt operation span was not recorded"); + const operationSkillEvents = records.filter( + (record): record is TraceSpanEventRecord => + record.type === "span_event" && record.spanId === operationStart.spanId, + ); + const readEvent = operationSkillEvents.find((event) => event.name === "diffex.skill.read"); + const readSkillId = readEvent?.attributes["diffex.skill.id"]; + expect(readSkillId).toEqual(expect.stringMatching(/^skill:[a-f0-9]{64}$/)); + const skillEvents = operationSkillEvents.filter((event) => event.attributes["diffex.skill.id"] === readSkillId); + expect(skillEvents.map((event) => event.name)).toEqual([ + "diffex.skill.available", + "diffex.skill.listed", + "diffex.skill.read", + ]); + for (const event of skillEvents) { + expect(event.attributes).toMatchObject({ + "diffex.skill.id": expect.stringMatching(/^skill:[a-f0-9]{64}$/), + "diffex.skill.version": expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + "diffex.skill.content_digest": expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + "diffex.skill.source_kind": "installed", + "diffex.workspace.repository_id": expect.any(String), + }); + } + expect(JSON.stringify(records)).not.toContain("diffex.skill.used"); + }); + it("does not store invocation traces when persisted self-evolution is off", async () => { const directories = setup(); writeFileSync(join(directories.agentDir, "settings.json"), JSON.stringify({ evolution: { enabled: false } })); diff --git a/packages/coding-agent/test/suite/agent-session-prompt.test.ts b/packages/coding-agent/test/suite/agent-session-prompt.test.ts index 7250b0842..43dd1d962 100644 --- a/packages/coding-agent/test/suite/agent-session-prompt.test.ts +++ b/packages/coding-agent/test/suite/agent-session-prompt.test.ts @@ -248,8 +248,9 @@ describe("AgentSession prompt characterization", () => { const user = context.messages.find((message) => message.role === "user"); admittedPrompt = user ? getMessageText(user) : ""; admittedSystemPrompt = context.systemPrompt ?? ""; - return fauxAssistantMessage("ok"); + return fauxAssistantMessage(fauxToolCall("read", { path: unselectedPath }), { stopReason: "toolUse" }); }, + fauxAssistantMessage("ok"), ]); await harness.session.prompt(`${selected.reference}\n\nUse this for the task.`); @@ -271,6 +272,31 @@ describe("AgentSession prompt characterization", () => { expect(promptOperation?.attributes["diffex.skill.references"]).toEqual([ `${selected.skillId}@${selected.version}`, ]); + const skillEvents = promptOperation?.events.filter((event) => event.name.startsWith("diffex.skill.")) ?? []; + expect(skillEvents.map((event) => event.name)).toEqual([ + "diffex.skill.available", + "diffex.skill.available", + "diffex.skill.listed", + "diffex.skill.explicitly_referenced", + "diffex.skill.force_loaded", + "diffex.skill.read", + ]); + expect(skillEvents.find((event) => event.name === "diffex.skill.listed")?.attributes).toMatchObject({ + "diffex.skill.id": references.find((reference) => reference.name === "unselected")?.skillId, + "diffex.skill.source_kind": "installed", + "diffex.skill.scope": "workspace", + "diffex.workspace.repository_id": expect.any(String), + "diffex.workspace.directory_id": expect.any(String), + }); + expect(skillEvents.find((event) => event.name === "diffex.skill.force_loaded")?.attributes).toMatchObject({ + "diffex.skill.id": selected.skillId, + "diffex.skill.version": selected.version, + "diffex.skill.content_digest": selected.contentDigest, + }); + expect(skillEvents.find((event) => event.name === "diffex.skill.read")?.attributes["diffex.skill.id"]).toBe( + references.find((reference) => reference.name === "unselected")?.skillId, + ); + expect(skillEvents.some((event) => event.name.includes("used"))).toBe(false); }); it("expands prompt templates before sending the prompt", async () => { diff --git a/packages/coding-agent/test/tools.test.ts b/packages/coding-agent/test/tools.test.ts index 63b7f6260..57e74d41f 100644 --- a/packages/coding-agent/test/tools.test.ts +++ b/packages/coding-agent/test/tools.test.ts @@ -76,7 +76,7 @@ describe("Coding Agent Tools", () => { expect(getTextOutput(result)).toBe(content); // No truncation message since file fits within limits expect(getTextOutput(result)).not.toContain("Use offset="); - expect(result.details).toBeUndefined(); + expect(result.details).toEqual({ contentDigest: expect.stringMatching(/^sha256:[a-f0-9]{64}$/) }); }); it("should handle non-existent files", async () => { From 2548ef1372edd308e69c3ac3dab3b40f1f425a09 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 12:17:18 -0400 Subject: [PATCH 05/10] docs(coding-agent): document skill trace attribution --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/docs/invocation-traces.md | 11 +++++++++++ packages/coding-agent/docs/skills.md | 5 +++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index cd5a62f87..caeb1d048 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Added versioned skill availability, listing, reference, force-load, and digest-matched read observations to invocation traces without inferring skill use or causality. - Added atomic harness revisions for workspace-scoped evolved skills. - Added `pi auth check` provider/model auth preflight with optional credential output ([#7152](https://github.com/earendil-works/pi/issues/7152)). - Added durable, branch-scoped sub-agent rosters and child histories for saved sessions, with interrupted restoration and explicit wake without automatic work, replay, or result delivery. diff --git a/packages/coding-agent/docs/invocation-traces.md b/packages/coding-agent/docs/invocation-traces.md index 5df59a49c..ccde71172 100644 --- a/packages/coding-agent/docs/invocation-traces.md +++ b/packages/coding-agent/docs/invocation-traces.md @@ -46,6 +46,17 @@ Each fact has one authoritative source so a trace remains compact and materializ Agent and extension events remain live notification APIs. Their durable facts are represented by the owning session entry, session record, or telemetry span listed above rather than by copying event payloads into the trace. +Prompt and queue operation spans record one event per immutable skill identity for these observable facts: + +- `diffex.skill.available`: the installed or evolved version was available when the request was admitted +- `diffex.skill.listed`: the version appeared in the system prompt sent to the model +- `diffex.skill.selected`: an automatic selector returned the version, when such a selector is used +- `diffex.skill.explicitly_referenced`: the admitted user request contained its versioned reference +- `diffex.skill.force_loaded`: a `/skills` reference expanded its captured content into the request +- `diffex.skill.read`: the model-callable `read` tool successfully read content whose path and digest match the catalog version + +Each event includes the skill ID, version, content digest, source kind, source, scope, and, when configured, the active workspace identity and applicable skill and active harness revisions. Prompt spans carry the top-level task-lineage identifier, and queue spans admitted during a task remain nested beneath that prompt; retries, tool calls, subagents, and repeated reads therefore remain observable within the same top-level lineage. Tool results and terminal operation outcomes remain independent evidence. Traces never emit an inferred `used` event or treat listing, selection, loading, reading, or co-occurrence as proof that a skill caused an outcome. + Startup and configuration failures are recorded as runtime diagnostic spans after trace creation. Failures that prevent trace creation remain an explicit limitation because no durable trace destination exists. diff --git a/packages/coding-agent/docs/skills.md b/packages/coding-agent/docs/skills.md index 5a86edd56..9b6ceb1aa 100644 --- a/packages/coding-agent/docs/skills.md +++ b/packages/coding-agent/docs/skills.md @@ -67,9 +67,10 @@ For project-level Claude Code skills, add to `.diffex/settings.json`: 2. The system prompt includes available skills in XML format per the [specification](https://agentskills.io/integrate-skills). 3. When a task matches, the agent uses `read` to load the full SKILL.md (models don't always do this; use `/skills` to reference it explicitly). 4. Explicit references use the catalog's captured version, so a file change does not alter a request that has already been admitted. Reload resources to catalog a changed installed skill. -5. The agent follows the instructions, using relative paths to reference scripts and assets. +5. Invocation traces record availability, model listing, explicit references, `/skills` force-loading, and digest-matched reads as separate facts for the exact catalog version. +6. The agent follows the instructions, using relative paths to reference scripts and assets. -This is progressive disclosure: only names, descriptions, and locations are always in context; full instructions load on-demand. The TUI, system prompt, SDK, and evolution snapshots all read from the same catalog. +This is progressive disclosure: only names, descriptions, and locations are always in context; full instructions load on-demand. The TUI, system prompt, SDK, and evolution snapshots all read from the same catalog. Trace observations do not infer that a skill was used or caused an outcome. ## Skill Commands From 512504ba84b975fe024ab790d6b1900813c04abe Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 12:18:05 -0400 Subject: [PATCH 06/10] feat(coding-agent): cover evolved skill trace metadata --- .../agent-session-harness-version.test.ts | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/test/suite/agent-session-harness-version.test.ts b/packages/coding-agent/test/suite/agent-session-harness-version.test.ts index e7a997ebf..a1c485ddf 100644 --- a/packages/coding-agent/test/suite/agent-session-harness-version.test.ts +++ b/packages/coding-agent/test/suite/agent-session-harness-version.test.ts @@ -1,6 +1,7 @@ import { mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { InMemoryTelemetryContext } from "@earendil-works/diffex-agent-core"; import { fauxAssistantMessage } from "@earendil-works/diffex-ai"; import { HarnessStateStore, type HarnessVersion } from "@earendil-works/diffex-harness-state"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -26,18 +27,21 @@ describe("AgentSession harness revisions", () => { harness: Harness; store: HarnessStateStore; rootVersion: HarnessVersion; + telemetry: InMemoryTelemetryContext; }> { const agentDir = join(tmpdir(), `diffex-harness-session-${Date.now()}-${Math.random().toString(36).slice(2)}`); mkdirSync(agentDir, { recursive: true }); agentDirs.push(agentDir); const store = new HarnessStateStore(agentDir); const root = await store.initialize(); + const telemetry = new InMemoryTelemetryContext(); const harness = await createHarness({ harnessStateStore: store, harnessVersionBundle: root, + telemetryContext: telemetry, }); harnesses.push(harness); - return { harness, store, rootVersion: root.manifest.version }; + return { harness, store, rootVersion: root.manifest.version, telemetry }; } it("applies an advanced EVOLVE revision only to requests after a live switch", async () => { @@ -74,7 +78,7 @@ describe("AgentSession harness revisions", () => { }); it("refreshes memory and evolved skills from one selected revision", async () => { - const { harness, store, rootVersion } = await createVersionedHarness(); + const { harness, store, rootVersion, telemetry } = await createVersionedHarness(); const workspace = await resolveHarnessWorkspace(harness.tempDir); const skillContent = "---\nname: atomic-review\ndescription: Review the active harness atomically.\n---\n\n# Atomic review\n"; @@ -116,6 +120,25 @@ describe("AgentSession harness revisions", () => { version: revision.state.skills[0]?.contentHash, }), ]); + const promptOperation = telemetry + .getSpans() + .find( + (span) => + span.name === "diffex.coding_agent.operation" && span.attributes["diffex.operation.kind"] === "prompt", + ); + const availableEvent = promptOperation?.events.find( + (event) => + event.name === "diffex.skill.available" && event.attributes["diffex.skill.source_kind"] === "evolved", + ); + expect(availableEvent?.attributes).toMatchObject({ + "diffex.skill.version": revision.state.skills[0]?.contentHash, + "diffex.skill.source_kind": "evolved", + "diffex.skill.scope": "workspace", + "diffex.skill.harness_revision": revision.manifest.version, + "diffex.harness.version": revision.manifest.version, + "diffex.workspace.repository_id": workspace.workspace.repositoryId, + "diffex.workspace.directory_id": workspace.workspace.directoryId, + }); await harness.session.switchHarnessVersion(rootVersion); expect(harness.session.systemPrompt).not.toContain("Use the matching evolved skill revision."); From 71371b93d00acb830796cbe23f19fe1f94a681bd Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 12:33:34 -0400 Subject: [PATCH 07/10] fix(agent): reject stale harness activation --- .../evolution/src/evolution-memory-writer.ts | 15 +++++++++++- .../test/evolution-memory-writer.test.ts | 23 ++++++++++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/packages/evolution/src/evolution-memory-writer.ts b/packages/evolution/src/evolution-memory-writer.ts index ad5c147f6..289b312f6 100644 --- a/packages/evolution/src/evolution-memory-writer.ts +++ b/packages/evolution/src/evolution-memory-writer.ts @@ -917,7 +917,20 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { ) { throw new Error(`Harness activation for batch "${batch.batchId}" produced an unexpected revision`); } - await this.#harnessState.selectVersionIfCurrent(evolved.manifest.version, candidate.expectedBaseVersion); + const selection = await this.#harnessState.selectVersionIfCurrent( + evolved.manifest.version, + candidate.expectedBaseVersion, + ); + if (selection.status === "conflict" && selection.actualCurrent !== evolved.manifest.version) { + throw new EvolutionExecutionError("Harness revision changed before evolution activation", { + reasonCode: "harness-base-conflict", + retryable: false, + diagnostics: { + expectedHarnessVersion: selection.expectedCurrent, + actualHarnessVersion: selection.actualCurrent, + }, + }); + } return { batchId: batch.batchId, status: "activated", diff --git a/packages/evolution/test/evolution-memory-writer.test.ts b/packages/evolution/test/evolution-memory-writer.test.ts index b87326eb4..35bfb9f28 100644 --- a/packages/evolution/test/evolution-memory-writer.test.ts +++ b/packages/evolution/test/evolution-memory-writer.test.ts @@ -770,7 +770,7 @@ describe("evolution memory writer", () => { expect((await harnessState.loadCurrent()).state.memory).toBe(initialMemory); }); - it("branches from the frozen base when another harness is selected before writing", async () => { + it("stages from the frozen base but rejects activation after the selected revision changes", async () => { const { harnessState, snapshot, writer } = await fixture(); const selectedBatch = batch(snapshot, [ { @@ -795,14 +795,25 @@ describe("evolution memory writer", () => { idempotencyKey: "write-frozen-base", lease: writerLease, }); - const activation = await writer.activate(selectedBatch, writeResult, { - idempotencyKey: "activate-frozen-base", - lease: writerLease, + await expect( + writer.activate(selectedBatch, writeResult, { + idempotencyKey: "activate-frozen-base", + lease: writerLease, + }), + ).rejects.toMatchObject({ + message: "Harness revision changed before evolution activation", + reasonCode: "harness-base-conflict", + retryable: false, + diagnostics: { + expectedHarnessVersion: snapshot.harnessVersion, + actualHarnessVersion: competing.bundle.manifest.version, + }, }); expect(await harnessState.loadCurrent()).toEqual(competing.bundle); - if (activation.resultingHarnessVersion === null) throw new Error("Expected an evolved harness version"); - const evolved = await harnessState.loadVersion(activation.resultingHarnessVersion); + const evolvedVersion = writeResult.output?.resultingHarnessVersion; + if (typeof evolvedVersion !== "string") throw new Error("Expected a staged evolved harness version"); + const evolved = await harnessState.loadVersion(evolvedVersion); expect(evolved.manifest.base).toBe(snapshot.harnessVersion); expect(evolved.state.memory).toContain("Keep this evolution on its frozen harness base."); expect(evolved.state.memory).not.toContain("Competing selected memory."); From f5494d368f1646100a4687ef78e20438c61ae1e9 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 12:34:45 -0400 Subject: [PATCH 08/10] feat(coding-agent): cover manual evolved skill rollback --- .../agent-session-harness-version.test.ts | 134 +++++++++++++++++- 1 file changed, 131 insertions(+), 3 deletions(-) diff --git a/packages/coding-agent/test/suite/agent-session-harness-version.test.ts b/packages/coding-agent/test/suite/agent-session-harness-version.test.ts index a1c485ddf..4a6982027 100644 --- a/packages/coding-agent/test/suite/agent-session-harness-version.test.ts +++ b/packages/coding-agent/test/suite/agent-session-harness-version.test.ts @@ -1,9 +1,18 @@ -import { mkdirSync, rmSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { InMemoryTelemetryContext } from "@earendil-works/diffex-agent-core"; import { fauxAssistantMessage } from "@earendil-works/diffex-ai"; -import { HarnessStateStore, type HarnessVersion } from "@earendil-works/diffex-harness-state"; +import { + computeHarnessVersion, + HARNESS_VERSION_SCHEMA_VERSION, + HarnessStateStore, + type HarnessVersion, + type HarnessVersionMetadata, + type HarnessWorkspaceIdentity, + harnessVersionDirectoryName, +} from "@earendil-works/diffex-harness-state"; import { afterEach, describe, expect, it, vi } from "vitest"; import { resolveHarnessWorkspace } from "../../src/core/harness-state-runtime.ts"; import { createHarness, type Harness } from "./harness.ts"; @@ -28,6 +37,7 @@ describe("AgentSession harness revisions", () => { store: HarnessStateStore; rootVersion: HarnessVersion; telemetry: InMemoryTelemetryContext; + agentDir: string; }> { const agentDir = join(tmpdir(), `diffex-harness-session-${Date.now()}-${Math.random().toString(36).slice(2)}`); mkdirSync(agentDir, { recursive: true }); @@ -41,7 +51,55 @@ describe("AgentSession harness revisions", () => { telemetryContext: telemetry, }); harnesses.push(harness); - return { harness, store, rootVersion: root.manifest.version, telemetry }; + return { harness, store, rootVersion: root.manifest.version, telemetry, agentDir }; + } + + function contentHash(content: string): HarnessVersion { + return `sha256:${createHash("sha256").update(content, "utf8").digest("hex")}`; + } + + function writeHarnessRevision(options: { + agentDir: string; + base: HarnessVersion; + metadata: HarnessVersionMetadata; + memory: string; + skill: { name: string; content: string; workspace: HarnessWorkspaceIdentity }; + }): HarnessVersion { + const skillPath = `skills/${options.skill.name}/SKILL.md`; + const artifacts = [ + { kind: "evolution-memory", path: "EVOLVE.md", contentHash: contentHash(options.memory) }, + { + kind: "skill", + path: skillPath, + contentHash: contentHash(options.skill.content), + workspace: options.skill.workspace, + }, + ]; + const version = computeHarnessVersion({ + schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, + base: options.base, + metadata: options.metadata, + artifacts, + }); + const revisionPath = join(options.agentDir, "harness-state", "versions", harnessVersionDirectoryName(version)); + mkdirSync(join(revisionPath, "skills", options.skill.name), { recursive: true }); + writeFileSync( + join(revisionPath, "manifest.json"), + `${JSON.stringify( + { + schemaVersion: HARNESS_VERSION_SCHEMA_VERSION, + version, + base: options.base, + metadata: options.metadata, + artifacts, + }, + null, + 2, + )}\n`, + ); + writeFileSync(join(revisionPath, "EVOLVE.md"), options.memory); + writeFileSync(join(revisionPath, ...skillPath.split("/")), options.skill.content); + return version; } it("applies an advanced EVOLVE revision only to requests after a live switch", async () => { @@ -146,6 +204,76 @@ describe("AgentSession harness revisions", () => { expect(harness.session.createSkillReferences()).toEqual([]); }); + it("loads a manually assembled skill revision and restores the prior catalog at the next prompt", async () => { + const { harness, store, rootVersion, telemetry, agentDir } = await createVersionedHarness(); + const root = await store.loadVersion(rootVersion); + const workspace = await resolveHarnessWorkspace(harness.tempDir); + const skillContent = + "---\nname: manual-review\ndescription: Trace the manually assembled review workflow.\n---\n\n# Manual review\n"; + const revision = writeHarnessRevision({ + agentDir, + base: rootVersion, + metadata: root.manifest.metadata, + memory: "Use the manually assembled harness revision.\n", + skill: { name: "manual-review", content: skillContent, workspace: workspace.workspace }, + }); + const selected = await store.selectVersionIfCurrent(revision, rootVersion); + expect(selected.status).toBe("selected"); + const providerPrompts: string[] = []; + harness.setResponses([ + (context) => { + providerPrompts.push(context.systemPrompt ?? ""); + return fauxAssistantMessage("manual revision"); + }, + (context) => { + providerPrompts.push(context.systemPrompt ?? ""); + return fauxAssistantMessage("restored revision"); + }, + ]); + + await harness.session.prompt("trace the selected skill"); + + expect(harness.session.activeHarnessVersion).toBe(revision); + expect(providerPrompts[0]).toContain("Use the manually assembled harness revision."); + expect(providerPrompts[0]).toContain("manual-review"); + const reference = harness.session.createSkillReferences()[0]; + expect(reference).toMatchObject({ + name: "manual-review", + sourceKind: "evolved", + harnessRevision: revision, + version: contentHash(skillContent), + }); + const firstPromptOperation = telemetry + .getSpans() + .find( + (span) => + span.name === "diffex.coding_agent.operation" && span.attributes["diffex.operation.kind"] === "prompt", + ); + expect( + firstPromptOperation?.events.find( + (event) => + event.name === "diffex.skill.available" && event.attributes["diffex.skill.source_kind"] === "evolved", + )?.attributes, + ).toMatchObject({ + "diffex.skill.version": contentHash(skillContent), + "diffex.skill.harness_revision": revision, + "diffex.harness.version": revision, + "diffex.workspace.repository_id": workspace.workspace.repositoryId, + "diffex.workspace.directory_id": workspace.workspace.directoryId, + }); + + const rolledBack = await store.selectVersionIfCurrent(rootVersion, revision); + expect(rolledBack.status).toBe("selected"); + expect(harness.session.activeHarnessVersion).toBe(revision); + + await harness.session.prompt("observe the restored catalog"); + + expect(harness.session.activeHarnessVersion).toBe(rootVersion); + expect(providerPrompts[1]).not.toContain("Use the manually assembled harness revision."); + expect(providerPrompts[1]).not.toContain("manual-review"); + expect(harness.session.createSkillReferences()).toEqual([]); + }); + it("rejects an evolved reference after the selected harness changes", async () => { const { harness, store, rootVersion } = await createVersionedHarness(); const workspace = await resolveHarnessWorkspace(harness.tempDir); From b55318cbd0d08d3ef56327c156af6a0c0af481fb Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 12:35:35 -0400 Subject: [PATCH 09/10] docs(coding-agent): note atomic activation conflict handling --- packages/coding-agent/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index caeb1d048..487c8f9fe 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -17,6 +17,7 @@ ### Fixed +- Fixed stale evolution activation from reporting success after another harness revision was selected. - Fixed the footer showing an unknown harness after self-evolution switched to the new version. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. From 9413a1578e440a218ab1e0b35db6cb7353f4cc1d Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 27 Aug 2026 12:37:14 -0400 Subject: [PATCH 10/10] docs(coding-agent): remove changelog updates --- packages/coding-agent/CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 487c8f9fe..b63185600 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,8 +4,6 @@ ### Added -- Added versioned skill availability, listing, reference, force-load, and digest-matched read observations to invocation traces without inferring skill use or causality. -- Added atomic harness revisions for workspace-scoped evolved skills. - Added `pi auth check` provider/model auth preflight with optional credential output ([#7152](https://github.com/earendil-works/pi/issues/7152)). - Added durable, branch-scoped sub-agent rosters and child histories for saved sessions, with interrupted restoration and explicit wake without automatic work, replay, or result delivery. - Added bounded child-to-parent escalation reports for root-controlled sub-agents, with hidden delivery that can resume coordinated work without consuming terminal results. @@ -17,8 +15,6 @@ ### Fixed -- Fixed stale evolution activation from reporting success after another harness revision was selected. -- Fixed the footer showing an unknown harness after self-evolution switched to the new version. - Fixed the evolution menu remaining on its loading screen after cancellation and flickering during active-job refreshes. - Fixed assistant Markdown headings and fenced code blocks appearing with literal source delimiters in the interactive transcript. - Fixed extension TUI method wrappers recursing indefinitely when delegating to the original method ([#7731](https://github.com/earendil-works/pi/issues/7731)).