From 11d6bd1f12ec478e007e32afd9a3abbb2ff21701 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Tue, 18 Aug 2026 19:07:14 -0700 Subject: [PATCH 1/3] feat(coding-agent): add trace cursors - Gives invocation traces stable session entry boundaries\n- Keeps session persistence independent from trace codecs --- .../coding-agent/src/core/session-manager.ts | 61 ++++++++++++++++++- .../test/session-manager-trace-cursor.test.ts | 58 ++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 packages/coding-agent/test/session-manager-trace-cursor.test.ts diff --git a/packages/coding-agent/src/core/session-manager.ts b/packages/coding-agent/src/core/session-manager.ts index f52392953..8f4a86ff3 100644 --- a/packages/coding-agent/src/core/session-manager.ts +++ b/packages/coding-agent/src/core/session-manager.ts @@ -1,6 +1,6 @@ import type { AgentMessage } from "@earendil-works/diffex-agent-core"; import { type ImageContent, type Message, type TextContent, type Usage, uuidv7 } from "@earendil-works/diffex-ai"; -import { randomUUID } from "crypto"; +import { createHash, randomUUID } from "crypto"; import { appendFileSync, closeSync, @@ -171,6 +171,22 @@ export interface SessionContext { model: { provider: string; modelId: string } | null; } +export interface SessionEntryCursor { + /** Zero-based ordinal among non-header session entries. */ + entryOrdinal: number; + entryId: string; + digest: { + kind: "content"; + algorithm: "sha256"; + value: string; + }; +} + +export interface SessionEntryCursorRange { + start: SessionEntryCursor; + end: SessionEntryCursor; +} + export interface SessionInfo { path: string; id: string; @@ -1302,6 +1318,49 @@ export class SessionManager { return this.fileEntries.filter((e): e is SessionEntry => e.type !== "session"); } + /** Return the number of non-header entries admitted to this session. */ + getEntryCount(): number { + return this.fileEntries.length > 0 && this.fileEntries[0].type === "session" + ? this.fileEntries.length - 1 + : this.fileEntries.length; + } + + /** + * Describe entries appended from the supplied zero-based ordinal through the + * current end of the session. + * + * Digests cover the exact JSON representation produced by SessionManager for + * each entry, giving trace consumers an immutable content reference without + * exposing persistence internals. + */ + getEntryCursorRange(firstEntryOrdinal: number): SessionEntryCursorRange | undefined { + if (!Number.isInteger(firstEntryOrdinal) || firstEntryOrdinal < 0) { + throw new RangeError("Session entry ordinal must be a non-negative integer"); + } + + const entries = this.getEntries(); + if (firstEntryOrdinal >= entries.length) return undefined; + + const cursor = (entryOrdinal: number): SessionEntryCursor => { + const entry = entries[entryOrdinal]; + if (!entry) throw new Error(`Session entry ${entryOrdinal} is unavailable`); + return { + entryOrdinal, + entryId: entry.id, + digest: { + kind: "content", + algorithm: "sha256", + value: createHash("sha256").update(JSON.stringify(entry)).digest("hex"), + }, + }; + }; + + return { + start: cursor(firstEntryOrdinal), + end: cursor(entries.length - 1), + }; + } + /** * Get the session as a tree structure. Returns a shallow defensive copy of all entries. * A well-formed session has exactly one root (first entry with parentId === null). diff --git a/packages/coding-agent/test/session-manager-trace-cursor.test.ts b/packages/coding-agent/test/session-manager-trace-cursor.test.ts new file mode 100644 index 000000000..3b44192f0 --- /dev/null +++ b/packages/coding-agent/test/session-manager-trace-cursor.test.ts @@ -0,0 +1,58 @@ +import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { SessionManager } from "../src/core/session-manager.ts"; + +describe("SessionManager trace cursors", () => { + const tempDirectories: string[] = []; + + afterEach(() => { + for (const directory of tempDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("returns immutable content references for appended entry boundaries", () => { + const directory = mkdtempSync(join(tmpdir(), "diffex-session-cursor-")); + tempDirectories.push(directory); + const manager = SessionManager.create(directory, join(directory, "sessions")); + + expect(manager.getEntryCount()).toBe(0); + manager.appendModelChange("faux", "model"); + const firstEntryOrdinal = manager.getEntryCount(); + manager.appendThinkingLevelChange("medium"); + manager.appendSessionInfo("trace cursor"); + + const entries = manager.getEntries(); + const range = manager.getEntryCursorRange(firstEntryOrdinal); + expect(range).toEqual({ + start: { + entryOrdinal: 1, + entryId: entries[1].id, + digest: { + kind: "content", + algorithm: "sha256", + value: createHash("sha256").update(JSON.stringify(entries[1])).digest("hex"), + }, + }, + end: { + entryOrdinal: 2, + entryId: entries[2].id, + digest: { + kind: "content", + algorithm: "sha256", + value: createHash("sha256").update(JSON.stringify(entries[2])).digest("hex"), + }, + }, + }); + expect(manager.getEntryCursorRange(manager.getEntryCount())).toBeUndefined(); + }); + + it("rejects invalid entry ordinals", () => { + const manager = SessionManager.inMemory(); + expect(() => manager.getEntryCursorRange(-1)).toThrow("non-negative integer"); + expect(() => manager.getEntryCursorRange(0.5)).toThrow("non-negative integer"); + }); +}); From c8c3c753f0badf1010e9fd08cf4cc01a2aa9f40c Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Tue, 18 Aug 2026 19:29:16 -0700 Subject: [PATCH 2/3] feat(coding-agent): trace CLI invocations - Preserve telemetry across persisted session replacements - Keep ephemeral commands and recorder failures passive --- .../coding-agent/docs/invocation-traces.md | 28 +++ packages/coding-agent/package.json | 7 +- .../src/core/agent-session-lifecycle.ts | 16 ++ .../src/core/agent-session-runtime.ts | 31 ++- .../src/core/agent-session-services.ts | 3 + .../coding-agent/src/core/agent-session.ts | 32 ++- .../coding-agent/src/core/invocation-trace.ts | 196 ++++++++++++++++++ packages/coding-agent/src/core/sdk.ts | 11 + packages/coding-agent/src/core/telemetry.ts | 23 +- packages/coding-agent/src/main.ts | 59 +++++- packages/coding-agent/src/modes/print-mode.ts | 10 +- packages/coding-agent/tsconfig.build.json | 1 + packages/harness-trace/src/recorder.ts | 6 +- scripts/local-release.mjs | 4 +- tsconfig.json | 1 + vitest.base.ts | 3 + 16 files changed, 410 insertions(+), 21 deletions(-) create mode 100644 packages/coding-agent/docs/invocation-traces.md create mode 100644 packages/coding-agent/src/core/agent-session-lifecycle.ts create mode 100644 packages/coding-agent/src/core/invocation-trace.ts diff --git a/packages/coding-agent/docs/invocation-traces.md b/packages/coding-agent/docs/invocation-traces.md new file mode 100644 index 000000000..55c43998d --- /dev/null +++ b/packages/coding-agent/docs/invocation-traces.md @@ -0,0 +1,28 @@ +# Invocation Traces + +Diffex creates one invocation trace for each CLI process whose initial session is persisted. +Ephemeral commands such as `--no-session`, `--help`, and `--list-models` do not create traces. + +Trace creation is passive. +If the trace file cannot be created, Diffex prints one warning and continues without durable telemetry. + +## Storage and Discovery + +Trace files are stored in `~/.diffex/agent/traces/` by default, or beneath the directory selected by `DIFFEX_CODING_AGENT_DIR`. +Each file is named `_.jsonl`. +The trace directory is restricted to the current user with mode `0700`, and each trace file uses mode `0600` on POSIX systems. + +Tools can discover traces by scanning this directory for `.jsonl` files and validating them with the harness trace reader. +The trace header identifies the producing Diffex build and runtime. +Session attachment records identify every referenced session ID, path, format, and transition reason. + +## Retention and Session Changes + +Traces are retained until the user explicitly removes them. +Diffex does not currently apply an automatic age or size limit. + +Deleting a session does not delete its traces. +Moving or renaming a session does not rewrite existing trace references. +This preserves invocation evidence while allowing readers to report that referenced session content is missing or moved. + +Trace files contain execution telemetry and immutable session entry references, not duplicated transcript content. diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 7a842781a..f209dcd39 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -3,6 +3,9 @@ "version": "0.2.0", "description": "Coding agent CLI with read, bash, edit, write, update_plan tools and session management", "type": "module", + "imports": { + "#harness-trace": "./dist/vendor/harness-trace/index.js" + }, "diffexConfig": { "configDir": ".diffex" }, @@ -35,8 +38,8 @@ "scripts": { "clean": "shx rm -rf dist", "build": "tsgo -p tsconfig.build.json && shx chmod +x dist/cli.js dist/rpc-entry.js && npm run copy-assets", - "build:binary": "npm --prefix ../harness-state run build && npm --prefix ../tui run build && npm --prefix ../telemetry run build && npm --prefix ../ai run build && npm --prefix ../agent run build && npm --prefix ../protocol run build && npm --prefix ../client run build && npm run build && bun build --compile --no-compile-autoload-bunfig ./dist/bun/cli.js ./src/utils/image-resize-worker.ts --outfile dist/pi && npm run copy-binary-assets", - "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/modes/interactive/assets && shx cp src/modes/interactive/assets/*.png dist/modes/interactive/assets/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/", + "build:binary": "npm --prefix ../harness-state run build && npm --prefix ../tui run build && npm --prefix ../telemetry run build && npm --prefix ../harness-trace run build && npm --prefix ../ai run build && npm --prefix ../agent run build && npm --prefix ../protocol run build && npm --prefix ../client run build && npm run build && bun build --compile --no-compile-autoload-bunfig ./dist/bun/cli.js ./src/utils/image-resize-worker.ts --outfile dist/pi && npm run copy-binary-assets", + "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/modes/interactive/assets && shx cp src/modes/interactive/assets/*.png dist/modes/interactive/assets/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/ && shx mkdir -p dist/vendor && shx cp -r ../harness-trace/dist dist/vendor/harness-trace", "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/assets && shx cp src/modes/interactive/assets/*.png dist/assets/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && shx cp -r docs dist/ && shx cp -r examples dist/ && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/", "test": "vitest --run", "shrinkwrap": "node ../../scripts/generate-coding-agent-shrinkwrap.mjs", diff --git a/packages/coding-agent/src/core/agent-session-lifecycle.ts b/packages/coding-agent/src/core/agent-session-lifecycle.ts new file mode 100644 index 000000000..c36db775f --- /dev/null +++ b/packages/coding-agent/src/core/agent-session-lifecycle.ts @@ -0,0 +1,16 @@ +import type { HarnessVersion } from "@earendil-works/diffex-harness-state"; +import type { SessionManager } from "./session-manager.ts"; + +export type InvocationSessionAttachmentReason = "startup" | "resume" | "new" | "import" | "fork" | "switch"; + +export interface AgentSessionOperationSettlement { + operationId: string; + sessionManager: SessionManager; + firstEntryOrdinal: number; +} + +/** Passive process-owned observation hooks for durable invocation tracing. */ +export interface AgentSessionLifecycleObserver { + operationSettled(settlement: AgentSessionOperationSettlement): void | Promise; + harnessActivated(version: HarnessVersion): void | Promise; +} diff --git a/packages/coding-agent/src/core/agent-session-runtime.ts b/packages/coding-agent/src/core/agent-session-runtime.ts index d6cc5d65c..3a1ba562a 100644 --- a/packages/coding-agent/src/core/agent-session-runtime.ts +++ b/packages/coding-agent/src/core/agent-session-runtime.ts @@ -3,6 +3,7 @@ import { basename, join, resolve } from "node:path"; import type { TelemetryContext } from "@earendil-works/diffex-agent-core"; import { resolvePath } from "../utils/paths.ts"; import type { AgentSession } from "./agent-session.ts"; +import type { InvocationSessionAttachmentReason } from "./agent-session-lifecycle.ts"; import type { AgentSessionRuntimeDiagnostic, AgentSessionServices } from "./agent-session-services.ts"; import type { ProjectTrustContext, @@ -40,8 +41,11 @@ export type CreateAgentSessionRuntimeFactory = (options: { sessionStartEvent?: SessionStartEvent; projectTrustContext?: ProjectTrustContext; telemetryContext?: TelemetryContext; + invocationSessionReason?: InvocationSessionAttachmentReason; }) => Promise; +export type InvocationTerminationStatus = "completed" | "failed" | "aborted"; + /** * Thrown when /import references a JSONL file path that does not exist. */ @@ -82,6 +86,7 @@ export class AgentSessionRuntime { private _diagnostics: AgentSessionRuntimeDiagnostic[]; private _modelFallbackMessage?: string; private disposePromise: Promise | undefined; + private readonly onDispose: ((status: InvocationTerminationStatus) => Promise) | undefined; constructor( _session: AgentSession, @@ -89,6 +94,7 @@ export class AgentSessionRuntime { createRuntime: CreateAgentSessionRuntimeFactory, _diagnostics: AgentSessionRuntimeDiagnostic[] = [], _modelFallbackMessage?: string, + onDispose?: (status: InvocationTerminationStatus) => Promise, ) { this._session = _session; this._services = _services; @@ -99,6 +105,7 @@ export class AgentSessionRuntime { }); this._diagnostics = _diagnostics; this._modelFallbackMessage = _modelFallbackMessage; + this.onDispose = onDispose; } get services(): AgentSessionServices { @@ -249,6 +256,7 @@ export class AgentSessionRuntime { sessionManager, sessionStartEvent: { type: "session_start", reason: "resume", previousSessionFile }, projectTrustContext: options?.projectTrustContextFactory?.(sessionManager.getCwd()), + invocationSessionReason: "switch", }), ); await this.finishSessionReplacement(options?.withSession); @@ -281,6 +289,7 @@ export class AgentSessionRuntime { agentDir: this.services.agentDir, sessionManager, sessionStartEvent: { type: "session_start", reason: "new", previousSessionFile }, + invocationSessionReason: "new", }), ); if (options?.setup) { @@ -335,6 +344,7 @@ export class AgentSessionRuntime { agentDir: this.services.agentDir, sessionManager, sessionStartEvent: { type: "session_start", reason: "fork", previousSessionFile }, + invocationSessionReason: "fork", }), ); await this.finishSessionReplacement(options?.withSession); @@ -358,6 +368,7 @@ export class AgentSessionRuntime { agentDir: this.services.agentDir, sessionManager, sessionStartEvent: { type: "session_start", reason: "fork", previousSessionFile }, + invocationSessionReason: "fork", }), ); await this.finishSessionReplacement(options?.withSession); @@ -377,6 +388,7 @@ export class AgentSessionRuntime { agentDir: this.services.agentDir, sessionManager, sessionStartEvent: { type: "session_start", reason: "fork", previousSessionFile }, + invocationSessionReason: "fork", }), ); await this.finishSessionReplacement(options?.withSession); @@ -421,13 +433,14 @@ export class AgentSessionRuntime { agentDir: this.services.agentDir, sessionManager, sessionStartEvent: { type: "session_start", reason: "resume", previousSessionFile }, + invocationSessionReason: "import", }), ); await this.finishSessionReplacement(); return { cancelled: false }; } - dispose(): Promise { + dispose(terminationStatus: InvocationTerminationStatus = "completed"): Promise { if (this.disposePromise) return this.disposePromise; let resolveDispose!: () => void; @@ -437,9 +450,20 @@ export class AgentSessionRuntime { rejectDispose = reject; }); this.disposePromise = disposePromise; - void this.teardownCurrent("quit").then(resolveDispose, rejectDispose); + void this.finishDispose(terminationStatus).then(resolveDispose, rejectDispose); return disposePromise; } + + private async finishDispose(terminationStatus: InvocationTerminationStatus): Promise { + let teardownFailure: unknown; + try { + await this.teardownCurrent("quit"); + } catch (error) { + teardownFailure = error; + } + await this.onDispose?.(teardownFailure === undefined ? terminationStatus : "failed"); + if (teardownFailure !== undefined) throw teardownFailure; + } } /** @@ -456,6 +480,8 @@ export async function createAgentSessionRuntime( sessionManager: SessionManager; sessionStartEvent?: SessionStartEvent; telemetryContext?: TelemetryContext; + invocationSessionReason?: InvocationSessionAttachmentReason; + onDispose?: (status: InvocationTerminationStatus) => Promise; }, ): Promise { assertSessionCwdExists(options.sessionManager, options.cwd); @@ -466,6 +492,7 @@ export async function createAgentSessionRuntime( createRuntime, result.diagnostics, result.modelFallbackMessage, + options.onDispose, ); } diff --git a/packages/coding-agent/src/core/agent-session-services.ts b/packages/coding-agent/src/core/agent-session-services.ts index 7ae2e2083..adabc28ce 100644 --- a/packages/coding-agent/src/core/agent-session-services.ts +++ b/packages/coding-agent/src/core/agent-session-services.ts @@ -4,6 +4,7 @@ import type { Model } from "@earendil-works/diffex-ai"; import type { HarnessStateStore, HarnessVersionBundle } from "@earendil-works/diffex-harness-state"; import { getAgentDir } from "../config.ts"; import { resolvePath } from "../utils/paths.ts"; +import type { AgentSessionLifecycleObserver } from "./agent-session-lifecycle.ts"; import type { SessionStartEvent, ToolDefinition } from "./extensions/index.ts"; import { loadGlobalHarnessState } from "./harness-state-runtime.ts"; import { ModelRuntime } from "./model-runtime.ts"; @@ -68,6 +69,7 @@ export interface CreateAgentSessionFromServicesOptions { noTools?: CreateAgentSessionOptions["noTools"]; customTools?: ToolDefinition[]; subagents?: CreateAgentSessionOptions["subagents"]; + lifecycleObserver?: AgentSessionLifecycleObserver; } /** @@ -243,6 +245,7 @@ export async function createAgentSessionFromServices( noTools: options.noTools, customTools: options.customTools, subagents: options.subagents, + lifecycleObserver: options.lifecycleObserver, sessionStartEvent: options.sessionStartEvent, }); } diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 5a785b304..2e3710367 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -60,6 +60,7 @@ import { stripFrontmatter } from "../utils/frontmatter.ts"; import { resolvePath } from "../utils/paths.ts"; import { sleep } from "../utils/sleep.ts"; import { normalizeToolResultImages } from "../utils/tool-result-images.ts"; +import type { AgentSessionLifecycleObserver } from "./agent-session-lifecycle.ts"; import { formatNoApiKeyFoundMessage, formatNoModelSelectedMessage } from "./auth-guidance.ts"; import { type BashResult, executeBashWithOperations } from "./bash-executor.ts"; import { @@ -256,6 +257,8 @@ export interface AgentSessionConfig { harnessVersionBundle?: HarnessVersionBundle; /** Parent context for passive runtime telemetry. */ telemetryContext?: TelemetryContext; + /** Process-owned observer for durable lifecycle boundaries. */ + lifecycleObserver?: AgentSessionLifecycleObserver; } export interface ExtensionBindings { @@ -412,6 +415,7 @@ export class AgentSession { private readonly _harnessStateStore: HarnessStateStore | undefined; private _harnessVersionBundle: HarnessVersionBundle | undefined; private readonly _telemetryContext: TelemetryContext; + private readonly _lifecycleObserver: AgentSessionLifecycleObserver | undefined; private _harnessSwitchInProgress = false; private _subagentWaitAbortController: AbortController | undefined; private _agentRunAborted = false; @@ -436,6 +440,7 @@ export class AgentSession { this._harnessStateStore = config.harnessStateStore; this._harnessVersionBundle = config.harnessVersionBundle; this._telemetryContext = config.telemetryContext ?? NOOP_TELEMETRY_CONTEXT; + this._lifecycleObserver = config.lifecycleObserver; this._armInitialCurrentPlanRestoration(); @@ -1197,6 +1202,7 @@ export class AgentSession { this._baseSystemPrompt = this._rebuildSystemPrompt(this.getActiveToolNames()); this._systemPromptOverride = undefined; this.agent.state.systemPrompt = this._baseSystemPrompt; + await this._notifyHarnessActivated(version); } finally { this._harnessSwitchInProgress = false; } @@ -1353,11 +1359,13 @@ export class AgentSession { kind: "prompt" | "compaction" | "navigation" | "abort" | "queue" | "bash" | "export", callback: (telemetryContext: TelemetryContext) => Result | Promise, ): Promise { - return startCodingAgentSpan( + const operationId = uuidv7(); + const firstEntryOrdinal = this.sessionManager.getEntryCount(); + const operation = startCodingAgentSpan( this.agent.telemetryContext, "diffex.coding_agent.operation", { - "diffex.operation.id": uuidv7(), + "diffex.operation.id": operationId, "diffex.session.id": this.sessionId, "diffex.operation.kind": kind, }, @@ -1385,6 +1393,26 @@ export class AgentSession { } }, ); + return operation.finally(() => this._notifyOperationSettled({ operationId, firstEntryOrdinal })); + } + + private async _notifyOperationSettled(options: { operationId: string; firstEntryOrdinal: number }): Promise { + try { + await this._lifecycleObserver?.operationSettled({ + ...options, + sessionManager: this.sessionManager, + }); + } catch { + // Lifecycle observers are passive and never change operation behavior. + } + } + + private async _notifyHarnessActivated(version: HarnessVersion): Promise { + try { + await this._lifecycleObserver?.harnessActivated(version); + } catch { + // Lifecycle observers are passive and never change activation behavior. + } } private async _runAgentPrompt(messages: AgentMessage | AgentMessage[]): Promise { diff --git a/packages/coding-agent/src/core/invocation-trace.ts b/packages/coding-agent/src/core/invocation-trace.ts new file mode 100644 index 000000000..4e9cda0dc --- /dev/null +++ b/packages/coding-agent/src/core/invocation-trace.ts @@ -0,0 +1,196 @@ +import { join } from "node:path"; +import { + AGENT_RUNTIME_TELEMETRY_SCHEMA, + AI_TELEMETRY_SCHEMA, + NOOP_TELEMETRY_CONTEXT, + type TelemetryContext, +} from "@earendil-works/diffex-agent-core"; +import { uuidv7 } from "@earendil-works/diffex-ai"; +import { + createJsonlTraceContext, + type JsonlTraceContext, + type SessionAttachmentReason, + type TraceTerminationStatus, +} from "#harness-trace"; +import { VERSION } from "../config.ts"; +import type { AgentSessionLifecycleObserver, InvocationSessionAttachmentReason } from "./agent-session-lifecycle.ts"; +import type { AgentSessionRuntimeDiagnostic } from "./agent-session-services.ts"; +import type { SessionManager } from "./session-manager.ts"; +import { CURRENT_SESSION_VERSION } from "./session-manager.ts"; +import { CODING_AGENT_TELEMETRY_SCHEMA } from "./telemetry.ts"; + +const TRACE_DIRECTORY_NAME = "traces"; + +export interface InvocationTraceCreationResult { + trace: InvocationTrace; + diagnostic?: AgentSessionRuntimeDiagnostic; +} + +export interface CreateInvocationTraceOptions { + agentDir: string; + sessionManager: SessionManager; + diffexVersion?: string; + sourceRevision?: string; +} + +function invocationTracePath(agentDir: string, traceId: string): string { + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + return join(agentDir, TRACE_DIRECTORY_NAME, `${timestamp}_${traceId}.jsonl`); +} + +function sourceRevision(version: string, configuredRevision: string | undefined): string { + return configuredRevision?.trim() || process.env.DIFFEX_SOURCE_REVISION?.trim() || `release:${version}`; +} + +function runtimeIdentity(): { kind: string; version: string } { + const bunVersion = process.versions.bun; + return bunVersion ? { kind: "bun", version: bunVersion } : { kind: "node", version: process.versions.node }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function finalTerminationStatus( + requested: TraceTerminationStatus, + signalTermination: TraceTerminationStatus | undefined, +): TraceTerminationStatus { + if (requested === "failed") return "failed"; + return signalTermination ?? requested; +} + +/** Process-owned adapter between coding-agent lifecycle facts and JSONL traces. */ +export class InvocationTrace implements AgentSessionLifecycleObserver { + readonly destinationPath: string | undefined; + readonly telemetryContext: TelemetryContext; + private readonly recorder: JsonlTraceContext | undefined; + private readonly sessionBindings = new WeakMap(); + private activeHarnessVersion: string | undefined; + private signalTermination: TraceTerminationStatus | undefined; + private readonly signalCleanupHandlers: Array<() => void> = []; + + constructor(recorder?: JsonlTraceContext) { + this.recorder = recorder; + this.destinationPath = recorder?.destinationPath; + this.telemetryContext = recorder ?? NOOP_TELEMETRY_CONTEXT; + if (recorder) this.registerSignalObservers(); + } + + private registerSignalObservers(): void { + const signals: NodeJS.Signals[] = ["SIGTERM"]; + if (process.platform !== "win32") signals.push("SIGHUP"); + for (const signal of signals) { + const handler = () => { + this.signalTermination = "aborted"; + }; + process.on(signal, handler); + this.signalCleanupHandlers.push(() => process.off(signal, handler)); + } + } + + async attachSession(sessionManager: SessionManager, reason: InvocationSessionAttachmentReason): Promise { + if (!this.recorder || !sessionManager.isPersisted() || this.sessionBindings.has(sessionManager)) return; + const path = sessionManager.getSessionFile(); + if (!path) return; + const bindingId = uuidv7(); + this.sessionBindings.set(sessionManager, bindingId); + this.recorder.attachSession({ + bindingId, + sessionId: sessionManager.getSessionId(), + path, + sessionFormat: "coding-agent", + formatVersion: CURRENT_SESSION_VERSION, + reason: reason satisfies SessionAttachmentReason, + }); + await this.recorder.flush(); + } + + async operationSettled(options: { + operationId: string; + sessionManager: SessionManager; + firstEntryOrdinal: number; + }): Promise { + if (!this.recorder) return; + const bindingId = this.sessionBindings.get(options.sessionManager); + if (bindingId) { + const cursor = options.sessionManager.getEntryCursorRange(options.firstEntryOrdinal); + if (cursor) { + this.recorder.recordSessionSegment({ + bindingId, + lane: "main", + operationId: options.operationId, + cursor: { kind: "coding-agent-v3", ...cursor }, + }); + } + } + await this.recorder.flush(); + } + + async harnessActivated(version: string): Promise { + if (!this.recorder || version === this.activeHarnessVersion) return; + this.recorder.activateHarness({ + harnessVersion: version, + reason: this.activeHarnessVersion === undefined ? "initial" : "switch", + }); + this.activeHarnessVersion = version; + await this.recorder.flush(); + } + + async recordDiagnostics(diagnostics: readonly AgentSessionRuntimeDiagnostic[]): Promise { + if (!this.recorder) return; + for (const diagnostic of diagnostics) { + await this.recorder.startSpan( + { + name: "diffex.coding_agent.diagnostic", + attributes: { + "diffex.diagnostic.severity": diagnostic.type, + "diffex.diagnostic.message": diagnostic.message, + }, + }, + (span) => { + if (diagnostic.type === "error") span.setStatus({ status: "error" }); + }, + ); + } + await this.recorder.flush(); + } + + async close(terminationStatus: TraceTerminationStatus = "completed"): Promise { + for (const cleanup of this.signalCleanupHandlers.splice(0)) cleanup(); + await this.recorder?.close(finalTerminationStatus(terminationStatus, this.signalTermination)); + } +} + +export async function createInvocationTrace( + options: CreateInvocationTraceOptions, +): Promise { + if (!options.sessionManager.isPersisted()) return { trace: new InvocationTrace() }; + + const traceId = uuidv7(); + const version = options.diffexVersion ?? VERSION; + const runtime = runtimeIdentity(); + try { + const recorder = await createJsonlTraceContext({ + destinationPath: invocationTracePath(options.agentDir, traceId), + traceId, + diffexVersion: version, + sourceRevision: sourceRevision(version, options.sourceRevision), + runtimeKind: runtime.kind, + runtimeVersion: runtime.version, + telemetrySchemaVersions: { + agent: AGENT_RUNTIME_TELEMETRY_SCHEMA.version, + ai: AI_TELEMETRY_SCHEMA.version, + "coding-agent": CODING_AGENT_TELEMETRY_SCHEMA.version, + }, + }); + return { trace: new InvocationTrace(recorder) }; + } catch (error) { + return { + trace: new InvocationTrace(), + diagnostic: { + type: "warning", + message: `Invocation tracing is unavailable: ${errorMessage(error)}`, + }, + }; + } +} diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index c0a66ab80..8b967821e 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -12,6 +12,7 @@ import type { HarnessStateStore, HarnessVersionBundle } from "@earendil-works/di import { getAgentDir } from "../config.ts"; import { resolvePath } from "../utils/paths.ts"; import { AgentSession } from "./agent-session.ts"; +import type { AgentSessionLifecycleObserver } from "./agent-session-lifecycle.ts"; import { formatNoModelsAvailableMessage } from "./auth-guidance.ts"; import { DEFAULT_THINKING_LEVEL } from "./defaults.ts"; import type { ExtensionRunner, LoadExtensionsResult, SessionStartEvent, ToolDefinition } from "./extensions/index.ts"; @@ -111,6 +112,8 @@ export interface CreateAgentSessionOptions { harnessVersionBundle?: HarnessVersionBundle; /** Optional parent for passive runtime telemetry. */ telemetryContext?: TelemetryContext; + /** Optional process-owned observer for durable lifecycle boundaries. */ + lifecycleObserver?: AgentSessionLifecycleObserver; } /** Result from createAgentSession */ @@ -485,7 +488,15 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} harnessStateStore: loadedHarnessState.store, harnessVersionBundle: loadedHarnessState.bundle, telemetryContext, + lifecycleObserver: options.lifecycleObserver, }); + if (session.activeHarnessVersion) { + try { + await options.lifecycleObserver?.harnessActivated(session.activeHarnessVersion); + } catch { + // Lifecycle observers are passive and never change session creation. + } + } subagentController?.bindParent(session); const extensionsResult = resourceLoader.getExtensions(); diff --git a/packages/coding-agent/src/core/telemetry.ts b/packages/coding-agent/src/core/telemetry.ts index 2c79c4405..e901e49d0 100644 --- a/packages/coding-agent/src/core/telemetry.ts +++ b/packages/coding-agent/src/core/telemetry.ts @@ -23,8 +23,29 @@ export function isInstallTelemetryEnabled( } export const CODING_AGENT_TELEMETRY_SCHEMA = { - version: 1, + version: 2, spans: { + "diffex.coding_agent.diagnostic": { + description: "One runtime diagnostic surfaced to the process user", + parents: { kind: "any" }, + startAttributes: { + "diffex.diagnostic.severity": { + type: "string", + required: true, + values: ["info", "warning", "error"], + description: "Diagnostic severity", + }, + "diffex.diagnostic.message": { + type: "string", + required: true, + cardinality: "high", + sensitive: true, + description: "Exact surfaced diagnostic message", + }, + }, + endAttributes: {}, + status: { default: "ok", errorWhen: "The diagnostic has error severity" }, + }, "diffex.coding_agent.operation": { description: "One user-visible coding-agent session operation", parents: { kind: "any" }, diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index d571fd12d..270cf155b 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -33,7 +33,12 @@ import { createProjectTrustContext } from "./cli/project-trust.ts"; import { selectSession } from "./cli/session-picker.ts"; import { shouldRunFirstTimeSetup, showFirstTimeSetup, showStartupSelector } from "./cli/startup-ui.ts"; import { APP_NAME, ENV_SESSION_DIR, expandTildePath, getAgentDir, getPackageDir, VERSION } from "./config.ts"; -import { type CreateAgentSessionRuntimeFactory, createAgentSessionRuntime } from "./core/agent-session-runtime.ts"; +import type { InvocationSessionAttachmentReason } from "./core/agent-session-lifecycle.ts"; +import { + type AgentSessionRuntime, + type CreateAgentSessionRuntimeFactory, + createAgentSessionRuntime, +} from "./core/agent-session-runtime.ts"; import { type AgentSessionRuntimeDiagnostic, createAgentSessionFromServices, @@ -44,6 +49,7 @@ import { AuthStorage, ReadOnlyAuthStorage } from "./core/auth-storage.ts"; import { exportFromFile } from "./core/export-html/index.ts"; import type { InlineExtension } from "./core/extensions/types.ts"; import { applyHttpProxySettings, configureHttpDispatcher } from "./core/http-dispatcher.ts"; +import { createInvocationTrace } from "./core/invocation-trace.ts"; import { resolveCliModel, resolveModelScope, type ScopedModel } from "./core/model-resolver.ts"; import { ModelRuntime } from "./core/model-runtime.ts"; import { restoreStdout, takeOverStdout } from "./core/output-guard.ts"; @@ -136,6 +142,15 @@ function isPlainRuntimeMetadataCommand(parsed: Args): boolean { return !parsed.print && parsed.mode === undefined && (parsed.help === true || parsed.listModels !== undefined); } +function initialSessionAttachmentReason( + parsed: Args, + sessionManager: SessionManager, +): InvocationSessionAttachmentReason { + if (parsed.fork) return "fork"; + if (parsed.resume || parsed.continue || parsed.session || sessionManager.getEntryCount() > 0) return "resume"; + return "startup"; +} + async function runAuthCommand(args: string[]): Promise { if (isAuthCommandHelp(args)) { printAuthCommandHelp(); @@ -656,7 +671,8 @@ export async function main(args: string[], options?: MainOptions) { time("runMigrations"); const startupSettingsManager = SettingsManager.create(cwd, agentDir); - reportDiagnostics(collectSettingsDiagnostics(startupSettingsManager, "startup session lookup")); + const startupSettingsDiagnostics = collectSettingsDiagnostics(startupSettingsManager, "startup session lookup"); + reportDiagnostics(startupSettingsDiagnostics); // Experimental first-time setup: theme choice and analytics opt-in. // Runs before any runtime services are created so the chosen settings apply everywhere. @@ -697,6 +713,12 @@ export async function main(args: string[], options?: MainOptions) { } sessionManager.appendSessionInfo(name); } + const invocationTraceResult = await createInvocationTrace({ agentDir, sessionManager }); + const invocationTrace = invocationTraceResult.trace; + if (invocationTraceResult.diagnostic) { + reportDiagnostics([invocationTraceResult.diagnostic]); + } + await invocationTrace.recordDiagnostics(startupSettingsDiagnostics); time("createSessionManager"); const trustStore = new ProjectTrustStore(agentDir); @@ -719,7 +741,9 @@ export async function main(args: string[], options?: MainOptions) { sessionStartEvent, projectTrustContext, telemetryContext, + invocationSessionReason = "startup", }) => { + await invocationTrace.attachSession(sessionManager, invocationSessionReason); const isInitialRuntime = sessionStartEvent === undefined; const projectTrustDiagnostics: AgentSessionRuntimeDiagnostic[] = []; const cachedProjectTrust = projectTrustByCwd.get(cwd); @@ -830,24 +854,39 @@ export async function main(args: string[], options?: MainOptions) { excludeTools: sessionOptions.excludeTools, noTools: sessionOptions.noTools, customTools: sessionOptions.customTools, + lifecycleObserver: invocationTrace, }); const cliThinkingOverride = parsed.thinking !== undefined || cliThinkingFromModel; if (created.session.model && cliThinkingOverride) { created.session.setThinkingLevel(created.session.thinkingLevel); } - return { + const result = { ...created, services, diagnostics, }; + await invocationTrace.recordDiagnostics(diagnostics); + return result; }; time("createRuntime"); - const runtime = await createAgentSessionRuntime(createRuntime, { - cwd: sessionManager.getCwd(), - agentDir, - sessionManager, - }); + let runtime: AgentSessionRuntime; + try { + runtime = await createAgentSessionRuntime(createRuntime, { + cwd: sessionManager.getCwd(), + agentDir, + sessionManager, + invocationSessionReason: initialSessionAttachmentReason(parsed, sessionManager), + telemetryContext: invocationTrace.telemetryContext, + onDispose: (status) => invocationTrace.close(status), + }); + } catch (error) { + await invocationTrace.recordDiagnostics([ + { type: "error", message: error instanceof Error ? error.message : String(error) }, + ]); + await invocationTrace.close("failed"); + throw error; + } time("createAgentSessionRuntime"); const { services, session, modelFallbackMessage } = runtime; const { settingsManager, modelRuntime, resourceLoader } = services; @@ -898,18 +937,21 @@ export async function main(args: string[], options?: MainOptions) { if (runtime.diagnostics.some((diagnostic) => diagnostic.message.includes("Failed to load extension"))) { console.error(chalk.yellow(EXTENSION_LOAD_FAILURE_HINT)); } + await runtime.dispose("failed"); process.exit(1); } time("createAgentSession"); if (appMode !== "interactive" && !session.model) { console.error(chalk.red(formatNoModelsAvailableMessage())); + await runtime.dispose("failed"); process.exit(1); } const startupBenchmark = isTruthyEnvFlag(process.env.DIFFEX_STARTUP_BENCHMARK); if (startupBenchmark && appMode !== "interactive") { console.error(chalk.red("Error: DIFFEX_STARTUP_BENCHMARK only supports interactive mode")); + await runtime.dispose("failed"); process.exit(1); } @@ -952,6 +994,7 @@ export async function main(args: string[], options?: MainOptions) { if (process.stderr.writableLength > 0) { await new Promise((resolve) => process.stderr.once("drain", resolve)); } + await runtime.dispose(); return; } diff --git a/packages/coding-agent/src/modes/print-mode.ts b/packages/coding-agent/src/modes/print-mode.ts index 54eda5393..f09c78d9f 100644 --- a/packages/coding-agent/src/modes/print-mode.ts +++ b/packages/coding-agent/src/modes/print-mode.ts @@ -37,14 +37,15 @@ export async function runPrintMode(runtimeHost: AgentSessionRuntime, options: Pr let unsubscribe: (() => void) | undefined; let unsubscribeBackpressure: (() => void) | undefined; let disposed = false; + let terminationStatus: "completed" | "failed" | "aborted" = "completed"; const signalCleanupHandlers: Array<() => void> = []; - const disposeRuntime = async (): Promise => { + const disposeRuntime = async (status = terminationStatus): Promise => { if (disposed) return; disposed = true; unsubscribe?.(); unsubscribeBackpressure?.(); - await runtimeHost.dispose(); + await runtimeHost.dispose(status); }; const registerSignalHandlers = (): void => { @@ -56,7 +57,8 @@ export async function runPrintMode(runtimeHost: AgentSessionRuntime, options: Pr for (const signal of signals) { const handler = () => { killTrackedDetachedChildren(); - void disposeRuntime().finally(() => { + terminationStatus = "aborted"; + void disposeRuntime("aborted").finally(() => { process.exit(signal === "SIGHUP" ? 129 : 143); }); }; @@ -157,12 +159,14 @@ export async function runPrintMode(runtimeHost: AgentSessionRuntime, options: Pr return exitCode; } catch (error: unknown) { + terminationStatus = "failed"; console.error(error instanceof Error ? error.message : String(error)); return 1; } finally { for (const cleanup of signalCleanupHandlers) { cleanup(); } + if (exitCode !== 0) terminationStatus = "failed"; await disposeRuntime(); await flushRawStdout(); } diff --git a/packages/coding-agent/tsconfig.build.json b/packages/coding-agent/tsconfig.build.json index 11c4eb393..36c614db9 100644 --- a/packages/coding-agent/tsconfig.build.json +++ b/packages/coding-agent/tsconfig.build.json @@ -3,6 +3,7 @@ "compilerOptions": { "outDir": "./dist", "paths": { + "#harness-trace": ["../harness-trace/dist/index.d.ts"], "@earendil-works/diffex-agent-core": ["../agent/dist/index.d.ts"], "@earendil-works/diffex-agent-core/*": ["../agent/dist/*.d.ts"], "@earendil-works/diffex-ai": ["../ai/dist/index.d.ts"], diff --git a/packages/harness-trace/src/recorder.ts b/packages/harness-trace/src/recorder.ts index e6a1746bc..faf419adf 100644 --- a/packages/harness-trace/src/recorder.ts +++ b/packages/harness-trace/src/recorder.ts @@ -1,4 +1,4 @@ -import { type FileHandle, mkdir, open } from "node:fs/promises"; +import { chmod, type FileHandle, mkdir, open } from "node:fs/promises"; import { dirname } from "node:path"; import { type AttributeValue, @@ -396,7 +396,9 @@ export async function createJsonlTraceContext(options: CreateJsonlTraceContextOp }, }; const encodedHeader = encodeTraceRecord(header); - await mkdir(dirname(options.destinationPath), { recursive: true, mode: 0o700 }); + const destinationDirectory = dirname(options.destinationPath); + await mkdir(destinationDirectory, { recursive: true, mode: 0o700 }); + await chmod(destinationDirectory, 0o700); const file = await open(options.destinationPath, "wx", 0o600); try { await file.writeFile(encodedHeader, { encoding: "utf8" }); diff --git a/scripts/local-release.mjs b/scripts/local-release.mjs index 9e2f9a873..56e9cbbc0 100644 --- a/scripts/local-release.mjs +++ b/scripts/local-release.mjs @@ -18,6 +18,8 @@ const packages = [ { directory: "packages/coding-agent", name: "@earendil-works/diffex-coding-agent" }, ]; +const privateBuildPackages = [{ directory: "packages/harness-trace" }]; + function printUsage() { console.log(`Usage: node scripts/local-release.mjs [options] @@ -223,7 +225,7 @@ if (!options.skipCheck) { run("npm", ["run", "check"], { cwd: repoRoot }); } -for (const pkg of packages) { +for (const pkg of [...privateBuildPackages, ...packages]) { run("npm", ["run", "clean"], { cwd: pkg.directory }); run("npm", ["run", pkg.directory === "packages/ai" ? "build:offline" : "build"], { cwd: pkg.directory }); } diff --git a/tsconfig.json b/tsconfig.json index aaebe66d1..b44378218 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,7 @@ "moduleResolution": "NodeNext", "paths": { "*": ["./*"], + "#harness-trace": ["./packages/harness-trace/src/index.ts"], "@earendil-works/diffex-harness-state": ["./packages/harness-state/src/index.ts"], "@earendil-works/diffex-harness-trace": ["./packages/harness-trace/src/index.ts"], "@earendil-works/diffex-telemetry": ["./packages/telemetry/src/index.ts"], diff --git a/vitest.base.ts b/vitest.base.ts index b4916fee8..2939974e0 100644 --- a/vitest.base.ts +++ b/vitest.base.ts @@ -10,6 +10,7 @@ export const workspaceSourcePaths = { aiProviders: fileURLToPath(new URL("./packages/ai/src/providers", import.meta.url)), agentIndex: fileURLToPath(new URL("./packages/agent/src/index.ts", import.meta.url)), harnessStateIndex: fileURLToPath(new URL("./packages/harness-state/src/index.ts", import.meta.url)), + harnessTraceIndex: fileURLToPath(new URL("./packages/harness-trace/src/index.ts", import.meta.url)), codingAgentIndex: fileURLToPath(new URL("./packages/coding-agent/src/index.ts", import.meta.url)), tuiIndex: fileURLToPath(new URL("./packages/tui/src/index.ts", import.meta.url)), } as const; @@ -17,6 +18,7 @@ export const workspaceSourcePaths = { export default defineConfig({ resolve: { alias: [ + { find: /^#harness-trace$/, replacement: workspaceSourcePaths.harnessTraceIndex }, { find: /^@earendil-works\/diffex-telemetry$/, replacement: workspaceSourcePaths.telemetryIndex }, { find: /^@earendil-works\/diffex-telemetry\/testing$/, replacement: workspaceSourcePaths.telemetryTesting }, { find: /^@earendil-works\/diffex-ai$/, replacement: workspaceSourcePaths.aiIndex }, @@ -28,6 +30,7 @@ export default defineConfig({ }, { find: /^@earendil-works\/diffex-agent-core$/, replacement: workspaceSourcePaths.agentIndex }, { find: /^@earendil-works\/diffex-harness-state$/, replacement: workspaceSourcePaths.harnessStateIndex }, + { find: /^@earendil-works\/diffex-harness-trace$/, replacement: workspaceSourcePaths.harnessTraceIndex }, { find: /^@earendil-works\/diffex-tui$/, replacement: workspaceSourcePaths.tuiIndex }, ], }, From 00d40608c5ded800a6c8ab491380bc09875e7374 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Tue, 18 Aug 2026 19:29:38 -0700 Subject: [PATCH 3/3] test(coding-agent): cover trace lifecycle - Verify persisted, resumed, replaced, failed, and aborted invocations - Protect ephemeral and degraded recorder behavior --- .../test/invocation-tracing.test.ts | 332 ++++++++++++++++++ .../agent-session-harness-version.test.ts | 25 ++ packages/coding-agent/test/suite/harness.ts | 3 + 3 files changed, 360 insertions(+) create mode 100644 packages/coding-agent/test/invocation-tracing.test.ts diff --git a/packages/coding-agent/test/invocation-tracing.test.ts b/packages/coding-agent/test/invocation-tracing.test.ts new file mode 100644 index 000000000..3473dff93 --- /dev/null +++ b/packages/coding-agent/test/invocation-tracing.test.ts @@ -0,0 +1,332 @@ +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { mkdir, open } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { + readJsonlTraceFile, + type TraceFooterRecord, + type TraceRecord, + type TraceSessionAttachRecord, +} from "@earendil-works/diffex-harness-trace"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ENV_AGENT_DIR } from "../src/config.ts"; +import { createInvocationTrace } from "../src/core/invocation-trace.ts"; +import { SessionManager } from "../src/core/session-manager.ts"; + +const cliPath = resolve(__dirname, "../src/cli.ts"); +const tsconfigPath = resolve(__dirname, "../../../tsconfig.json"); +const FAUX_EXTENSION_SOURCE = ` +import { fauxAssistantMessage, registerFauxProvider } from "@earendil-works/diffex-ai/compat"; + +const faux = registerFauxProvider(); +faux.setResponses([ + process.env.DIFFEX_TRACE_FAUX_FAILURE === "1" + ? fauxAssistantMessage("", { + stopReason: "error", + errorMessage: "intentional tracing test failure", + }) + : fauxAssistantMessage("first response"), + fauxAssistantMessage("replacement response"), +]); + +export default function (diffex) { + const model = faux.getModel(); + diffex.registerProvider(model.provider, { + baseUrl: model.baseUrl, + apiKey: "faux-key", + api: faux.api, + models: faux.models.map((registeredModel) => ({ + id: registeredModel.id, + name: registeredModel.name, + api: registeredModel.api, + reasoning: registeredModel.reasoning, + input: registeredModel.input, + cost: registeredModel.cost, + contextWindow: registeredModel.contextWindow, + maxTokens: registeredModel.maxTokens, + baseUrl: registeredModel.baseUrl, + })), + }); + diffex.registerCommand("replace-trace-session", { + description: "Replace the active session during tracing tests", + handler: async (_args, context) => { + await context.newSession({ + withSession: async (replacement) => { + await replacement.sendUserMessage("replacement prompt"); + }, + }); + }, + }); +} +`; + +interface TestDirectories { + root: string; + agentDir: string; + projectDir: string; + extensionPath: string; +} + +interface CliResult { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +} + +const tempDirectories: string[] = []; +const activeChildren = new Set(); + +afterEach(() => { + for (const child of activeChildren) child.kill("SIGKILL"); + activeChildren.clear(); + for (const directory of tempDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } + vi.restoreAllMocks(); +}); + +function setup(): TestDirectories { + const root = mkdtempSync(join(tmpdir(), "diffex-invocation-trace-")); + tempDirectories.push(root); + const agentDir = join(root, "agent"); + const projectDir = join(root, "project"); + const extensionPath = join(root, "faux-extension.ts"); + mkdirSync(agentDir, { recursive: true }); + mkdirSync(projectDir, { recursive: true }); + writeFileSync(extensionPath, FAUX_EXTENSION_SOURCE); + return { root, agentDir, projectDir, extensionPath }; +} + +function commonArgs(directories: TestDirectories): string[] { + return ["-e", directories.extensionPath, "--model", "faux/faux-1"]; +} + +function startCli( + directories: TestDirectories, + args: string[], + extraEnv?: NodeJS.ProcessEnv, +): ChildProcessWithoutNullStreams { + const child = spawn(process.execPath, [`--import=${import.meta.resolve("tsx")}`, cliPath, ...args], { + cwd: directories.projectDir, + env: { + ...process.env, + [ENV_AGENT_DIR]: directories.agentDir, + DIFFEX_OFFLINE: "1", + TSX_TSCONFIG_PATH: tsconfigPath, + ...extraEnv, + }, + stdio: ["pipe", "pipe", "pipe"], + }); + activeChildren.add(child); + return child; +} + +async function collectCli(child: ChildProcessWithoutNullStreams): Promise { + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + + return await new Promise((resolvePromise, reject) => { + const timeout = setTimeout(() => child.kill("SIGKILL"), 15_000); + child.on("error", (error) => { + clearTimeout(timeout); + activeChildren.delete(child); + reject(error); + }); + child.on("close", (code, signal) => { + clearTimeout(timeout); + activeChildren.delete(child); + resolvePromise({ code, signal, stdout, stderr }); + }); + }); +} + +async function runCli(directories: TestDirectories, args: string[], extraEnv?: NodeJS.ProcessEnv): Promise { + const child = startCli(directories, args, extraEnv); + const resultPromise = collectCli(child); + child.stdin.end(); + return resultPromise; +} + +function tracePaths(agentDir: string): string[] { + const directory = join(agentDir, "traces"); + try { + return readdirSync(directory) + .filter((name) => name.endsWith(".jsonl")) + .sort() + .map((name) => join(directory, name)); + } catch { + return []; + } +} + +function onlyTracePath(agentDir: string): string { + const paths = tracePaths(agentDir); + expect(paths).toHaveLength(1); + const [path] = paths; + if (!path) throw new Error("Invocation trace file was not found"); + return path; +} + +async function readTrace(path: string): Promise { + const result = await readJsonlTraceFile(path); + expect(result.diagnostics).toEqual([]); + return result.records; +} + +function sessionFile(agentDir: string): string { + const sessionsRoot = join(agentDir, "sessions"); + for (const directoryName of readdirSync(sessionsRoot)) { + const directory = join(sessionsRoot, directoryName); + for (const fileName of readdirSync(directory)) { + if (fileName.endsWith(".jsonl")) return join(directory, fileName); + } + } + throw new Error("Persisted session file was not found"); +} + +function attachments(records: readonly TraceRecord[]): readonly TraceSessionAttachRecord[] { + return records.filter((record): record is TraceSessionAttachRecord => record.type === "session_attach"); +} + +function footer(records: readonly TraceRecord[]): TraceFooterRecord | undefined { + return records.find((record): record is TraceFooterRecord => record.type === "trace_footer"); +} + +describe("CLI invocation tracing", () => { + it("records completed and resumed persisted invocations", async () => { + const directories = setup(); + const first = await runCli(directories, [...commonArgs(directories), "-p", "hello"]); + expect(first).toMatchObject({ code: 0, signal: null, stdout: "first response\n", stderr: "" }); + + const firstTracePath = onlyTracePath(directories.agentDir); + const firstRecords = await readTrace(firstTracePath); + expect(attachments(firstRecords)).toHaveLength(1); + expect(attachments(firstRecords)[0]).toMatchObject({ reason: "startup", sessionFormat: "coding-agent" }); + expect(firstRecords.some((record) => record.type === "harness_activate")).toBe(true); + expect(firstRecords.some((record) => record.type === "session_segment")).toBe(true); + expect(footer(firstRecords)?.terminationStatus).toBe("completed"); + + if (process.platform !== "win32") { + expect(statSync(join(directories.agentDir, "traces")).mode & 0o777).toBe(0o700); + expect(statSync(firstTracePath).mode & 0o777).toBe(0o600); + } + + const persistedSession = sessionFile(directories.agentDir); + const resumed = await runCli(directories, [ + ...commonArgs(directories), + "--session", + persistedSession, + "-p", + "again", + ]); + expect(resumed.code).toBe(0); + + const allTracePaths = tracePaths(directories.agentDir); + expect(allTracePaths).toHaveLength(2); + const resumedRecords = await readTrace(allTracePaths[1]); + expect(attachments(resumedRecords)[0]).toMatchObject({ reason: "resume", path: persistedSession }); + expect(footer(resumedRecords)?.terminationStatus).toBe("completed"); + }); + + it("attaches a new binding when the active session is replaced", async () => { + const directories = setup(); + const result = await runCli(directories, [...commonArgs(directories), "-p", "/replace-trace-session"]); + expect(result.code).toBe(0); + + const records = await readTrace(onlyTracePath(directories.agentDir)); + expect(attachments(records).map((attachment) => attachment.reason)).toEqual(["startup", "new"]); + expect(new Set(attachments(records).map((attachment) => attachment.bindingId)).size).toBe(2); + expect(footer(records)?.terminationStatus).toBe("completed"); + }); + + it("does not trace explicitly ephemeral commands", async () => { + const directories = setup(); + expect((await runCli(directories, ["--help"])).code).toBe(0); + expect((await runCli(directories, [...commonArgs(directories), "--list-models"])).code).toBe(0); + expect((await runCli(directories, [...commonArgs(directories), "--no-session", "-p", "hello"])).code).toBe(0); + expect(tracePaths(directories.agentDir)).toEqual([]); + }); + + it("continues after trace creation fails and reports one warning", async () => { + const directories = setup(); + writeFileSync(join(directories.agentDir, "traces"), "not a directory"); + + const result = await runCli(directories, [...commonArgs(directories), "-p", "hello"]); + + expect(result.code).toBe(0); + expect(result.stdout).toBe("first response\n"); + expect(result.stderr.match(/Invocation tracing is unavailable/g)).toHaveLength(1); + expect(tracePaths(directories.agentDir)).toEqual([]); + }); + + it("marks a failed invocation as failed", async () => { + const directories = setup(); + const result = await runCli(directories, [...commonArgs(directories), "-p", "fail"], { + DIFFEX_TRACE_FAUX_FAILURE: "1", + }); + + expect(result.code).toBe(1); + expect(result.stderr).toContain("intentional tracing test failure"); + const records = await readTrace(onlyTracePath(directories.agentDir)); + expect(footer(records)?.terminationStatus).toBe("failed"); + }); + + it.each([ + { signal: "SIGTERM" as const, exitCode: 143 }, + ...(process.platform === "win32" ? [] : [{ signal: "SIGHUP" as const, exitCode: 129 }]), + ])("marks $signal-driven RPC shutdown as aborted", async ({ signal, exitCode }) => { + const directories = setup(); + const child = startCli(directories, [...commonArgs(directories), "--mode", "rpc"]); + const resultPromise = collectCli(child); + await vi.waitFor( + () => { + const path = tracePaths(directories.agentDir)[0]; + expect(path).toBeDefined(); + expect(readFileSync(path, "utf8")).toContain('"harness_activate"'); + }, + { timeout: 10_000, interval: 20 }, + ); + + child.kill(signal); + const result = await resultPromise; + expect(result.code).toBe(exitCode); + expect(result.signal).toBeNull(); + + const records = await readTrace(onlyTracePath(directories.agentDir)); + expect(footer(records)?.terminationStatus).toBe("aborted"); + }); + + it("keeps runtime behavior passive after a trace writer failure", async () => { + const directories = setup(); + const manager = SessionManager.create(directories.projectDir, join(directories.agentDir, "sessions")); + const { trace } = await createInvocationTrace({ agentDir: directories.agentDir, sessionManager: manager }); + const probePath = join(directories.root, "probe", "writer.jsonl"); + await mkdir(dirname(probePath), { recursive: true }); + const probe = await open(probePath, "wx"); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { writeFile: typeof probe.writeFile }; + await probe.close(); + const writeFailure = vi + .spyOn(fileHandlePrototype, "writeFile") + .mockRejectedValue(new Error("simulated trace write failure")); + + await expect(trace.attachSession(manager, "startup")).resolves.toBeUndefined(); + manager.appendSessionInfo("writer survived"); + await expect( + trace.operationSettled({ operationId: "operation", sessionManager: manager, firstEntryOrdinal: 0 }), + ).resolves.toBeUndefined(); + writeFailure.mockRestore(); + await trace.close("completed"); + + const result = await readJsonlTraceFile(trace.destinationPath!); + expect(result.records.map((record) => record.type)).toEqual(["trace_header"]); + expect(result.diagnostics.map((diagnostic) => diagnostic.code)).toContain("missing_footer"); + }); +}); diff --git a/packages/coding-agent/test/suite/agent-session-harness-version.test.ts b/packages/coding-agent/test/suite/agent-session-harness-version.test.ts index 5ba58b734..c9a9f6f4a 100644 --- a/packages/coding-agent/test/suite/agent-session-harness-version.test.ts +++ b/packages/coding-agent/test/suite/agent-session-harness-version.test.ts @@ -72,6 +72,31 @@ describe("AgentSession harness revisions", () => { ); }); + it("reports a live activation to the process lifecycle observer", async () => { + const activatedVersions: HarnessVersion[] = []; + const agentDir = join(tmpdir(), `diffex-harness-observer-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(agentDir, { recursive: true }); + agentDirs.push(agentDir); + const store = new HarnessStateStore(agentDir); + const root = await store.initialize(); + const successor = await store.saveVersion({ memory: "Observed memory." }, root.manifest.version); + const harness = await createHarness({ + harnessStateStore: store, + harnessVersionBundle: root, + lifecycleObserver: { + operationSettled: () => {}, + harnessActivated: (version) => { + activatedVersions.push(version); + }, + }, + }); + harnesses.push(harness); + + await harness.session.switchHarnessVersion(successor.manifest.version); + + expect(activatedVersions).toEqual([successor.manifest.version]); + }); + it.each([ { property: "isStreaming", message: "while a response is streaming" }, { property: "isCompacting", message: "while compaction is in progress" }, diff --git a/packages/coding-agent/test/suite/harness.ts b/packages/coding-agent/test/suite/harness.ts index 2523ae3dd..d59b31e5e 100644 --- a/packages/coding-agent/test/suite/harness.ts +++ b/packages/coding-agent/test/suite/harness.ts @@ -17,6 +17,7 @@ import type { import { registerFauxProvider, streamSimple } from "@earendil-works/diffex-ai/compat"; import type { HarnessStateStore, HarnessVersionBundle } from "@earendil-works/diffex-harness-state"; import { AgentSession, type AgentSessionEvent } from "../../src/core/agent-session.ts"; +import type { AgentSessionLifecycleObserver } from "../../src/core/agent-session-lifecycle.ts"; import { AuthStorage } from "../../src/core/auth-storage.ts"; import type { ExtensionRunner, SessionStartEvent } from "../../src/core/extensions/index.ts"; import { convertToLlm } from "../../src/core/messages.ts"; @@ -78,6 +79,7 @@ export interface HarnessOptions { harnessStateStore?: HarnessStateStore; harnessVersionBundle?: HarnessVersionBundle; telemetryContext?: TelemetryContext; + lifecycleObserver?: AgentSessionLifecycleObserver; } export interface Harness { @@ -202,6 +204,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise