diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 08b98b13e..664f5acda 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -158,7 +158,9 @@ describe("the mutating-tool table", () => { "addCameraFullscreen", "addSpeed", "addTrim", + "addTrims", "addZoom", + "addZooms", "moveClip", "removeClip", "removeModifier", @@ -227,6 +229,189 @@ describe("executeAgentTool", () => { expect(result.summary).toMatch(/added trim 0:20\.0 – 0:22\.0/); }); + it("addTrims lands exactly what the same calls one at a time would", () => { + // The property the batch tools exist to have: they save round trips and + // change nothing else. If this ever diverges, the batch has grown a second + // implementation of the rules and the two will drift. + const ranges = [ + { startSec: 1, endSec: 2, reason: "silence" }, + { startSec: 40, endSec: 41, reason: "silence" }, + { startSec: 5, endSec: 4, reason: "silence" }, // reversed on purpose + ]; + + let oneAtATime = fixtureDocument(); + for (const range of ranges) { + const step = executeAgentTool(oneAtATime, "addTrim", JSON.stringify(range)); + expect(step.ok).toBe(true); + oneAtATime = step.document as AxcutDocument; + } + + const batch = executeAgentTool(fixtureDocument(), "addTrims", JSON.stringify({ ranges })); + expect(batch.ok).toBe(true); + + const shape = (doc: AxcutDocument) => + doc.timeline.trimRanges.map((t) => ({ + startSec: t.startSec, + endSec: t.endSec, + reason: t.reason, + origin: t.origin, + clipId: t.clipId, + })); + expect(shape(batch.document as AxcutDocument)).toEqual(shape(oneAtATime)); + }); + + it("addTrims applies the good ranges and refuses the bad one by itself", () => { + // `replaceTimeline`, the repo's other array-taking tool, refuses in one + // block. That is right for rebuilding a timeline and ruinous here: one bad + // bound must not cost the other nine, and the model must be able to see + // WHICH one without re-reading the document. + const result = executeAgentTool( + fixtureDocument(), + "addTrims", + JSON.stringify({ + ranges: [ + { startSec: 1, endSec: 2 }, + { startSec: 25, endSec: 35 }, // spans both clips of asset_1 — ambiguous + { startSec: 40, endSec: 41 }, + ], + }), + ); + + expect(result.ok).toBe(true); + const payload = JSON.parse(result.resultJson); + expect(payload.requested).toBe(3); + expect(payload.appliedCount).toBe(2); + expect(payload.refusedCount).toBe(1); + expect(payload.refused).toHaveLength(1); + expect(payload.refused[0].index).toBe(1); + // The refusal keeps the unitary wording, which names the clips and the fix. + expect(payload.refused[0].error).toMatch(/clipId/); + expect(payload.applied.map((a: { index: number }) => a.index)).toEqual([0, 2]); + // The fixture starts with one trim; two more landed. + expect(result.document?.timeline.trimRanges).toHaveLength(3); + expect(result.summary).toMatch(/added 2 trims, 1 refused/); + }); + + it("addTrims refuses a MALFORMED range by itself, not the whole call", () => { + // The batch schema advertises the element shape without enforcing it, so a + // bad entry reaches the unitary executor and is refused at its index. If it + // were enforced at the container, one typo would cost every other cut — + // which is precisely what `applyBatch` says it exists to prevent. + const result = executeAgentTool( + fixtureDocument(), + "addTrims", + JSON.stringify({ + ranges: [{ startSec: 1, endSec: 2 }, { startSec: "oops" }, { startSec: 40, endSec: 41 }], + }), + ); + + expect(result.ok).toBe(true); + const payload = JSON.parse(result.resultJson); + expect(payload.appliedCount).toBe(2); + expect(payload.refused).toEqual([{ index: 1, error: expect.stringMatching(/endSec/) }]); + expect(result.document?.timeline.trimRanges).toHaveLength(3); + }); + + it("addZooms refuses a MALFORMED region by itself, not the whole call", () => { + const result = executeAgentTool( + fixtureDocument(), + "addZooms", + JSON.stringify({ + regions: [ + { startSec: 1, endSec: 3, depth: 9 }, // depth is an ordinal 1–6 + { startSec: 10, endSec: 12 }, + ], + }), + ); + + expect(result.ok).toBe(true); + const payload = JSON.parse(result.resultJson); + expect(payload.appliedCount).toBe(1); + expect(payload.refused[0].index).toBe(0); + expect(result.document?.zoomRanges).toHaveLength(1); + }); + + it("addTrims still refuses a batch that is not a non-empty list", () => { + for (const args of ['{"ranges":[]}', '{"ranges":"1-2"}', "{}"]) { + const result = executeAgentTool(fixtureDocument(), "addTrims", args); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + } + }); + + it("addTrims reports a whole-batch refusal as a failure, not an empty success", () => { + const result = executeAgentTool( + fixtureDocument(), + "addTrims", + JSON.stringify({ + ranges: [ + { startSec: 1, endSec: 2, assetId: "asset_missing" }, + { startSec: 3, endSec: 4, assetId: "asset_missing" }, + ], + }), + ); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + const error = JSON.parse(result.resultJson).error; + expect(error).toMatch(/\[0\]/); + expect(error).toMatch(/\[1\]/); + expect(error).toMatch(/Nothing was modified/); + }); + + it("addZooms lands the reachable regions and names the one covering no clip", () => { + const result = executeAgentTool( + fixtureDocument(), + "addZooms", + JSON.stringify({ + regions: [ + { startSec: 1, endSec: 3, depth: 2 }, + { startSec: 400, endSec: 402 }, // past the end of the timeline + { startSec: 10, endSec: 12, depth: 4 }, + ], + }), + ); + + expect(result.ok).toBe(true); + const payload = JSON.parse(result.resultJson); + expect(payload.appliedCount).toBe(2); + expect(payload.refused[0].index).toBe(1); + // Each applied entry still carries what the unitary tool reports, so the + // model can quote the rendered scale instead of the depth ordinal. + expect(payload.applied[0].renderedScale).toBe(ZOOM_DEPTH_SCALES[2]); + expect(payload.applied[1].renderedScale).toBe(ZOOM_DEPTH_SCALES[4]); + expect(result.document?.zoomRanges).toHaveLength(2); + }); + + it("addZooms leaves overlapping regions overlapping, exactly as one-at-a-time does", () => { + // A deliberate non-decision, pinned so it stays deliberate. + // + // `timelineMap.ts` forbids two zooms of different identities from + // overlapping, but only the `set*` path clamps (via `replacePillSpan`) — + // no `add*` does, in the agent OR in the UI. So two overlapping addZoom + // calls already produce an overlapping document today. Deconflicting + // inside the batch would make `addZooms` mean something its unitary + // sibling does not, and the model would get different results depending on + // how it chose to group its calls. The batch saves round trips; it does + // not quietly hold different rules. The bench still flags the overlap + // (`editorial.ts` zoomIssues), which is where that argument belongs. + const regions = [ + { startSec: 1, endSec: 6 }, + { startSec: 4, endSec: 9 }, + ]; + + let oneAtATime = fixtureDocument(); + for (const region of regions) { + oneAtATime = executeAgentTool(oneAtATime, "addZoom", JSON.stringify(region)) + .document as AxcutDocument; + } + const batch = executeAgentTool(fixtureDocument(), "addZooms", JSON.stringify({ regions })); + + const spans = (doc: AxcutDocument) => + doc.zoomRanges.map((z) => ({ startMs: z.startMs, endMs: z.endMs, depth: z.depth })); + expect(spans(batch.document as AxcutDocument)).toEqual(spans(oneAtATime)); + expect(batch.document?.zoomRanges).toHaveLength(2); + }); + it("addTrim rejects unknown assets", () => { const result = executeAgentTool( fixtureDocument(), diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 4118daed5..b630e12d1 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -348,6 +348,32 @@ export const addTrimArgs = z.object({ reason: z.string().default(""), }); +/** + * ponytail: the element schema is `addTrimArgs` itself, not a copy of it. + * + * A batch is N unitary calls and nothing else — same validation, same clip + * resolution, same refusal wording — so the two can never drift into meaning + * different things. A separate element schema would be one more place to forget + * `clipId` the next time the unitary one grows a field. + * + * The `union(…, unknown)` is what makes "each item stands or falls alone" true for + * MALFORMED items too, not just unplaceable ones. A bare `z.array(addTrimArgs)` + * rejects the whole call the moment one entry is bad — and it rejects it in + * LangChain, before `applyBatch` runs — so nine good cuts would be thrown away + * with the tenth and `refused[index]` could never name it. Advertising the + * union keeps the element shape in the JSON schema the model reads (it shows up + * as `anyOf: [addTrim, {}]`) while letting a bad entry through to the unitary + * executor, which refuses it by itself with the wording it always uses. + * + * No cap on the array. A half-hour recording has hundreds of silences, and the + * point of this tool is precisely that it should not have to guess how many are + * too many. Picking a number here would repeat the mistake `getTranscript` made + * with its 800. + */ +export const addTrimsArgs = z.object({ + ranges: z.array(z.union([addTrimArgs, z.unknown()])).min(1), +}); + export const setTrimArgs = z.object({ trimRangeId: z.string().min(1), startSec: secondsSchema, @@ -410,6 +436,12 @@ export const addZoomArgs = z.object({ focus: focusSchema.default({ cx: 0.5, cy: 0.5 }), }); +/** Same contract as `addTrimsArgs`: the element schema IS the unitary one, and + * it is advertised rather than enforced so a bad region is refused by itself. */ +export const addZoomsArgs = z.object({ + regions: z.array(z.union([addZoomArgs, z.unknown()])).min(1), +}); + export const setZoomArgs = z.object({ zoomId: z.string().min(1), startSec: secondsSchema.optional(), @@ -486,6 +518,8 @@ export const removeClipArgs = z.object({ */ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "addTrim", + "addTrims", + "addZooms", "setTrim", "setClipRange", "moveClip", @@ -654,6 +688,79 @@ function failure(message: string): AgentToolExecution { return { ok: false, resultJson: JSON.stringify({ error: message }) }; } +/** + * Runs `unitName` once per item, folding the document forward. + * + * ponytail: the batch tools exist to save ROUND TRIPS, not to mean something new. + * Replaying the unitary executor is what guarantees that — anchoring, clip + * resolution, clamping, the wording of every refusal, all identical by + * construction rather than by a second implementation staying in step. A batch + * of N is exactly N unitary calls minus N-1 round trips, and `agent-tools.test` + * asserts that against a document built the long way. + * + * ponytail: PARTIAL application, deliberately. `replaceTimeline` is the repo's + * other array-taking tool and it refuses in one block — "Refused … Nothing was + * modified" — which is right for a tool that rebuilds the whole timeline and + * ruinous for one that adds ten independent cuts: a single bad bound would throw + * away nine good ones and the model would have to guess which. So each item + * stands or falls alone, and the result says which did what. `ok:false` is kept + * for the case where NOTHING landed, because that is the only one where the + * document did not move. + */ +function applyBatch( + document: AxcutDocument, + unitName: "addTrim" | "addZoom", + items: unknown[], + options: AgentToolOptions | undefined, + noun: string, +): AgentToolExecution { + let current = document; + const applied: Array> = []; + const refused: Array<{ index: number; error: string }> = []; + + items.forEach((item, index) => { + const execution = executeAgentTool(current, unitName, JSON.stringify(item), options); + let payload: Record = {}; + try { + payload = JSON.parse(execution.resultJson) as Record; + } catch { + payload = { error: execution.resultJson }; + } + if (execution.ok && execution.document) { + current = execution.document; + applied.push({ index, ...payload }); + } else { + refused.push({ index, error: String(payload.error ?? "refused") }); + } + }); + + // Nothing landed: the document is untouched, so say so the way every other + // refusal does rather than reporting a success with an empty list. + if (applied.length === 0) { + return failure( + `No ${noun} was added. ` + + refused.map((r) => `[${r.index}] ${r.error}`).join(" | ") + + " Nothing was modified.", + ); + } + + const refusedSuffix = refused.length ? `, ${refused.length} refused` : ""; + return { + ok: true, + document: current, + // The counts come first on purpose: the model must be able to see that one + // of ten was refused WITHOUT re-reading the document, and know which one. + resultJson: JSON.stringify({ + requested: items.length, + appliedCount: applied.length, + refusedCount: refused.length, + applied, + ...(refused.length ? { refused } : {}), + }), + summary: `added ${applied.length} ${noun}${applied.length === 1 ? "" : "s"}${refusedSuffix}`, + }; +} + /** The clips as the model would have to name them, for an error about an id it * got wrong — a bare "Unknown clip: demo" leaves it guessing twice. */ function clipRoster(document: AxcutDocument): string { @@ -932,6 +1039,12 @@ export function executeAgentTool( }; } + case "addTrims": { + const parsed = addTrimsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + return applyBatch(document, "addTrim", parsed.data.ranges, options, "trim"); + } + case "setTrim": { const parsed = setTrimArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -1167,6 +1280,12 @@ export function executeAgentTool( }; } + case "addZooms": { + const parsed = addZoomsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + return applyBatch(document, "addZoom", parsed.data.regions, options, "zoom"); + } + case "setZoom": { const parsed = setZoomArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index ec7770ac7..43e963c22 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -42,11 +42,13 @@ const OPENSCREEN_TOOLS = [ "getTranscript", "getCursorTrack", "addTrim", + "addTrims", "setTrim", "setClipRange", "moveClip", "replaceTimeline", "addZoom", + "addZooms", "setZoom", "addSpeed", "setSpeed", @@ -84,11 +86,13 @@ const ARGS: Record = { getTranscript: {}, getCursorTrack: {}, addTrim: { startSec: 1, endSec: 2 }, + addTrims: { ranges: [{ startSec: 1, endSec: 2 }] }, setTrim: { trimRangeId: "trim_1", startSec: 1, endSec: 2 }, setClipRange: { clipId: "clip_1", sourceStartSec: 0, sourceEndSec: 10 }, moveClip: { clipId: "clip_1", beforeClipId: null }, replaceTimeline: { intervals: [{ startSec: 0, endSec: 10 }] }, addZoom: { startSec: 1, endSec: 2 }, + addZooms: { regions: [{ startSec: 1, endSec: 2 }] }, setZoom: { zoomId: "zoom_nope" }, addSpeed: { startSec: 1, endSec: 2 }, setSpeed: { speedId: "speed_nope" }, @@ -171,7 +175,7 @@ function recordingSink(): { sink: OpenScreenAgentSink; events: SinkEvent[] } { } /** `buildTools` returns a tuple with a DISTINCT type per tool, one per zod - * schema, so `tools.find(...)` is a 19-way union — and `.invoke` is generic, a + * schema, so `tools.find(...)` is a 21-way union — and `.invoke` is generic, a * shape TypeScript will not call through a union (TS2349). Widening to the * interface every one of them implements is what the model is handed anyway: * `createAgent` takes them as `ClientTool`, i.e. exactly this. Nothing the @@ -187,7 +191,7 @@ function toolsFor(document: AxcutDocument) { } describe("the tool surface handed to the model", () => { - it("is exactly OpenScreen's 19 tools", () => { + it("is exactly OpenScreen's 21 tools", () => { const { tools } = toolsFor(fixtureDocument()); expect(tools.map((t) => t.name)).toEqual(OPENSCREEN_TOOLS); }); @@ -288,6 +292,23 @@ describe("the sink announces each call exactly once, with the real verdict", () expect(events[1]).toMatchObject({ kind: "toolEnd", name: "getTranscript", ok: false }); }); + it("lets a malformed batch entry reach the executor instead of throwing at the schema", async () => { + // LangChain parses the tool's schema BEFORE calling us, so a batch schema + // that enforced its element shape would reject the whole call here — the + // per-item `refused[index]` that `addTrims` promises the model could never + // happen on the product path, only in a direct-executor test. + const { tools, holder } = toolsFor(fixtureDocument()); + const tool = tools.find((t) => t.name === "addTrims"); + if (!tool) throw new Error("addTrims is not built"); + + const result = JSON.parse( + String(await tool.invoke({ ranges: [{ startSec: 1, endSec: 2 }, { startSec: "oops" }] })), + ); + expect(result.appliedCount).toBe(1); + expect(result.refused[0].index).toBe(1); + expect(holder.current.timeline.trimRanges).toHaveLength(2); + }); + it("advances the holder on a write, and leaves it alone on a refusal", async () => { const { tools, holder } = toolsFor(fixtureDocument()); const before = holder.current; @@ -398,7 +419,7 @@ describe("the prompt when the user has turned project edits off", () => { }); describe("the tools when the user has turned project edits off", () => { - it("still builds all 18 — the model has to be able to NAME the edit", () => { + it("still builds all 21 — the model has to be able to NAME the edit", () => { const { sink } = recordingSink(); const tools: BuiltTool[] = buildTools({ current: fixtureDocument() }, sink, false); expect(tools.map((t) => t.name)).toEqual(OPENSCREEN_TOOLS); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index bc10c415c..e025827e1 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -30,7 +30,9 @@ import { addCameraFullscreenArgs, addSpeedArgs, addTrimArgs, + addTrimsArgs, addZoomArgs, + addZoomsArgs, type CursorTelemetryLoad, executeAgentTool, getCursorTrackArgs, @@ -110,7 +112,7 @@ const BASE_SYSTEM_PROMPT = [ // happened to list and silently misses every paraphrase — and every language // other than English. Say what the tool does; let the model do the matching. "How the tools map to intent — pick the most specific one, and prefer the smallest edit that satisfies the request:", - "- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip: one addTrim per range. The placed clip stays the canonical cut; it is not rebuilt to drop them.", + "- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text on screen. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", @@ -142,7 +144,9 @@ export const TOOL_DESCRIPTIONS: Record = { getCursorTrack: "Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.", addTrim: - "Add a trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. Call this once per silent range. A cut belongs to ONE clip: `clipId` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the `clipId` you mean (ids come from getCurrentDocument).", + "Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: `clipId` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the `clipId` you mean (ids come from getCurrentDocument).", + addTrims: + "Add MANY trim ranges in one call: `ranges` is a list, each entry taking exactly the fields addTrim takes. Use this whenever you have more than one cut to make — 'remove the silences' on a half-hour recording is hundreds of cuts, and sending them one at a time costs one round trip each. Each range stands or falls ALONE: one that cannot be placed is refused by itself and listed in `refused` with its index and the reason, while every other range is still applied. Nothing is rolled back, so a single bad bound never costs you the rest. The result leads with requested / appliedCount / refusedCount so you can see a partial outcome without re-reading the document — report what was refused rather than claiming the whole list landed.", setTrim: "Move or resize an existing trim range by id. Times are source-time seconds. The cut follows to whichever clip the new range lands in, when that clip is unambiguous.", setClipRange: @@ -152,6 +156,7 @@ export const TOOL_DESCRIPTIONS: Record = { replaceTimeline: "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.", addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). Use for 'zoom in on …' and the smart-zoom pass.`, + addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth.`, setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale.`, addSpeed: "Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down (default 1.5×). Use to speed through slow stretches without cutting them.", @@ -298,11 +303,13 @@ export function buildTools( build("getTranscript", getTranscriptArgs), build("getCursorTrack", getCursorTrackArgs), build("addTrim", addTrimArgs), + build("addTrims", addTrimsArgs), build("setTrim", setTrimArgs), build("setClipRange", setClipRangeArgs), build("moveClip", moveClipArgs), build("replaceTimeline", replaceTimelineArgs), build("addZoom", addZoomArgs), + build("addZooms", addZoomsArgs), build("setZoom", setZoomArgs), build("addSpeed", addSpeedArgs), build("setSpeed", setSpeedArgs), diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.md index 47634129d..d87e642fd 100644 --- a/technical-documentation/architecture/ai-agent.md +++ b/technical-documentation/architecture/ai-agent.md @@ -41,19 +41,21 @@ The agent is built with LangChain's `createAgent`, not `deepagents`' `createDeep ## Tool schema -The model never free-writes the project document. It can only call the fixed set of 19 tools built by `deep-agent/service.ts#buildTools`, described by `TOOL_DESCRIPTIONS` in the same file and validated against the Zod argument schemas in `agent-tools.ts`; the executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. Those three surfaces (descriptions, built tools, executor cases) are pinned to one another by `deep-agent/service.test.ts` — an earlier fourth surface, `AGENT_TOOL_SPECS`, described the tools in JSON Schema for a provider it had stopped reaching, and drifted. The tools operate on the same [document model](document-model.md) as manual editing. +The model never free-writes the project document. It can only call the fixed set of 21 tools built by `deep-agent/service.ts#buildTools`, described by `TOOL_DESCRIPTIONS` in the same file and validated against the Zod argument schemas in `agent-tools.ts`; the executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. Those three surfaces (descriptions, built tools, executor cases) are pinned to one another by `deep-agent/service.test.ts` — an earlier fourth surface, `AGENT_TOOL_SPECS`, described the tools in JSON Schema for a provider it had stopped reaching, and drifted. The tools operate on the same [document model](document-model.md) as manual editing. | Tool | What it does | What it mutates | |---|---|---| | `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert. | Nothing. | | `getTranscript` | Reads up to 800 transcript segments for an asset or the primary asset. | Nothing. | | `getCursorTrack` | Reads the recorded pointer telemetry for an asset as a DIGEST: the moments the cursor sat still or clicked, each with its hold, its average position, its click count, its source time and the `virtualSec` that `addZoom` takes — never the raw samples. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. | -| `addTrim` | Adds a source-time cut inside a clip. | `timeline.trimRanges`. | +| `addTrim` | Adds one source-time cut inside a clip. | `timeline.trimRanges`. | +| `addTrims` | Adds many cuts in one call, replaying `addTrim` per entry so the rules cannot drift apart. Each range stands alone: one that cannot be placed is refused by itself and named with its index and reason while the rest are applied, and the result leads with `requested` / `appliedCount` / `refusedCount`. Only a batch where nothing landed is an error. | `timeline.trimRanges`. | | `setTrim` | Moves or resizes an existing source-time trim. | The matching `timeline.trimRanges` entry. | | `setClipRange` | Changes a clip's source in/out points and relays clips back-to-back. | The clip range and any anchored regions clamped or removed by the shared timeline mutator. | | `moveClip` | Reorders a placed clip by naming the clip it should play before (`null` = last). Preserves every clip id, source range, trim and anchored modifier. | Timeline clip order; anchored modifiers' derived ms follow their clip. | | `replaceTimeline` | Rebuilds the primary-asset timeline from kept source-time intervals. Preserves the id, origin and label of every clip an interval reproduces exactly, carries existing trims through, and never touches another asset's trims. Refused when it would merge away, shorten or drop a clip, or when the intervals are not ascending (a reorder it cannot perform — the refusal points at `moveClip`). | Timeline clips and trim ranges. | | `addZoom` | Adds a clip-anchored zoom over virtual timeline time. `depth` is an ordinal selecting from `ZOOM_DEPTH_SCALES` (1.25×–5.0×, non-linear); the result reports the resulting `renderedScale`. | `zoomRanges`. | +| `addZooms` | Adds many zooms in one call, replaying `addZoom` per entry, with the same per-entry refusal and reporting contract as `addTrims`. | `zoomRanges`. | | `setZoom` | Moves, resizes, or restyles a zoom pill. Changing `depth` clears any `customScale` on that pill — otherwise the write is a no-op at render — and says so in the result. | The clip-anchored `zoomRanges` fragments represented by that pill. | | `addSpeed` | Adds a clip-anchored speed region over virtual timeline time. | `legacyEditor.speedRegions`. | | `setSpeed` | Moves, resizes, or changes an existing speed pill. | The corresponding `legacyEditor.speedRegions` fragments. | diff --git a/workbench/l0/oracles.wb.ts b/workbench/l0/oracles.wb.ts index a67b70e32..879b1c89a 100644 --- a/workbench/l0/oracles.wb.ts +++ b/workbench/l0/oracles.wb.ts @@ -109,6 +109,36 @@ describe("diffMatches", () => { expect(diffMatches(document, call)).toBe(true); }); + it("vérifie CHAQUE élément d'un appel par lot, pas seulement l'enveloppe", () => { + // Un lot ne porte aucun id au premier niveau : ils sont dans `applied`. + // Sans la branche qui les lit, ce check rendrait « vrai à vide » sur + // exactement les appels qui écrivent le plus, et il s'éteindrait sans + // qu'un seul test devienne rouge. + const { document, call } = apply(singleClip(), "addTrims", { + ranges: [ + { startSec: 2, endSec: 4 }, + { startSec: 8, endSec: 10 }, + ], + }); + expect(JSON.parse(call.resultJson ?? "{}").appliedCount).toBe(2); + expect(diffMatches(document, call)).toBe(true); + + // Le même appel dont UN élément ment sur ses bornes doit tomber. + const lying: WireCall = { + ...call, + resultJson: JSON.stringify({ + requested: 2, + appliedCount: 2, + refusedCount: 0, + applied: [ + { index: 0, ...JSON.parse(call.resultJson ?? "{}").applied[0] }, + { index: 1, trimRangeId: "trim_nope", startSec: 30, endSec: 40 }, + ], + }), + }; + expect(diffMatches(document, lying)).toBe(false); + }); + it("catches a report about a region the document does not carry", () => { // The shape of DSL-4 and of a silently re-derived-away region: the tool // answers with the bounds it was ASKED for, the document says otherwise. diff --git a/workbench/lib/oracles.ts b/workbench/lib/oracles.ts index eb1ce4e39..9841c5062 100644 --- a/workbench/lib/oracles.ts +++ b/workbench/lib/oracles.ts @@ -289,6 +289,24 @@ export function diffMatches(after: AxcutDocument, call: WireCall): boolean { } catch { return true; } + // ponytail: a batch tool (`addTrims`, `addZooms`) reports one entry per + // element under `applied`, and nothing at the top level this check can + // falsify. Reading only the envelope would return "vacuously true" for + // exactly the calls that write the most — the check would go dark without a + // single test turning red. Every entry has to hold. + const applied = result.applied; + if (Array.isArray(applied)) { + return applied.every((entry) => + entry && typeof entry === "object" + ? claimSurvives(after, entry as Record) + : true, + ); + } + return claimSurvives(after, result); +} + +/** One reported write against the document it claims to have produced. */ +function claimSurvives(after: AxcutDocument, result: Record): boolean { const idKey = ID_KEYS.find((key) => typeof result[key] === "string"); if (!idKey) return true; const id = result[idKey] as string;