From e2bf083f9fcc0c48493b674f83dfd69e5de2e421 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 16 Sep 2026 23:24:18 -0400 Subject: [PATCH 01/24] =?UTF-8?q?feat(#1001):=20AI=20agent=20harness=20?= =?UTF-8?q?=E2=80=94=20AIAgentManager=20state=20machine,=20capability=20re?= =?UTF-8?q?gistry,=20constrained=20tool=20protocol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The orchestration layer above AIChatManager/LLMManager (#1000, epic #818 Track C6): User -> AIAgentManager -> Planner -> Capability router -> Executor -> Observer -> Replan/Finish. - AIAgentTypes: Plan/Step/Observation/Limits — ALL task state lives here, never in the prompt; deterministic summarize() is the final message - AICapabilityRegistry (#1002): ~170 MCP tools grouped into 20 capabilities; the planner sees a 20-line index + full docs only for the routed capabilities and can ask for more ({"need_capabilities"}); docs are generated from the live buildToolsList() schema - constrained protocol (#1003): validateArguments checks/coerces every planned call against the schema before it reaches the server; unknown tools and bad enums never execute - AIAgentManager (#1001): Planning/Executing/Observing/Replanning/ AwaitingConfirmation/Completed/Failed/Cancelled; retry once, then bounded replans; identical failing action twice = stop; cancel in any state; one undo group per task (#1021b); destructive steps pause for Allow / Always allow / Skip unless trusted mode (#1021d); scene summary injected per turn (#1021c); planner + executor are injected interfaces - AIChatManager facade: agentMode (default on) delegates sendMessage to the agent; v1 loop kept behind the toggle; McpToolExecutor wired in setMcpServer; LlmPlannerBackend forwards LLM signals only while pending - QML: agent/ask-trusted toggles, live plan card, confirmation bar, model tip (Qwen 2.5 7B recommended for tool calling, #1021e) - tests: AIAgentManager_test (14 fixture cases: 5 dependent steps + one undo group, retry, replan, repeated failure, cancel, invalid args, unknown tool, confirmations, capability expansion, Q&A, malformed planner, no-model, context injection) + AICapabilityRegistry_test — headless, no LLM; four guards mutation-verified Co-Authored-By: Claude Fable 5.1 --- CLAUDE.md | 1 + README.md | 2 +- qml/AIChatPanel.qml | 242 ++++++++++- src/AIAgentManager.cpp | 673 ++++++++++++++++++++++++++++++ src/AIAgentManager.h | 213 ++++++++++ src/AIAgentManager_test.cpp | 459 ++++++++++++++++++++ src/AIAgentTypes.cpp | 207 +++++++++ src/AIAgentTypes.h | 145 +++++++ src/AICapabilityRegistry.cpp | 414 ++++++++++++++++++ src/AICapabilityRegistry.h | 106 +++++ src/AICapabilityRegistry_test.cpp | 182 ++++++++ src/AIChatManager.cpp | 95 +++++ src/AIChatManager.h | 16 +- src/CMakeLists.txt | 6 + src/LLMManager.cpp | 2 +- src/mainwindow.cpp | 6 + 16 files changed, 2749 insertions(+), 20 deletions(-) create mode 100644 src/AIAgentManager.cpp create mode 100644 src/AIAgentManager.h create mode 100644 src/AIAgentManager_test.cpp create mode 100644 src/AIAgentTypes.cpp create mode 100644 src/AIAgentTypes.h create mode 100644 src/AICapabilityRegistry.cpp create mode 100644 src/AICapabilityRegistry.h create mode 100644 src/AICapabilityRegistry_test.cpp diff --git a/CLAUDE.md b/CLAUDE.md index d72a5684d..0a711a27a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -416,6 +416,7 @@ Video/webcam -> facial morph + head + skeletal body animation, built on the exis - **LLMManager** (`src/LLMManager.h/cpp`): QML_SINGLETON wrapping llama.cpp for local inference. - **LLMWorker**: Runs inference in a worker thread. - **ModelDownloader**: Downloads GGUF/ONNX models from HuggingFace — the ONE download path behind all 21 model consumers. **#1029 (CWE-494) hardening:** `startDownload` **refuses any URL that is not `https://` or a host-less `file://`** (plain http is a byte-for-byte MITM injection point, and every base URL is user-overridable via env/QSettings; `file:///share/…` is refused too — on Windows that is a UNC/SMB fetch over the network wearing a local scheme; only an empty or `localhost` host is accepted, and the error names the reason) — refused BEFORE any filesystem side effect, so no `.part` or directory is created. An optional 4th argument `expectedSha256` (case-insensitive hex; empty = legacy no-check, so the 21 consumers + the two QML call sites in `AISettingsDialog.qml` work unchanged) is verified by a local streamed `QCryptographicHash` helper **on the finished `.part` on disk, before the rename** (deliberately NOT the updater's `UpdateVerifier::sha256HexOfFile`: `qtmesh_updater` is only built/linked under `ENABLE_AUTO_UPDATER`, while `ModelDownloader` compiles unconditionally — reusing it broke the `-DENABLE_AUTO_UPDATER=OFF` link, caught in review; six lines of Qt API beat coupling every model download to the optional updater + libsodium) — never as a running hash in `onReadyRead`, because a resumed download appends to bytes this process never saw. On mismatch the `.part` is **deleted** (a poisoned partial must not be resumed from or cached) and `downloadError` fires. **Two gotchas verified empirically:** (1) the scheme refusal is emitted **QUEUED** (`QMetaObject::invokeMethod` + `Qt::QueuedConnection`), never synchronously — consumers connect, call `startDownload`, then `loop.exec()`, so a synchronous `loop.quit()` fires before `exec()` and is lost, hanging them for their full timeout (the #1017 review race); queued delivery lands inside `exec()` for every consumer without touching any — proven on `PhotoDepth`, which has no `settled` guard, failing in 1s instead of 600s. (2) Both refusals log at **`qCritical`**, not `qWarning`: `cliMessageHandler` drops warnings unless `--verbose`, and every consumer discards `downloadError`'s text (#1037), so a warning would leave the user with "offline?" and no way to learn the cause. **Resume verification (#1036):** a resume sends `Range: bytes=N-`, but a server that ignores it (`file://` always; any proxy that strips the header) answers **200 with the whole body**, and appending that after the stale `.part` produced a corrupt model — reproduced: a 29-byte stale prefix yielded a 208,044,845-byte file that **still loaded and ran** (ORT parsed the garbage as an unknown protobuf field) while a 30-byte prefix failed with "Protobuf parsing failed"; load success proves nothing about integrity. So on the FIRST `readyRead` of a resumed request (`m_resumeUnverified`, armed at both Range-request sites) the downloader requires **206 + a `Content-Range` starting exactly at the resume offset**; anything else is treated as the full body — the `.part` is reopened `Truncate`, `m_resumeOffset`/`m_bytesReceived` reset to 0 (else progress adds a phantom offset), and the download continues from byte 0 with no error (a 200 is recoverable). The range unit is compared **case-insensitively** (RFC 9110 §14.1 — `Bytes 9-12/13` is a valid honoured resume; a case-sensitive match misread it as "ignored" and fell into the truncate path with a PARTIAL body, i.e. an incomplete file a no-digest caller would rename — caught in review). A 206 whose window is neither ours nor the whole resource is a genuinely partial body we did not ask for: writing it from byte 0 would yield an INCOMPLETE file, so that case **aborts** with `downloadError` and removes the `.part` (next attempt starts clean); only a 206 covering exactly `0..total-1` — a full body wearing a partial status — is truncated-and-taken like a 200. Checked once, not per chunk. `FakeNetworkReply` in the tests defaults to a plain 200 (what an ignoring server returns) and `withPartialContent(first,last,total)` models an honoured resume. Both guards mutation-verified with correct selectivity. `ModelDownloader::isAllowedDownloadUrl(url)` is the pure predicate for checking a base URL up front. Tests: `ModelDownloader_test.cpp` (both guards mutation-verified). **Review round on the resume fix (#1039):** (1) *honoured* now also requires the window to REACH THE END — `first == offset && last == total-1` — because `bytes 9-10/13` starts right yet leaves 11-12 missing, and appending it would have promoted an incomplete file (that finding had been marked addressed by the bot without the code changing; verify against the code, not the bot's annotation); (2) every 'this partial is unusable' exit goes through ONE path, `discardPartialAndFail` — remove-or-truncate the `.part`, reset `m_bytesReceived`/`m_resumeOffset` (a stale offset would make `resumeDownload` re-request the old range against a fresh file), abort the reply under `m_abortingInternally` so the synchronously-delivered `errorOccurred`/`finished` step aside, end the download, emit exactly ONE `downloadError` (the test's `FakeNetworkReply::signalOnAbort` models the synchronous delivery); (3) `onDownloadFinished` refuses to promote when `m_resumeUnverified` is still set (a resume reply that finished without ever delivering data — verification never ran) or when the `.part` size differs from the size the response committed to (`m_expectedTotalBytes`: the 206 total, else a full body's Content-Length) — the `.part` is KEPT as a valid prefix for the next resume, not discarded like a digest mismatch. A server that declares no size, with no digest configured, is accepted with a `qWarning` (chunked transfer; HF/GitHub and QNAM's `file://` backend always send Content-Length, so real downloads take the strict path). +- **AI agent harness** (`src/AIAgentManager.{h,cpp}`, `src/AIAgentTypes.{h,cpp}`, `src/AICapabilityRegistry.{h,cpp}`, #1000/#1001/#1002/#1003 + #1021 a–d): the orchestration layer above `AIChatManager`/`LLMManager` — `User → AIAgentManager → Planner → Capability router → Executor → Observer → Replan/Finish`. **All task state lives in `AIAgent::Plan`/`Step`/`Observation`, never in the prompt**; the prompt is rebuilt from that state on every planner call, so nothing scrolls out of a history window and the whole state machine (`Planning → Executing → Observing → Replanning/AwaitingConfirmation → Completed/Failed/Cancelled`) is driven headless in `AIAgentManager_test.cpp` by a scripted `AgentPlannerBackend` + `AgentToolExecutor` (no LLM, no Ogre). **Capabilities, not a catalog:** `AICapabilityRegistry` groups the ~170 MCP tools into 20 capabilities (scene, scene_io, view, materials, lighting, textures_ai, mesh_optimize, uv, rigging, segmentation, generation_3d, animation, motion_ai, morph_pose, node_animation, paint, mocap, cloud, ps1, other — `taxonomy()` + prefix rules; an unmapped tool lands in `other`, never dropped). The planner sees a ~20-line capability index plus the FULL per-tool docs only for the capabilities `routeByKeywords()` pre-selected; it may answer `{"need_capabilities":[...]}` to have more docs added before planning (dynamic discovery — the v1 loop's hard-coded 19-tool subset silently excluded rigging/segmentation/generation). Tool docs are generated from the live `buildToolsList()` schema (name, one-line description, params with type/required/enum), so they cannot drift. **Constrained protocol (#1003):** `validateArguments` checks every planned call against the schema BEFORE it reaches the server — required present, types checked, chatty-model output coerced (`"2"`→2, `"yes"`→true, `"2, 2, 2"`→array) with warnings, enums enforced; an invalid call becomes an `invalid_arguments` observation and a replan, never a tool call. **Observations** (`observationFromToolResult`) parse facts out of tool text (`Vertices: N`, JSON `boneCount`, `fallbackReason`→warning), collect file artifacts, keep `raw` for the transcript only — the replan prompt gets one compact line per observation. **Recovery:** a failing step is retried once (same call), then the planner is asked to repair the tail (`{"steps":[...]}` or `{"done":true,"summary"}`), bounded by `Limits` (12 steps, 2 replans, 2 planner retries); the same `Step::signature()` failing twice stops the task ("stuck, not unlucky"). **One undo group per task** (`QUndoStack::beginMacro(plan.title)` on the first non-read-only step, `endMacro` on every terminal path — `isReadOnly()` = get_*/list_*/toggle_*/screenshots/validators). **Safety rail (#1021d):** `destructiveReason()` names deletes/geometry rewrites and overwrites of an EXISTING file; unless `trustedMode` (QSettings `ai/agentTrustedMode`) the task pauses in `AwaitingConfirmation` and the panel shows Allow / Always allow / Skip step. **Scene context per turn (#1021c):** `setContextProvider` — `AIChatManager::sceneSummaryForAgent()` (scene info + user materials + recent files) is appended to every planner prompt. The final message is the deterministic `summarize()` (steps, statuses, parsed facts, artifacts, warnings), so a flaky model cannot misreport what happened; a `{"summary"}` reply with no steps answers a question without tools. **Facade:** `AIChatManager::agentMode` (QSettings `ai/agentMode`, default ON) routes `sendMessage` to `AIAgentManager::startTask`; while the agent drives, the v1 slots ignore `LLMManager` callbacks (`m_agentDriving`) and `LlmPlannerBackend` forwards generation signals only while it has a pending request. QML: `qml/AIChatPanel.qml` gained the agent/ask-trusted toggles, a live plan card, the confirmation bar, and a model tip (#1021e: Qwen 2.5 7B Q4_K_M is the recommended tool-calling model — its `ModelInfo` description says so). Sentry `ai.agent.plan|step|retry|replan|verify|confirm|done|fail|cancel`. **Test gotcha:** fixture tests are `TEST_F(AgentFixture, …)` — a `--gtest_filter='AIAgent*'` does NOT run them; use `AgentFixture*`. Follow-ups (children of #1000): #1004 local VLM viewport inspection (mtmd is already linked for the captioner), #1005 model lifecycle/memory budget, #1006 verifier layer, #1007 e2e benchmark, #1008 batch factory. - **ModelFetch** (`src/ModelFetch.{h,cpp}`, #1037): the ONE blocking "make sure this model file is on disk" primitive — `ModelFetch::ensureBlocking(Request{url,destination,label,timeoutMs,expectedSha256}) -> Outcome{ok,timedOut,path,error}`. Twenty consumers used to hand-roll the same nested-QEventLoop wait around `ModelDownloader`; the copies drifted: 18 DISCARDED the downloader's error text (so "refusing http://" / "SHA-256 mismatch" reached the user as "unavailable (offline?)"), and only 3 (`TextureInpaint`, `FaceRig/ArkitTemplate`, `FaceRig/FaceLandmarkDetector`) guarded the **synchronous-rejection race** (`startDownload` emits `downloadError` synchronously when busy → the handler's `loop.quit()` fires before `exec()` and is lost → the caller hangs for its full timeout). `ensureBlocking` owns only the wait and returns the downloader's own words; consumers keep what genuinely varies. **#1025 — an EXISTING file is verified when a digest is known:** with `Request::expectedSha256` set, a destination that already exists is hashed (`ModelDownloader::sha256HexOfFile`, cached per process by path+size+mtime so a 1.2 GB decoder is hashed once, not per rig) and a mismatch DELETES it (+ its `.part`) and re-fetches it, reporting `Outcome::replacedCorrupt`; without a digest "exists" still means ok. This is what caught the UniRig case below — a same-size corrupted download that loaded fine and produced NaN for months. Consumers with published digests (HF LFS oids) should pass them; UniRig does for its default hosting only (a mirror override may serve a different export) (base-URL env/QSettings/default resolution, the `*_NO_DOWNLOAD` guard, the timeout). **Convention for exposing the reason:** `ensureModelBlocking(QString* error = nullptr)` — done for `PhotoDepth` and `TextureInpaint` (their 4 CLI/MCP sites now print e.g. `…unavailable: Refusing to download …: scheme 'http' is not https://`, proven e2e); the other migrated consumers keep their signatures (no message site to enrich yet). Migrated (behaviour-preserving): every consumer with a blocking wait — `PhotoDepth`, `TextureInpaint` (with the `error` out-param), `AIAssistManager`, `ImageTo3D/ImageCaptioner`, `ImageTo3D/MeshGenPredictor`, `ImageTo3D/TripoSGPredictor`, `ImageTo3D/BackgroundRemover`, `MeshSegmenter`, `MotionInbetween`, `MotionGenerator`, `SkinTokensPredictor`, `UniRigPredictor`, `FaceRig/ArkitTemplate`, `FaceRig/FaceLandmarkDetector`, `Mocap/FaceCapPredictor`, `Mocap/PoseCapPredictor`, `Mocap/HandCapPredictor`. The two FaceRig consumers had their own `done` race guard — now `ModelFetch`'s `settled`; their `(timeout)` breadcrumb annotation comes from `Outcome::timedOut`. `MotionGenerator`'s old `guard` timer never cancelled the transfer on timeout; `ModelFetch` does. **Deliberately NOT migrated:** `MotionLibrary::ensureLibraryBlocking` (V1→V2 upgrade logic where a failed download must fall back to the local V1 file — `haveLocal ? dest : QString()` — not the canonical shape; the only hand-rolled **`ModelDownloader`** wait left — other `QEventLoop`s in the tree belong to the cloud client, HDR downloads and the updater, which have their own network paths). **Gotcha from this migration:** at least one source file is not valid UTF-8 — a Python `open(p).read()` sweep over `src/` raises `UnicodeDecodeError`; use `encoding='utf-8', errors='surrogateescape'` for read AND write so bytes round-trip exactly. `AIModelCatalog`/`LLMSettingsWidget` are GUI-async (no event loop) and already show the error text. Tests: `ModelFetch_test.cpp` drives the REAL singleton through QNAM's `file://` backend (existing-file short-circuit, real fetch, refusal text verbatim, missing-file network error, synchronous rejection returns in <1 s not after the timeout); the race guard and error capture are mutation-verified. ### AI Texture Generation diff --git a/README.md b/README.md index cb6a841cc..70db499b9 100755 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ Split View|Skeleton Animation Controls - **AI part segmentation & PartOps** — detect a character's parts (head/torso/arms/legs), then **split** into named submeshes, **explode** into separate scene nodes, **join** them back, or **solidify** thin-shell parts; GUI + `qtmesh segment --split-parts / --explode-parts` + MCP (`split_mesh_by_segments`, `explode_mesh_parts`, `join_mesh_parts`) — see [docs/PART_OPS.md](docs/PART_OPS.md) - **Scene management** — duplicate (Ctrl+D), group (Ctrl+G), snap, pivot modes - **Performance capture** — video/webcam → facial morph animation (ARKit blendshapes), head pose, and full-body skeletal capture onto humanoid rigs; live preview + record in the editor, `qtmesh mocap` on the CLI (`-DENABLE_MOCAP` builds) -- **AI chat** — natural language scene editing via local LLMs +- **AI agent** — describe a task in natural language; a local LLM plans it over the editor's tools, runs it step by step with structured observations and replanning, groups it into ONE undo step, and asks before deleting or overwriting (agent mode; the simple chat loop is one click away) - **MCP server** — 57+ tools for AI agents (Claude, Cursor, etc.), including HDR/IBL (`set_hdr_environment`, `set_tonemap`, …) and QtMesh Cloud (`cloud_*`) - **REST API** — opt-in HTTP interface for external automation (`--with-mcp --http-port 8080`). Bound to **localhost** by default; tools execute only via `POST /api/tools/` (`GET /api/tools` lists them); set `QTMESH_HTTP_TOKEN` (or `--http-token-file `; a secret on the command line is refused, since `ps` shows it to every local user) to require `Authorization: Bearer ` on every request, and `--http-bind 0.0.0.0` only when you mean to expose it diff --git a/qml/AIChatPanel.qml b/qml/AIChatPanel.qml index 0af1f16e7..af541d3e7 100644 --- a/qml/AIChatPanel.qml +++ b/qml/AIChatPanel.qml @@ -16,6 +16,9 @@ Rectangle { Qt.callLater(() => inputField.forceActiveFocus()) } + readonly property bool agentBusy: AIAgentManager.busy + readonly property bool awaitingConfirm: AIAgentManager.state === "awaiting_confirmation" + // ---- Header bar ---- Rectangle { id: header @@ -28,12 +31,68 @@ Rectangle { spacing: 6 Text { - text: "AI Chat" + text: AIChatManager.agentMode ? "AI Agent" : "AI Chat" color: PropertiesPanelController.textColor font.pixelSize: 13; font.bold: true Layout.fillWidth: true } + // Agent mode toggle (#1021): plan → execute → observe, one undo group. + Rectangle { + width: agentToggleText.implicitWidth + 12; height: 20; radius: 3 + color: AIChatManager.agentMode + ? PropertiesPanelController.highlightColor + : PropertiesPanelController.buttonColor + border.color: PropertiesPanelController.borderColor + ToolTip.visible: agentToggleArea.containsMouse + ToolTip.delay: 500 + ToolTip.text: AIChatManager.agentMode + ? "Agent mode: multi-step plans, structured observations, one undo group per task, confirmations for destructive steps. Click for the simple chat loop." + : "Simple chat loop (one tool per reply). Click for agent mode." + Text { + id: agentToggleText + anchors.centerIn: parent + text: "agent" + color: PropertiesPanelController.textColor + font.pixelSize: 10 + } + MouseArea { + id: agentToggleArea + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + enabled: !AIChatManager.isGenerating + onClicked: AIChatManager.agentMode = !AIChatManager.agentMode + } + } + + // Trusted mode (#1021d): skip confirmations for destructive steps. + Rectangle { + visible: AIChatManager.agentMode + width: trustText.implicitWidth + 12; height: 20; radius: 3 + color: AIAgentManager.trustedMode ? "#aa5533" : PropertiesPanelController.buttonColor + border.color: PropertiesPanelController.borderColor + ToolTip.visible: trustArea.containsMouse + ToolTip.delay: 500 + ToolTip.text: AIAgentManager.trustedMode + ? "Trusted mode ON: the agent deletes and overwrites without asking. Click to require confirmations again." + : "Destructive steps (delete, overwrite a file, rewrite geometry) ask for confirmation. Click to trust the agent for this machine." + Text { + id: trustText + anchors.centerIn: parent + text: AIAgentManager.trustedMode ? "trusted" : "ask" + color: PropertiesPanelController.textColor + font.pixelSize: 10 + } + MouseArea { + id: trustArea + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + onClicked: AIAgentManager.trustedMode = !AIAgentManager.trustedMode + } + } + // Model status dot Rectangle { width: 8; height: 8; radius: 4 @@ -47,7 +106,7 @@ Rectangle { color: PropertiesPanelController.textColor font.pixelSize: 10 elide: Text.ElideMiddle - Layout.maximumWidth: 160 + Layout.maximumWidth: 120 } // Clear button @@ -69,12 +128,35 @@ Rectangle { } } + // ---- Model recommendation (#1021e) ---- + Rectangle { + id: modelHint + anchors { top: header.bottom; left: parent.left; right: parent.right } + visible: AIChatManager.agentMode && AIChatManager.modelAvailable + && AIChatManager.currentModelName.indexOf("7b") < 0 + && AIChatManager.currentModelName.indexOf("7B") < 0 + && AIChatManager.currentModelName.indexOf("12b") < 0 + && AIChatManager.currentModelName.indexOf("12B") < 0 + height: visible ? hintText.implicitHeight + 8 : 0 + color: Qt.rgba(1, 0.8, 0.3, 0.08) + Text { + id: hintText + anchors { left: parent.left; right: parent.right; verticalCenter: parent.verticalCenter; margins: 8 } + wrapMode: Text.Wrap + font.pixelSize: 9 + opacity: 0.75 + color: PropertiesPanelController.textColor + text: "Tip: small models struggle with multi-step tool calls. " + AIAgentManager.recommendedModelName + + " is the recommended agent model (AI → AI Model Settings → Recommended)." + } + } + // ---- Single selectable conversation area ---- Flickable { id: msgFlick anchors { - top: header.bottom; left: parent.left; right: parent.right - bottom: thinkingRow.top; bottomMargin: 0 + top: modelHint.bottom; left: parent.left; right: parent.right + bottom: planCard.top; bottomMargin: 0 } clip: true contentWidth: width @@ -108,9 +190,6 @@ Rectangle { } } - // When the user finishes selecting (mouse released with no selection), - // return focus to the input field. Use onActiveFocusChanged instead of - // onSelectedTextChanged to avoid stealing focus mid-drag. onActiveFocusChanged: { if (!activeFocus && selectedText.length === 0) Qt.callLater(() => inputField.forceActiveFocus()) @@ -126,13 +205,127 @@ Rectangle { } } + // ---- Plan card (#1021b): live step list of the running/last agent task ---- + Rectangle { + id: planCard + anchors { bottom: confirmBar.top; left: parent.left; right: parent.right; margins: visible ? 6 : 0 } + visible: AIChatManager.agentMode && AIAgentManager.plan.length > 0 + && (root.agentBusy || root.planCardPinned) + height: visible ? planCol.implicitHeight + 12 : 0 + radius: 4 + color: PropertiesPanelController.headerColor + border.color: PropertiesPanelController.borderColor + property bool planCardPinned: false + + Column { + id: planCol + anchors { left: parent.left; right: parent.right; top: parent.top; margins: 6 } + spacing: 2 + RowLayout { + width: parent.width + Text { + text: AIAgentManager.planTitle.length > 0 ? AIAgentManager.planTitle : "Plan" + color: PropertiesPanelController.textColor + font.pixelSize: 11; font.bold: true + elide: Text.ElideRight + Layout.fillWidth: true + } + Text { + text: AIAgentManager.state + color: PropertiesPanelController.textColor + opacity: 0.6 + font.pixelSize: 9 + } + } + Repeater { + model: AIAgentManager.plan + Row { + spacing: 6 + width: planCol.width + Text { + width: 14 + text: modelData.status === "succeeded" ? "✓" + : modelData.status === "failed" ? "✗" + : modelData.status === "running" ? "▶" + : modelData.status === "skipped" ? "–" : "○" + color: modelData.status === "succeeded" ? "#66cc66" + : modelData.status === "failed" ? "#dd5555" + : modelData.status === "running" ? PropertiesPanelController.accentColor + : PropertiesPanelController.textColor + font.pixelSize: 11 + } + Text { + width: parent.width - 20 + text: (modelData.index + 1) + ". " + modelData.tool + + (modelData.why.length > 0 ? " — " + modelData.why : "") + + (modelData.status === "failed" && modelData.error.length > 0 ? " (" + modelData.error + ")" : "") + color: PropertiesPanelController.textColor + opacity: modelData.status === "skipped" ? 0.5 : 0.9 + font.pixelSize: 10 + elide: Text.ElideRight + maximumLineCount: 2 + wrapMode: Text.Wrap + } + } + } + } + } + + // ---- Confirmation bar (#1021d) ---- + Rectangle { + id: confirmBar + anchors { bottom: thinkingRow.top; left: parent.left; right: parent.right; margins: visible ? 6 : 0 } + visible: root.awaitingConfirm + height: visible ? confirmCol.implicitHeight + 12 : 0 + radius: 4 + color: Qt.rgba(0.9, 0.5, 0.2, 0.15) + border.color: "#cc7733" + + Column { + id: confirmCol + anchors { left: parent.left; right: parent.right; top: parent.top; margins: 6 } + spacing: 6 + Text { + width: parent.width + wrapMode: Text.Wrap + text: "⚠ " + AIAgentManager.pendingConfirmation + ". Allow?" + color: PropertiesPanelController.textColor + font.pixelSize: 11 + } + Flow { + width: parent.width + spacing: 6 + Repeater { + model: [ + { label: "Allow", approve: true, always: false }, + { label: "Always allow", approve: true, always: true }, + { label: "Skip step", approve: false, always: false } + ] + Rectangle { + width: btnText.implicitWidth + 16; height: 22; radius: 3 + color: btnArea.containsMouse ? PropertiesPanelController.accentColor : PropertiesPanelController.buttonColor + border.color: PropertiesPanelController.borderColor + Text { id: btnText; anchors.centerIn: parent; text: modelData.label; color: PropertiesPanelController.textColor; font.pixelSize: 10 } + MouseArea { + id: btnArea + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + onClicked: AIAgentManager.confirmPendingStep(modelData.approve, modelData.always) + } + } + } + } + } + } + // ---- Thinking dots (no tokens yet) ---- Row { id: thinkingRow anchors { bottom: inputRow.top; left: parent.left; leftMargin: 12; bottomMargin: 6 } height: visible ? 14 : 0 spacing: 4 - visible: AIChatManager.isGenerating + visible: AIChatManager.isGenerating && !root.awaitingConfirm Repeater { model: 3 @@ -148,6 +341,14 @@ Rectangle { } } } + Text { + visible: AIChatManager.agentMode && root.agentBusy + text: AIAgentManager.state + color: PropertiesPanelController.textColor + opacity: 0.5 + font.pixelSize: 9 + anchors.verticalCenter: parent.verticalCenter + } } // ---- Input row ---- @@ -166,7 +367,8 @@ Rectangle { leftMargin: 8; rightMargin: 6 } height: Math.min(implicitHeight, 80) placeholderText: AIChatManager.modelAvailable - ? "Ask AI to do something…" + ? (AIChatManager.agentMode ? "Describe a task — e.g. \"load the wolf, rig it as a quadruped and export a glb\"" + : "Ask AI to do something…") : "Load an AI model first (AI → AI Model Settings)" color: PropertiesPanelController.textColor background: null @@ -231,13 +433,18 @@ Rectangle { } Text { anchors.horizontalCenter: parent.horizontalCenter - text: "Ask me to control the editor" + text: AIChatManager.agentMode ? "Give me a task — I plan it, run it step by step, and it undoes as one" + : "Ask me to control the editor" color: PropertiesPanelController.textColor - font.pixelSize: 13; opacity: 0.7 + font.pixelSize: 12; opacity: 0.7 + horizontalAlignment: Text.AlignHCenter + width: root.width - 40 + wrapMode: Text.Wrap } Text { anchors.horizontalCenter: parent.horizontalCenter - text: "\"make the selected mesh twice as large\"" + text: AIChatManager.agentMode ? "\"segment the wolf, rig it as a quadruped and skin it\"" + : "\"make the selected mesh twice as large\"" color: PropertiesPanelController.textColor font.pixelSize: 11; opacity: 0.45; font.italic: true } @@ -268,11 +475,12 @@ Rectangle { var isTool = msg.isTool var role = msg.role var roleColor, roleLabel - if (isTool) { roleColor = "#88cc88"; roleLabel = "⚙ tool" } - else if (role === "user"){ roleColor = "#88aacc"; roleLabel = "you" } - else { roleColor = "#aaaaaa"; roleLabel = "assistant" } + if (isTool) { roleColor = "#88cc88"; roleLabel = "⚙ tool" } + else if (role === "user") { roleColor = "#88aacc"; roleLabel = "you" } + else if (role === "plan") { roleColor = "#ccaa66"; roleLabel = "plan" } + else { roleColor = "#aaaaaa"; roleLabel = "assistant" } - // Assistant messages may be structured JSON — render appropriately. + // Assistant messages may be structured JSON (v1 loop) — render appropriately. var displayText = msg.text if (role === "assistant" && !isTool) { var trimmed = msg.text.trim() @@ -299,7 +507,7 @@ Rectangle { } } - // In-progress streaming text + // In-progress streaming text (v1 loop only) if (AIChatManager.streamingText.length > 0) { if (msgs.length > 0) html += "
" html += 'assistant
' diff --git a/src/AIAgentManager.cpp b/src/AIAgentManager.cpp new file mode 100644 index 000000000..05743584f --- /dev/null +++ b/src/AIAgentManager.cpp @@ -0,0 +1,673 @@ +#include "AIAgentManager.h" + +#include "LLMManager.h" +#include "MCPServer.h" +#include "SentryReporter.h" +#include "UndoManager.h" + +#include +#include +#include +#include + +using namespace AIAgent; + +namespace { +constexpr const char* kTrustedModeKey = "ai/agentTrustedMode"; +constexpr const char* kRecommendedModel = "Qwen 2.5 7B Q4_K_M"; + +/// LLMManager-backed planner. Forwards LLMManager's generation signals only +/// while it has an outstanding request, so a material-generation run by +/// another component never lands in the agent. +class LlmPlannerBackend : public AgentPlannerBackend +{ +public: + explicit LlmPlannerBackend(QObject* parent) : AgentPlannerBackend(parent) + { + auto* llm = LLMManager::instance(); + connect(llm, &LLMManager::generationCompleted, this, [this](const QString& t) { + if (!m_pending) return; m_pending = false; emit completed(t); + }); + connect(llm, &LLMManager::generationError, this, [this](const QString& e) { + if (!m_pending) return; m_pending = false; emit failed(e); + }); + connect(llm, &LLMManager::generationStopped, this, [this]() { + if (!m_pending) return; m_pending = false; emit stopped(); + }); + } + bool available() const override { return LLMManager::instance()->isModelLoaded(); } + QString modelName() const override { return LLMManager::instance()->currentModelName(); } + void request(const QString& systemPrompt, const QString& userPrompt, int maxTokens) override + { + m_pending = true; + LLMManager::instance()->generateText(systemPrompt, userPrompt, maxTokens); + } + void stop() override { if (m_pending) LLMManager::instance()->stopGeneration(); } +private: + bool m_pending = false; +}; +} // namespace + +// --------------------------------------------------------------------------- +// McpToolExecutor + +McpToolExecutor::McpToolExecutor(MCPServer* server) : m_server(server) {} +McpToolExecutor::~McpToolExecutor() = default; + +QJsonArray McpToolExecutor::toolList() +{ + return m_server ? m_server->buildToolsList() : QJsonArray{}; +} + +QJsonObject McpToolExecutor::callTool(const QString& name, const QJsonObject& args) +{ + if (!m_server) return QJsonObject{{"isError", true}, + {"content", QJsonArray{QJsonObject{{"type", "text"}, {"text", "Error: MCP server not available"}}}}}; + return m_server->callTool(name, args); +} + +// --------------------------------------------------------------------------- + +AIAgentManager* AIAgentManager::s_instance = nullptr; + +AIAgentManager* AIAgentManager::instance() +{ + if (!s_instance) s_instance = new AIAgentManager(); + return s_instance; +} + +AIAgentManager* AIAgentManager::qmlInstance(QQmlEngine* engine, QJSEngine* scriptEngine) +{ + Q_UNUSED(engine); Q_UNUSED(scriptEngine); + auto* inst = instance(); + QQmlEngine::setObjectOwnership(inst, QQmlEngine::CppOwnership); + return inst; +} + +void AIAgentManager::kill() +{ + delete s_instance; + s_instance = nullptr; +} + +AIAgentManager::AIAgentManager(QObject* parent) : QObject(parent) +{ + QSettings settings; + m_trustedMode = settings.value(QLatin1String(kTrustedModeKey), false).toBool(); +} + +AIAgentManager::~AIAgentManager() +{ + closeUndoGroup(); +} + +void AIAgentManager::setExecutor(std::shared_ptr executor) +{ + m_executor = std::move(executor); + m_registry = AICapabilityRegistry(m_executor ? m_executor->toolList() : QJsonArray{}); +} + +void AIAgentManager::setPlanner(AgentPlannerBackend* planner) +{ + if (m_planner) { m_planner->disconnect(this); m_planner->deleteLater(); } + m_planner = planner; + if (!m_planner) return; + m_planner->setParent(this); + connect(m_planner, &AgentPlannerBackend::completed, this, &AIAgentManager::onPlannerCompleted); + connect(m_planner, &AgentPlannerBackend::failed, this, &AIAgentManager::onPlannerFailed); + connect(m_planner, &AgentPlannerBackend::stopped, this, &AIAgentManager::onPlannerStopped); +} + +void AIAgentManager::ensurePlanner() +{ + if (!m_planner) setPlanner(new LlmPlannerBackend(this)); +} + +void AIAgentManager::setTrustedMode(bool on) +{ + if (m_trustedMode == on) return; + m_trustedMode = on; + QSettings settings; + settings.setValue(QLatin1String(kTrustedModeKey), on); + SentryReporter::addBreadcrumb("ai.agent.confirm", on ? "trusted mode ON" : "trusted mode OFF"); + emit trustedModeChanged(); +} + +QString AIAgentManager::recommendedModelName() const +{ + return QLatin1String(kRecommendedModel); +} + +QVariantList AIAgentManager::planModel() const +{ + QVariantList out; + for (int i = 0; i < m_plan.steps.size(); ++i) { + const Step& s = m_plan.steps[i]; + out << QVariantMap{ + {"index", i}, {"tool", s.tool}, {"why", s.why}, + {"status", Step::statusName(s.status)}, {"error", s.error}, {"attempts", s.attempts}, + }; + } + return out; +} + +void AIAgentManager::setState(State s) +{ + if (m_state == s) return; + m_state = s; + emit stateChanged(); +} + +void AIAgentManager::say(const QString& text) { emit chatMessage(QStringLiteral("assistant"), text, false); } +void AIAgentManager::sayTool(const QString& text) { emit chatMessage(QStringLiteral("tool"), text, true); } + +// --------------------------------------------------------------------------- +// control + +bool AIAgentManager::startTask(const QString& request) +{ + const QString goal = request.trimmed(); + if (goal.isEmpty()) return false; + if (busy()) { say(QStringLiteral("A task is already running — cancel it first.")); return false; } + if (!m_executor) { say(QStringLiteral("The tool server is not available.")); return false; } + ensurePlanner(); + if (!m_planner->available()) { + say(QStringLiteral("No AI model is loaded (AI → AI Model Settings). Recommended for multi-step tasks: %1.") + .arg(recommendedModelName())); + return false; + } + if (m_registry.isEmpty()) m_registry = AICapabilityRegistry(m_executor->toolList()); + + m_plan = Plan{}; + m_plan.goal = goal; + m_observations.clear(); + m_failureCounts.clear(); + m_docCapabilities = m_registry.routeByKeywords(goal); + m_currentStep = -1; m_pendingIndex = -1; m_pendingReason.clear(); + m_replans = 0; m_plannerRetries = 0; m_replanFailedIndex = -1; + m_cancelRequested = false; m_lastSummary.clear(); m_lastError.clear(); + emit planChanged(); emit confirmationChanged(); + + SentryReporter::addBreadcrumb("ai.agent.plan", QStringLiteral("task started (%1 routed capabilities)").arg(m_docCapabilities.size())); + requestPlan(); + return true; +} + +void AIAgentManager::cancel() +{ + if (!busy()) return; + m_cancelRequested = true; + SentryReporter::addBreadcrumb("ai.agent.cancel", QStringLiteral("cancelled in state %1").arg(stateName())); + if (m_state == State::Planning || m_state == State::Replanning) { + m_awaiting = Awaiting::None; + if (m_planner) m_planner->stop(); + } + if (m_state == State::AwaitingConfirmation && m_pendingIndex >= 0 && m_pendingIndex < m_plan.steps.size()) { + m_plan.steps[m_pendingIndex].status = Step::Skipped; + m_pendingIndex = -1; m_pendingReason.clear(); emit confirmationChanged(); + } + finish(State::Cancelled); +} + +void AIAgentManager::confirmPendingStep(bool approve, bool alwaysAllow) +{ + if (m_state != State::AwaitingConfirmation || m_pendingIndex < 0) return; + const int idx = m_pendingIndex; + m_pendingIndex = -1; + const QString reason = m_pendingReason; + m_pendingReason.clear(); + emit confirmationChanged(); + if (alwaysAllow) setTrustedMode(true); + if (approve) { + SentryReporter::addBreadcrumb("ai.agent.confirm", QStringLiteral("approved: %1").arg(reason)); + runStep(idx); + return; + } + SentryReporter::addBreadcrumb("ai.agent.confirm", QStringLiteral("denied: %1").arg(reason)); + Step& s = m_plan.steps[idx]; + s.status = Step::Skipped; + s.error = QStringLiteral("denied by user"); + Observation ob; ob.stepIndex = idx; ob.tool = s.tool; ob.status = QStringLiteral("denied"); ob.error = s.error; + m_observations << ob; + sayTool(QStringLiteral("[%1] skipped — %2").arg(s.tool, reason)); + emit planChanged(); + emit stepFinished(idx, false); + QTimer::singleShot(0, this, &AIAgentManager::executeNext); +} + +// --------------------------------------------------------------------------- +// planning + +QString AIAgentManager::sceneContext() const +{ + return m_contextProvider ? m_contextProvider() : QString(); +} + +QString AIAgentManager::systemPrompt(const QStringList& capabilityIds) const +{ + QString s = QStringLiteral( + "You are the planner of QtMeshEditor's AI agent. You control a 3D mesh editor by choosing tool calls.\n" + "Reply with ONE JSON object and nothing else.\n\n" + "Capabilities (groups of tools):\n%1\n" + "Tools you may use now (capabilities: %2):\n%3\n" + "If the task needs a capability whose tools are NOT listed above, reply exactly:\n" + "{\"need_capabilities\": [\"capability_id\"]}\n" + "If the task is a question you can answer from the scene state, reply:\n" + "{\"summary\": \"the answer\"}\n" + "Otherwise reply with the plan:\n" + "{\"title\": \"short task title\", \"steps\": [{\"tool\": \"tool_name\", \"arguments\": {\"param\": \"value\"}, \"why\": \"few words\"}]}\n\n" + "Rules:\n" + "1. Only tool names and parameter names listed above. Never invent either.\n" + "2. 1 to %4 steps, in execution order, minimal — only what the user asked for.\n" + "3. Up/down is +Y/-Y; forward is -Z; ground is Y=0. 'twice as large' = scale [2,2,2].\n" + "4. Use exact object/material names from the scene state. Call get_scene_info first only when a needed name is unknown.\n" + "5. Never repeat a step.\n") + .arg(m_registry.promptIndex(), capabilityIds.join(", "), m_registry.promptToolsFor(capabilityIds)) + .arg(m_limits.maxSteps); + const QString ctx = sceneContext(); + if (!ctx.isEmpty()) s += QStringLiteral("\nScene state:\n%1\n").arg(ctx); + return s; +} + +void AIAgentManager::requestPlan(const QString& extraInstruction) +{ + setState(State::Planning); + m_awaiting = Awaiting::Plan; + QString user = QStringLiteral("Task: %1\n").arg(m_plan.goal); + if (!extraInstruction.isEmpty()) user += extraInstruction + '\n'; + user += QStringLiteral("JSON:"); + m_planner->request(systemPrompt(m_docCapabilities), user, 700); +} + +void AIAgentManager::requestReplan(int failedIndex) +{ + setState(State::Replanning); + m_awaiting = Awaiting::Replan; + m_replanFailedIndex = failedIndex; + ++m_replans; + SentryReporter::addBreadcrumb("ai.agent.replan", QStringLiteral("replan %1 after step %2").arg(m_replans).arg(failedIndex + 1)); + + QStringList obsLines; + for (const Observation& ob : m_observations) obsLines << ob.toPromptLine(); + QJsonArray planArr; + for (const Step& s : m_plan.steps) planArr.append(s.toJson()); + const Step& failed = m_plan.steps[failedIndex]; + QString user = QStringLiteral( + "Original task: %1\n" + "Plan so far: %2\n" + "Observations:\n%3\n" + "Step %4 (%5) failed: %6\n" + "Reply with ONE JSON object: {\"steps\": [remaining steps to run now, fixed], \"done\": false}\n" + "or {\"done\": true, \"summary\": \"what was achieved / why it cannot be completed\"}.\n" + "Do not repeat steps that already succeeded. Do not repeat the failing call unchanged.\n" + "JSON:") + .arg(m_plan.goal, QString::fromUtf8(QJsonDocument(planArr).toJson(QJsonDocument::Compact)), + obsLines.join('\n')) + .arg(failedIndex + 1).arg(failed.tool, failed.error.left(300)); + m_planner->request(systemPrompt(m_docCapabilities), user, 700); +} + +QString AIAgentManager::extractJsonObject(const QString& text) +{ + // First balanced {...}; tolerate a missing opening brace (models primed + // with "{" sometimes omit it) by trying the prefixed variant second. + auto balanced = [](const QString& t) -> QString { + const int start = t.indexOf('{'); + if (start < 0) return {}; + int depth = 0; bool inStr = false; bool esc = false; + for (int i = start; i < t.size(); ++i) { + const QChar c = t[i]; + if (inStr) { + if (esc) esc = false; + else if (c == '\\') esc = true; + else if (c == '"') inStr = false; + continue; + } + if (c == '"') inStr = true; + else if (c == '{') ++depth; + else if (c == '}') { if (--depth == 0) return t.mid(start, i - start + 1); } + } + return {}; + }; + QString block = balanced(text); + if (block.isEmpty()) block = balanced('{' + text.trimmed()); + return block; +} + +bool AIAgentManager::parsePlanReply(const QString& text, Plan* out, QStringList* needCapabilities, + QString* answer, QString* error) +{ + const QString block = extractJsonObject(text); + QJsonParseError perr; + const QJsonDocument doc = QJsonDocument::fromJson(block.toUtf8(), &perr); + if (block.isEmpty() || perr.error != QJsonParseError::NoError || !doc.isObject()) { + if (error) *error = QStringLiteral("reply is not a JSON object"); + return false; + } + const QJsonObject o = doc.object(); + if (o.contains("need_capabilities")) { + QStringList ids; + for (const QJsonValue& v : o["need_capabilities"].toArray()) ids << v.toString(); + if (ids.isEmpty() && o["need_capabilities"].isString()) ids << o["need_capabilities"].toString(); + if (needCapabilities) *needCapabilities = ids; + return true; + } + const QJsonArray steps = o["steps"].toArray(); + if (steps.isEmpty()) { + const QString summary = o["summary"].toString().isEmpty() ? o["response"].toString() : o["summary"].toString(); + if (summary.isEmpty()) { if (error) *error = QStringLiteral("plan has no steps"); return false; } + if (answer) *answer = summary; + return true; + } + Plan p; + p.title = o["title"].toString().simplified(); + for (const QJsonValue& v : steps) { + const QJsonObject so = v.toObject(); + Step s; + s.tool = so["tool"].toString().isEmpty() ? so["command"].toString() : so["tool"].toString(); + s.tool = s.tool.trimmed(); + s.arguments = so["arguments"].toObject(); + if (s.arguments.isEmpty()) s.arguments = so["args"].toObject(); + s.why = so["why"].toString().simplified(); + if (s.tool.isEmpty()) { if (error) *error = QStringLiteral("a step has no tool name"); return false; } + p.steps.push_back(s); + } + if (out) *out = p; + return true; +} + +bool AIAgentManager::parseReplanReply(const QString& text, QVector* steps, bool* done, + QString* summary, QString* error) +{ + const QString block = extractJsonObject(text); + QJsonParseError perr; + const QJsonDocument doc = QJsonDocument::fromJson(block.toUtf8(), &perr); + if (block.isEmpty() || perr.error != QJsonParseError::NoError || !doc.isObject()) { + if (error) *error = QStringLiteral("reply is not a JSON object"); + return false; + } + const QJsonObject o = doc.object(); + const bool isDone = o["done"].toBool(false); + QVector out; + for (const QJsonValue& v : o["steps"].toArray()) { + const QJsonObject so = v.toObject(); + Step s; + s.tool = (so["tool"].toString().isEmpty() ? so["command"].toString() : so["tool"].toString()).trimmed(); + s.arguments = so["arguments"].toObject(); + if (s.arguments.isEmpty()) s.arguments = so["args"].toObject(); + s.why = so["why"].toString().simplified(); + if (!s.tool.isEmpty()) out.push_back(s); + } + if (!isDone && out.isEmpty()) { if (error) *error = QStringLiteral("replan has no steps and is not done"); return false; } + if (steps) *steps = out; + if (done) *done = isDone; + if (summary) *summary = o["summary"].toString(); + return true; +} + +void AIAgentManager::onPlannerCompleted(const QString& text) +{ + if (m_awaiting == Awaiting::None) return; + if (m_cancelRequested) { m_awaiting = Awaiting::None; return; } + const Awaiting what = m_awaiting; + m_awaiting = Awaiting::None; + + if (what == Awaiting::Plan) { + Plan plan; QStringList need; QString answer; QString err; + if (!parsePlanReply(text, &plan, &need, &answer, &err)) { + if (++m_plannerRetries <= m_limits.maxPlannerRetries) { + SentryReporter::addBreadcrumb("ai.agent.plan", QStringLiteral("malformed plan, retry %1").arg(m_plannerRetries)); + requestPlan(QStringLiteral("Your previous reply was not valid: %1. Reply with the JSON object only.").arg(err)); + return; + } + m_lastError = QStringLiteral("the model could not produce a valid plan (%1)").arg(err); + say(QStringLiteral("I could not turn that into a plan: %1.").arg(err)); + finish(State::Failed); + return; + } + if (!need.isEmpty()) { + // Dynamic discovery: the planner asked for docs it did not have. + QStringList added; + for (const QString& id : need) + if (m_registry.capability(id) && !m_docCapabilities.contains(id)) { m_docCapabilities << id; added << id; } + if (added.isEmpty() || ++m_plannerRetries > m_limits.maxPlannerRetries + 1) { + m_lastError = QStringLiteral("the model asked for unknown capabilities: %1").arg(need.join(", ")); + say(QStringLiteral("I could not find tools for: %1.").arg(need.join(", "))); + finish(State::Failed); + return; + } + SentryReporter::addBreadcrumb("ai.agent.plan", QStringLiteral("expanded capabilities: %1").arg(added.join(", "))); + requestPlan(); + return; + } + if (!answer.isEmpty()) { + // A question answered from context — no tools needed. + m_plan.title = QStringLiteral("AI: answer"); + finish(State::Completed, answer); + return; + } + plan.goal = m_plan.goal; + plan.capabilities = m_docCapabilities; + if (plan.title.isEmpty()) plan.title = plan.goal.left(48); + if (!plan.title.startsWith(QLatin1String("AI:"))) plan.title = QStringLiteral("AI: ") + plan.title; + if (plan.steps.size() > m_limits.maxSteps) plan.steps.resize(m_limits.maxSteps); + adoptPlan(std::move(plan)); + return; + } + + // ---- replan ---- + QVector steps; bool done = false; QString summary; QString err; + if (!parseReplanReply(text, &steps, &done, &summary, &err)) { + if (++m_plannerRetries <= m_limits.maxPlannerRetries) { + requestReplan(m_replanFailedIndex); + --m_replans; // a retry of the same replan round does not count + return; + } + m_lastError = QStringLiteral("the model could not repair the plan (%1)").arg(err); + finish(State::Failed); + return; + } + if (done) { finish(steps.isEmpty() && m_plan.allDone() ? State::Completed : State::Failed, summary); return; } + // Replace the remaining pending steps with the repaired tail. + for (Step& s : m_plan.steps) if (s.status == Step::Pending) s.status = Step::Skipped; + int room = m_limits.maxSteps - m_plan.steps.size(); + for (Step& s : steps) { if (room-- <= 0) break; m_plan.steps.push_back(s); } + emit planChanged(); + say(QStringLiteral("Adjusted the plan: %1 new step(s).").arg(steps.size())); + QTimer::singleShot(0, this, &AIAgentManager::executeNext); +} + +void AIAgentManager::onPlannerFailed(const QString& error) +{ + if (m_awaiting == Awaiting::None) return; + m_awaiting = Awaiting::None; + m_lastError = QStringLiteral("planner error: %1").arg(error); + say(QStringLiteral("The AI model failed: %1").arg(error)); + finish(State::Failed); +} + +void AIAgentManager::onPlannerStopped() +{ + if (m_awaiting == Awaiting::None) return; + m_awaiting = Awaiting::None; + if (!AIAgent::isTerminal(m_state)) finish(State::Cancelled); +} + +void AIAgentManager::adoptPlan(Plan plan) +{ + m_plan = std::move(plan); + emit planChanged(); + QStringList lines{QStringLiteral("Plan — %1").arg(m_plan.title.mid(4))}; + for (int i = 0; i < m_plan.steps.size(); ++i) { + const Step& s = m_plan.steps[i]; + lines << QStringLiteral("%1. %2%3").arg(i + 1).arg(s.tool, s.why.isEmpty() ? QString() : QStringLiteral(" — %1").arg(s.why)); + } + emit chatMessage(QStringLiteral("plan"), lines.join('\n'), false); + SentryReporter::addBreadcrumb("ai.agent.plan", QStringLiteral("%1 steps: %2").arg(m_plan.steps.size()).arg(m_plan.title)); + QTimer::singleShot(0, this, &AIAgentManager::executeNext); +} + +// --------------------------------------------------------------------------- +// execution + +void AIAgentManager::executeNext() +{ + if (AIAgent::isTerminal(m_state)) return; + if (m_cancelRequested) { finish(State::Cancelled); return; } + const int idx = m_plan.nextPendingIndex(); + if (idx < 0) { + setState(State::Verifying); + SentryReporter::addBreadcrumb("ai.agent.verify", QStringLiteral("all %1 steps done").arg(m_plan.steps.size())); + finish(State::Completed); + return; + } + if (idx >= m_limits.maxSteps) { + m_lastError = QStringLiteral("step limit (%1) reached").arg(m_limits.maxSteps); + finish(State::Failed); + return; + } + Step& s = m_plan.steps[idx]; + m_currentStep = idx; + + // ---- constrained protocol: validate before the call ever reaches the server ---- + QJsonObject coerced; QString err; QStringList warnings; + if (!m_registry.validateArguments(s.tool, s.arguments, &coerced, &err, &warnings)) { + Observation ob; ob.stepIndex = idx; ob.tool = s.tool; + ob.status = QStringLiteral("invalid_arguments"); ob.error = err; + m_observations << ob; + s.attempts++; s.status = Step::Failed; s.error = err; + sayTool(QStringLiteral("[%1] rejected before running: %2").arg(s.tool, err)); + emit planChanged(); + emit stepFinished(idx, false); + handleStepFailure(idx, ob); + return; + } + s.arguments = coerced; + + // ---- safety rail: destructive steps need a confirmation unless trusted ---- + const QString reason = AICapabilityRegistry::destructiveReason(s.tool, s.arguments); + if (!reason.isEmpty() && !m_trustedMode) { + m_pendingIndex = idx; + m_pendingReason = QStringLiteral("Step %1 (%2) %3").arg(idx + 1).arg(s.tool, reason); + setState(State::AwaitingConfirmation); + emit confirmationChanged(); + SentryReporter::addBreadcrumb("ai.agent.confirm", QStringLiteral("waiting: %1").arg(m_pendingReason)); + return; + } + runStep(idx); +} + +void AIAgentManager::openUndoGroupIfNeeded(const QString& tool) +{ + if (m_macroOpen || AICapabilityRegistry::isReadOnly(tool)) return; + QUndoStack* stack = m_undoStack ? m_undoStack : UndoManager::getSingleton()->stack(); + if (!stack) return; + stack->beginMacro(m_plan.title); + m_macroOpen = true; +} + +void AIAgentManager::closeUndoGroup() +{ + if (!m_macroOpen) return; + m_macroOpen = false; + QUndoStack* stack = m_undoStack ? m_undoStack : UndoManager::getSingleton()->stack(); + if (stack) stack->endMacro(); +} + +void AIAgentManager::runStep(int idx) +{ + Step& s = m_plan.steps[idx]; + s.status = Step::Running; + s.attempts++; + setState(State::Executing); + emit planChanged(); + SentryReporter::addBreadcrumb("ai.agent.step", QStringLiteral("%1/%2 %3 (attempt %4)").arg(idx + 1).arg(m_plan.steps.size()).arg(s.tool).arg(s.attempts)); + + openUndoGroupIfNeeded(s.tool); + const QJsonObject result = m_executor->callTool(s.tool, s.arguments); + + // Cancel may have been requested from inside the tool call (GUI event + // processing) — honour it before observing. + setState(State::Observing); + Observation ob = observationFromToolResult(idx, s.tool, result); + m_observations << ob; + + if (ob.status == QLatin1String("success")) { + s.status = Step::Succeeded; + s.error.clear(); + QString line = ob.raw.section('\n', 0, 0).trimmed(); + if (line.size() > 160) line = line.left(157) + QLatin1String("..."); + sayTool(QStringLiteral("[%1] %2").arg(s.tool, line.isEmpty() ? QStringLiteral("ok") : line)); + emit planChanged(); + emit stepFinished(idx, true); + QTimer::singleShot(0, this, &AIAgentManager::executeNext); + return; + } + + s.error = ob.error; + sayTool(QStringLiteral("[%1] failed: %2").arg(s.tool, ob.error.left(200))); + SentryReporter::addBreadcrumb("ai.agent.step", QStringLiteral("%1 failed: %2").arg(s.tool, ob.error.left(120)), "error"); + + const bool fatal = ob.error.contains(QLatin1String("Unknown tool"), Qt::CaseInsensitive) + || ob.error.contains(QLatin1String("could not be initialized"), Qt::CaseInsensitive); + if (!fatal && s.attempts < m_limits.maxAttemptsPerStep) { + // Recoverable: try the same call once more (transient failures). + s.status = Step::Pending; + SentryReporter::addBreadcrumb("ai.agent.retry", QStringLiteral("%1 retry").arg(s.tool)); + emit planChanged(); + QTimer::singleShot(0, this, &AIAgentManager::executeNext); + return; + } + s.status = Step::Failed; + emit planChanged(); + emit stepFinished(idx, false); + handleStepFailure(idx, ob); +} + +void AIAgentManager::handleStepFailure(int idx, const Observation& ob) +{ + Q_UNUSED(ob); + const Step& s = m_plan.steps[idx]; + const int n = ++m_failureCounts[s.signature()]; + if (n >= m_limits.maxRepeatedFailures) { + m_lastError = QStringLiteral("the same action failed %1 times: %2 — %3").arg(n).arg(s.tool, s.error.left(160)); + say(QStringLiteral("Stopping: %1").arg(m_lastError)); + finish(State::Failed); + return; + } + if (m_cancelRequested) { finish(State::Cancelled); return; } + if (m_replans >= m_limits.maxReplans) { + m_lastError = QStringLiteral("step %1 (%2) failed and the replan budget is spent: %3").arg(idx + 1).arg(s.tool, s.error.left(160)); + finish(State::Failed); + return; + } + m_plannerRetries = 0; + requestReplan(idx); +} + +void AIAgentManager::finish(State terminal, const QString& plannerSummary) +{ + closeUndoGroup(); + m_awaiting = Awaiting::None; + m_currentStep = -1; + QString summary; + if (!m_plan.steps.isEmpty()) summary = summarize(m_plan, m_observations, terminal); + if (!plannerSummary.isEmpty()) summary = summary.isEmpty() ? plannerSummary : plannerSummary + '\n' + summary; + if (summary.isEmpty()) { + switch (terminal) { + case State::Cancelled: summary = QStringLiteral("Cancelled."); break; + case State::Failed: summary = m_lastError.isEmpty() ? QStringLiteral("The task could not be completed.") : m_lastError; break; + default: summary = QStringLiteral("Done."); break; + } + } else if (terminal == State::Failed && !m_lastError.isEmpty() && !summary.contains(m_lastError)) { + summary += '\n' + m_lastError; + } + m_lastSummary = summary; + setState(terminal); + SentryReporter::addBreadcrumb(terminal == State::Completed ? "ai.agent.done" : (terminal == State::Cancelled ? "ai.agent.cancel" : "ai.agent.fail"), + QStringLiteral("%1 steps, %2 replans").arg(m_plan.steps.size()).arg(m_replans), + terminal == State::Failed ? "error" : "info"); + say(summary); + emit planChanged(); + emit taskFinished(terminal == State::Completed, summary); +} diff --git a/src/AIAgentManager.h b/src/AIAgentManager.h new file mode 100644 index 000000000..c41217a0e --- /dev/null +++ b/src/AIAgentManager.h @@ -0,0 +1,213 @@ +#ifndef AIAGENTMANAGER_H +#define AIAGENTMANAGER_H + +// AIAgentManager — the orchestration layer above AIChatManager/LLMManager +// (#1000 / #1001 / #1003, epic #818 Track C6). +// +// User -> AIAgentManager -> Planner -> Capability router -> Executor +// -> Observer -> (Verifier) -> Replan / Finish +// +// One bounded step at a time. ALL task state (plan, step status, attempts, +// observations) lives in this object, never in the prompt; the prompt is +// rebuilt from it on every planner call. The planner and the tool executor +// are injected interfaces so the whole state machine is unit-tested +// headless with scripted fakes — no LLM, no Ogre. +// +// Production wiring: `McpToolExecutor` (MCPServer::callTool) and +// `LlmPlannerBackend` (LLMManager::generateText). AIChatManager stays the +// conversation facade and delegates multi-step work here. + +#include "AIAgentTypes.h" +#include "AICapabilityRegistry.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class QUndoStack; +class MCPServer; + +/// Executes tools. Production: MCPServer. Tests: a scripted fake. +class AgentToolExecutor +{ +public: + virtual ~AgentToolExecutor() = default; + virtual QJsonArray toolList() = 0; + virtual QJsonObject callTool(const QString& name, const QJsonObject& args) = 0; +}; + +/// Asynchronous planner (LLM) backend. `request` must eventually emit exactly +/// one of completed / failed / stopped. +class AgentPlannerBackend : public QObject +{ + Q_OBJECT +public: + using QObject::QObject; + virtual bool available() const = 0; + virtual QString modelName() const { return {}; } + virtual void request(const QString& systemPrompt, const QString& userPrompt, int maxTokens) = 0; + virtual void stop() = 0; +signals: + void completed(const QString& text); + void failed(const QString& error); + void stopped(); +}; + +/// Production executor over the live MCP server. +class McpToolExecutor : public AgentToolExecutor +{ +public: + explicit McpToolExecutor(MCPServer* server); // defined in the .cpp: QPointer needs the full type + ~McpToolExecutor() override; + QJsonArray toolList() override; + QJsonObject callTool(const QString& name, const QJsonObject& args) override; +private: + QPointer m_server; +}; + +class AIAgentManager : public QObject +{ + Q_OBJECT + QML_ELEMENT + QML_SINGLETON + + Q_PROPERTY(QString state READ stateName NOTIFY stateChanged) + Q_PROPERTY(bool busy READ busy NOTIFY stateChanged) + Q_PROPERTY(QString planTitle READ planTitle NOTIFY planChanged) + Q_PROPERTY(QVariantList plan READ planModel NOTIFY planChanged) + Q_PROPERTY(int currentStep READ currentStep NOTIFY planChanged) + Q_PROPERTY(QString pendingConfirmation READ pendingConfirmation NOTIFY confirmationChanged) + Q_PROPERTY(bool trustedMode READ trustedMode WRITE setTrustedMode NOTIFY trustedModeChanged) + Q_PROPERTY(QString lastSummary READ lastSummary NOTIFY stateChanged) + Q_PROPERTY(QString recommendedModelName READ recommendedModelName CONSTANT) + +public: + static AIAgentManager* instance(); + static AIAgentManager* qmlInstance(QQmlEngine* engine, QJSEngine* scriptEngine); + static void kill(); + + // ---- dependency injection ---- + void setExecutor(std::shared_ptr executor); + /// Takes ownership (parented). Production creates an LlmPlannerBackend lazily. + void setPlanner(AgentPlannerBackend* planner); + void setLimits(const AIAgent::Limits& limits) { m_limits = limits; } + /// Undo stack the task's mutating steps are grouped on (default: UndoManager's). + void setUndoStack(QUndoStack* stack) { m_undoStack = stack; } + /// Extra text appended to every planner prompt (the scene summary). The + /// facade refreshes it per turn (#1021c). + void setContextProvider(std::function provider) { m_contextProvider = std::move(provider); } + + // ---- state ---- + AIAgent::State state() const { return m_state; } + QString stateName() const { return AIAgent::stateName(m_state); } + bool busy() const { return m_state != AIAgent::State::Idle && !AIAgent::isTerminal(m_state); } + const AIAgent::Plan& plan() const { return m_plan; } + const QVector& observations() const { return m_observations; } + QString planTitle() const { return m_plan.title; } + QVariantList planModel() const; + int currentStep() const { return m_currentStep; } + QString pendingConfirmation() const { return m_pendingReason; } + QString lastSummary() const { return m_lastSummary; } + int replanCount() const { return m_replans; } + QString lastError() const { return m_lastError; } + bool trustedMode() const { return m_trustedMode; } + void setTrustedMode(bool on); + QString recommendedModelName() const; + const AICapabilityRegistry& registry() const { return m_registry; } + + // ---- control ---- + /// Start a task. Returns false (with a chat error) when busy, no executor, + /// or the planner is unavailable (no model loaded). + Q_INVOKABLE bool startTask(const QString& request); + Q_INVOKABLE void cancel(); + /// Answer a pending destructive-step confirmation. `alwaysAllow` also + /// switches trusted mode on for the session. + Q_INVOKABLE void confirmPendingStep(bool approve, bool alwaysAllow = false); + + // ---- pure helpers (unit-tested) ---- + static QString extractJsonObject(const QString& text); + /// Parses a planner reply into `out`. `needCapabilities` receives the + /// ids when the model asked for more docs instead of planning; + /// `answer` receives a direct answer when the model returned a summary + /// with no steps (a question about the scene). + static bool parsePlanReply(const QString& text, AIAgent::Plan* out, + QStringList* needCapabilities, QString* answer, QString* error); + static bool parseReplanReply(const QString& text, QVector* steps, + bool* done, QString* summary, QString* error); + +signals: + void stateChanged(); + void planChanged(); + void confirmationChanged(); + void trustedModeChanged(); + /// A line for the chat transcript: role = "assistant" | "tool" | "plan". + void chatMessage(const QString& role, const QString& text, bool isTool); + void stepFinished(int index, bool ok); + void taskFinished(bool ok, const QString& summary); + +private slots: + void onPlannerCompleted(const QString& text); + void onPlannerFailed(const QString& error); + void onPlannerStopped(); + +private: + explicit AIAgentManager(QObject* parent = nullptr); + ~AIAgentManager() override; + + enum class Awaiting { None, Plan, Replan }; + + void setState(AIAgent::State s); + void ensurePlanner(); + void requestPlan(const QString& extraInstruction = QString()); + void requestReplan(int failedIndex); + QString systemPrompt(const QStringList& capabilityIds) const; + QString sceneContext() const; + void adoptPlan(AIAgent::Plan plan); + void executeNext(); + void runStep(int index); + void handleStepFailure(int index, const AIAgent::Observation& ob); + void openUndoGroupIfNeeded(const QString& tool); + void closeUndoGroup(); + void finish(AIAgent::State terminal, const QString& plannerSummary = QString()); + void say(const QString& text); + void sayTool(const QString& text); + + static AIAgentManager* s_instance; + + std::shared_ptr m_executor; + AgentPlannerBackend* m_planner = nullptr; + QUndoStack* m_undoStack = nullptr; + std::function m_contextProvider; + AICapabilityRegistry m_registry; + AIAgent::Limits m_limits; + + AIAgent::State m_state = AIAgent::State::Idle; + Awaiting m_awaiting = Awaiting::None; + AIAgent::Plan m_plan; + QVector m_observations; + QStringList m_docCapabilities; // capabilities whose docs the planner has seen + QHash m_failureCounts; // step signature → failures + int m_currentStep = -1; + int m_pendingIndex = -1; + QString m_pendingReason; + int m_replans = 0; + int m_plannerRetries = 0; + int m_replanFailedIndex = -1; + bool m_macroOpen = false; + bool m_cancelRequested = false; + bool m_trustedMode = false; + QString m_lastSummary; + QString m_lastError; +}; + +#endif // AIAGENTMANAGER_H diff --git a/src/AIAgentManager_test.cpp b/src/AIAgentManager_test.cpp new file mode 100644 index 000000000..7f05fbbeb --- /dev/null +++ b/src/AIAgentManager_test.cpp @@ -0,0 +1,459 @@ +// Headless tests for the AI agent state machine (#1001). A scripted planner +// stands in for the LLM and a scripted executor for the MCP server, so the +// whole plan → execute → observe → replan → finish loop runs without a +// model, Ogre, or a GUI. The custom test main owns the QApplication. +#include + +#include "AIAgentManager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace AIAgent; + +namespace { + +QJsonObject prop(const char* type, const char* desc, QJsonArray enumVals = {}) +{ + QJsonObject p{{"type", type}, {"description", desc}}; + if (!enumVals.isEmpty()) p["enum"] = enumVals; + return p; +} +QJsonObject tool(const char* name, const char* desc, QJsonObject props, QJsonArray required = {}) +{ + QJsonObject schema{{"type", "object"}, {"properties", props}}; + if (!required.isEmpty()) schema["required"] = required; + return {{"name", name}, {"description", desc}, {"inputSchema", schema}}; +} +QJsonObject ok(const QString& text) +{ + return {{"content", QJsonArray{QJsonObject{{"type", "text"}, {"text", text}}}}}; +} +QJsonObject err(const QString& text) +{ + return {{"isError", true}, {"content", QJsonArray{QJsonObject{{"type", "text"}, {"text", text}}}}}; +} + +/// Scripted MCP stand-in: per-tool queues of results (default: success), +/// a call log, and an optional undo stack it pushes a command onto for +/// every mutating call (so undo grouping is observable). +class FakeExecutor : public AgentToolExecutor +{ +public: + QJsonArray toolList() override + { + return { + tool("get_scene_info", "Scene info.", {}), + tool("create_primitive", "Create a primitive.", {{"type", prop("string", "kind", {"box", "sphere"})}, {"name", prop("string", "name")}}, {"type"}), + tool("transform_mesh", "Transform.", {{"name", prop("string", "node")}, {"scale", prop("array", "xyz")}}, {"name"}), + tool("apply_material", "Apply.", {{"mesh", prop("string", "m")}, {"material", prop("string", "mat")}}, {"mesh", "material"}), + tool("auto_rig", "Rig.", {{"template", prop("string", "t")}}), + tool("delete_entity", "Delete.", {{"entity_name", prop("string", "e")}}, {"entity_name"}), + tool("export_mesh", "Export.", {{"output_path", prop("string", "p")}}, {"output_path"}), + }; + } + QJsonObject callTool(const QString& name, const QJsonObject& args) override + { + calls << name; + callArgs << args; + if (onCall) onCall(name); + if (undoStack && !AICapabilityRegistry::isReadOnly(name)) + undoStack->push(new QUndoCommand(name)); + auto& q = scripted[name]; + if (!q.isEmpty()) return q.takeFirst(); + return ok(QStringLiteral("Created %1").arg(args.value("name").toString("thing"))); + } + QStringList calls; + QList callArgs; + QHash> scripted; + QUndoStack* undoStack = nullptr; + std::function onCall; +}; + +/// Scripted LLM: replies are dequeued in order and delivered asynchronously +/// (a queued call, like the real worker thread). Records every prompt. +class FakePlanner : public AgentPlannerBackend +{ +public: + using AgentPlannerBackend::AgentPlannerBackend; + bool available() const override { return isAvailable; } + void request(const QString& sys, const QString& user, int) override + { + systemPrompts << sys; userPrompts << user; + if (replies.isEmpty()) { QTimer::singleShot(0, this, [this]() { emit failed("no scripted reply"); }); return; } + const QString r = replies.takeFirst(); + QTimer::singleShot(0, this, [this, r]() { if (!stoppedFlag) emit completed(r); }); + } + void stop() override { stoppedFlag = true; QTimer::singleShot(0, this, [this]() { emit stopped(); }); } + bool isAvailable = true; + bool stoppedFlag = false; + QStringList replies, systemPrompts, userPrompts; +}; + +QString planJson(const QList>& steps, const QString& title = "test plan") +{ + QJsonArray arr; + for (const auto& s : steps) arr.append(QJsonObject{{"tool", s.first}, {"arguments", s.second}, {"why", "because"}}); + return QString::fromUtf8(QJsonDocument(QJsonObject{{"title", title}, {"steps", arr}}).toJson(QJsonDocument::Compact)); +} +QString replanJson(const QList>& steps, bool done = false, const QString& summary = {}) +{ + QJsonArray arr; + for (const auto& s : steps) arr.append(QJsonObject{{"tool", s.first}, {"arguments", s.second}}); + QJsonObject o{{"steps", arr}, {"done", done}}; + if (!summary.isEmpty()) o["summary"] = summary; + return QString::fromUtf8(QJsonDocument(o).toJson(QJsonDocument::Compact)); +} + +/// Pump the event loop until the manager reaches a terminal state or the +/// given one (confirmation waits), with a hard timeout. +bool pumpUntil(AIAgentManager* m, std::function pred, int ms = 3000) +{ + QElapsedTimer t; t.start(); + while (t.elapsed() < ms) { + QCoreApplication::processEvents(QEventLoop::AllEvents, 20); + if (pred()) return true; + } + return pred(); +} +bool pumpToEnd(AIAgentManager* m) { return pumpUntil(m, [m]() { return isTerminal(m->state()); }); } + +struct AgentFixture : public ::testing::Test { + AIAgentManager* m = nullptr; + std::shared_ptr exec; + FakePlanner* planner = nullptr; // owned by the manager + QUndoStack undo; + QStringList transcript; + + void SetUp() override + { + AIAgentManager::kill(); + m = AIAgentManager::instance(); + exec = std::make_shared(); + exec->undoStack = &undo; + m->setExecutor(exec); + planner = new FakePlanner(); + m->setPlanner(planner); + m->setUndoStack(&undo); + m->setTrustedMode(false); + QObject::connect(m, &AIAgentManager::chatMessage, [this](const QString& role, const QString& text, bool) { + transcript << role + ": " + text; + }); + } + void TearDown() override { AIAgentManager::kill(); } +}; + +} // namespace + +// --------------------------------------------------------------------------- + +TEST(AIAgentManagerParse, ExtractsTheFirstBalancedObjectEvenWithoutTheOpeningBrace) +{ + EXPECT_EQ(AIAgentManager::extractJsonObject("Sure! {\"a\": {\"b\": 1}} trailing"), "{\"a\": {\"b\": 1}}"); + EXPECT_EQ(AIAgentManager::extractJsonObject("\"title\": \"x\", \"steps\": []}"), "{\"title\": \"x\", \"steps\": []}"); + EXPECT_EQ(AIAgentManager::extractJsonObject("{\"s\": \"a } inside a string\"}"), "{\"s\": \"a } inside a string\"}"); + EXPECT_TRUE(AIAgentManager::extractJsonObject("no json here").isEmpty()); +} + +TEST(AIAgentManagerParse, PlanReplyVariants) +{ + Plan p; QStringList need; QString answer, err; + ASSERT_TRUE(AIAgentManager::parsePlanReply(planJson({{"get_scene_info", {}}, {"create_primitive", {{"type", "box"}}}}), &p, &need, &answer, &err)); + EXPECT_EQ(p.steps.size(), 2); + EXPECT_EQ(p.steps[1].arguments["type"].toString(), "box"); + + ASSERT_TRUE(AIAgentManager::parsePlanReply("{\"need_capabilities\": [\"rigging\", \"uv\"]}", &p, &need, &answer, &err)); + EXPECT_EQ(need, QStringList({"rigging", "uv"})); + + ASSERT_TRUE(AIAgentManager::parsePlanReply("{\"summary\": \"There are 2 entities.\"}", &p, &need, &answer, &err)); + EXPECT_EQ(answer, "There are 2 entities."); + + EXPECT_FALSE(AIAgentManager::parsePlanReply("I think we should...", &p, &need, &answer, &err)); + EXPECT_FALSE(AIAgentManager::parsePlanReply("{\"title\": \"t\", \"steps\": [{\"arguments\": {}}]}", &p, &need, &answer, &err)) << "a step without a tool"; + // legacy v1 field names are accepted + ASSERT_TRUE(AIAgentManager::parsePlanReply("{\"steps\": [{\"command\": \"get_scene_info\", \"args\": {}}]}", &p, &need, &answer, &err)); + EXPECT_EQ(p.steps[0].tool, "get_scene_info"); +} + +TEST_F(AgentFixture, FiveDependentStepsRunInOrderWithObservableStateAndOneUndoGroup) +{ + planner->replies << planJson({ + {"get_scene_info", {}}, + {"create_primitive", {{"type", "box"}, {"name", "Crate"}}}, + {"transform_mesh", {{"name", "Crate"}, {"scale", QJsonArray{2, 2, 2}}}}, + {"apply_material", {{"mesh", "Crate"}, {"material", "Wood"}}}, + {"auto_rig", {{"template", "generic"}}}, + }, "build a crate"); + exec->scripted["get_scene_info"] << ok("Scene Information:\n- Scene Nodes: 1\n- Entities: 1\n - Floor (material: BaseWhite)"); + exec->scripted["auto_rig"] << ok("{\"applied\":true,\"boneCount\":7,\"template\":\"generic\",\"skinned\":false}"); + + QSignalSpy stateSpy(m, &AIAgentManager::stateChanged); + ASSERT_TRUE(m->startTask("build a crate and rig it")); + EXPECT_EQ(m->state(), State::Planning); + ASSERT_TRUE(pumpToEnd(m)); + + EXPECT_EQ(m->state(), State::Completed); + EXPECT_EQ(exec->calls, QStringList({"get_scene_info", "create_primitive", "transform_mesh", "apply_material", "auto_rig"})); + ASSERT_EQ(m->plan().steps.size(), 5); + for (const Step& s : m->plan().steps) EXPECT_EQ(s.status, Step::Succeeded); + EXPECT_EQ(m->observations().size(), 5); + EXPECT_DOUBLE_EQ(m->observations()[0].facts["entities"].toDouble(), 1.0) << "facts parsed from the tool text"; + EXPECT_EQ(m->observations()[4].facts["boneCount"].toInt(), 7) << "facts lifted from JSON results"; + EXPECT_EQ(m->plan().title, "AI: build a crate"); + // ONE undo group named after the task, covering the 4 mutating steps. + EXPECT_EQ(undo.count(), 1); + EXPECT_EQ(undo.text(0), "AI: build a crate"); + EXPECT_TRUE(m->lastSummary().startsWith("Done — 5 of 5 steps succeeded.")) << m->lastSummary().toStdString(); + EXPECT_GT(stateSpy.count(), 4) << "state transitions are observable"; + // transcript: the plan card, one line per tool, the summary + EXPECT_TRUE(transcript.first().startsWith("plan: Plan — build a crate")); + EXPECT_EQ(transcript.filter(QRegularExpression("^tool: ")).size(), 5); + // the planner saw the compact capability index, not 170 tool docs + EXPECT_TRUE(planner->systemPrompts.first().contains("Capabilities (groups of tools):")); + EXPECT_TRUE(planner->systemPrompts.first().contains("- auto_rig:")) << "keyword routing exposed rigging docs"; +} + +TEST_F(AgentFixture, RecoverableErrorIsRetriedOnceThenSucceeds) +{ + planner->replies << planJson({{"create_primitive", {{"type", "box"}, {"name", "A"}}}, {"transform_mesh", {{"name", "A"}}}}); + exec->scripted["transform_mesh"] << err("Error: node 'A' busy") << ok("Transformed A"); + ASSERT_TRUE(m->startTask("make a box and move it")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Completed); + EXPECT_EQ(exec->calls, QStringList({"create_primitive", "transform_mesh", "transform_mesh"})); + EXPECT_EQ(m->plan().steps[1].attempts, 2); + EXPECT_EQ(m->replanCount(), 0) << "a retry is not a replan"; +} + +TEST_F(AgentFixture, PersistentFailureTriggersOneReplanAndTheRepairedTailSucceeds) +{ + planner->replies << planJson({{"create_primitive", {{"type", "box"}, {"name", "A"}}}, {"apply_material", {{"mesh", "A"}, {"material", "Gold"}}}}); + // Replan: the model realises Gold does not exist and applies Wood instead. + planner->replies << replanJson({{"apply_material", {{"mesh", "A"}, {"material", "Wood"}}}}); + exec->scripted["apply_material"] << err("Error: material 'Gold' not found") << err("Error: material 'Gold' not found") << ok("Applied Wood to A"); + ASSERT_TRUE(m->startTask("golden box")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Completed) << m->lastSummary().toStdString(); + EXPECT_EQ(m->replanCount(), 1); + EXPECT_EQ(exec->calls.size(), 4); + EXPECT_EQ(exec->callArgs.last()["material"].toString(), "Wood"); + // the replan prompt carried the structured observations, not raw dumps + ASSERT_EQ(planner->userPrompts.size(), 2); + EXPECT_TRUE(planner->userPrompts[1].contains("\"status\":\"error\"")); + EXPECT_TRUE(planner->userPrompts[1].contains("Step 2 (apply_material) failed")); + // old pending step marked skipped, repaired step appended + ASSERT_EQ(m->plan().steps.size(), 3); + EXPECT_EQ(m->plan().steps[1].status, Step::Failed); + EXPECT_EQ(m->plan().steps[2].status, Step::Succeeded); +} + +TEST_F(AgentFixture, RepeatedIdenticalFailingActionIsDetectedAndStopsTheTask) +{ + Limits lim; lim.maxAttemptsPerStep = 1; lim.maxReplans = 5; lim.maxRepeatedFailures = 2; + m->setLimits(lim); + planner->replies << planJson({{"apply_material", {{"mesh", "A"}, {"material", "Gold"}}}}); + // The model stubbornly re-issues the exact same call. + planner->replies << replanJson({{"apply_material", {{"mesh", "A"}, {"material", "Gold"}}}}); + planner->replies << replanJson({{"apply_material", {{"mesh", "A"}, {"material", "Gold"}}}}); + exec->scripted["apply_material"] << err("Error: no") << err("Error: no") << err("Error: no") << err("Error: no"); + ASSERT_TRUE(m->startTask("gold")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Failed); + EXPECT_EQ(exec->calls.size(), 2) << "stopped after the second identical failure, not after 5 replans"; + EXPECT_TRUE(m->lastError().contains("same action failed 2 times")) << m->lastError().toStdString(); +} + +TEST_F(AgentFixture, ReplanBudgetExhaustedFailsCleanly) +{ + Limits lim; lim.maxAttemptsPerStep = 1; lim.maxReplans = 1; lim.maxRepeatedFailures = 10; + m->setLimits(lim); + planner->replies << planJson({{"apply_material", {{"mesh", "A"}, {"material", "X"}}}}); + planner->replies << replanJson({{"apply_material", {{"mesh", "A"}, {"material", "Y"}}}}); + exec->scripted["apply_material"] << err("Error: no X") << err("Error: no Y"); + ASSERT_TRUE(m->startTask("paint")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Failed); + EXPECT_TRUE(m->lastError().contains("replan budget")) << m->lastError().toStdString(); + // The failing calls still pushed commands (the fake mimics tools that + // partially mutate before erroring), so ONE group exists — and it must be + // CLOSED: a command pushed after the task lands as a new entry, not + // inside a dangling macro. + EXPECT_EQ(undo.count(), 1); + undo.push(new QUndoCommand("later")); + EXPECT_EQ(undo.count(), 2) << "the undo group was left open"; + EXPECT_EQ(undo.text(1), "later"); +} + +TEST_F(AgentFixture, CancellationStopsBetweenStepsAndDuringPlanning) +{ + planner->replies << planJson({{"create_primitive", {{"type", "box"}, {"name", "A"}}}, {"create_primitive", {{"type", "box"}, {"name", "B"}}}, {"create_primitive", {{"type", "box"}, {"name", "C"}}}}); + exec->onCall = [this](const QString&) { if (exec->calls.size() == 1) m->cancel(); }; + ASSERT_TRUE(m->startTask("three boxes")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Cancelled); + EXPECT_EQ(exec->calls.size(), 1) << "no further tool calls after cancel"; + EXPECT_TRUE(m->lastSummary().startsWith("Cancelled after")) << m->lastSummary().toStdString(); + EXPECT_FALSE(m->busy()); + + // cancel while the planner is thinking: the reply that arrives later is ignored + AIAgentManager::kill(); SetUp(); + planner->replies << planJson({{"create_primitive", {{"type", "box"}}}}); + ASSERT_TRUE(m->startTask("a box")); + m->cancel(); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Cancelled); + QCoreApplication::processEvents(QEventLoop::AllEvents, 50); + EXPECT_TRUE(exec->calls.isEmpty()); +} + +TEST_F(AgentFixture, InvalidArgumentsNeverReachTheToolAndTriggerAReplan) +{ + planner->replies << planJson({{"create_primitive", {{"type", "pyramid"}}}}); // not in the enum + planner->replies << replanJson({{"create_primitive", {{"type", "box"}}}}); + ASSERT_TRUE(m->startTask("a pyramid")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Completed); + EXPECT_EQ(exec->calls, QStringList({"create_primitive"})) << "the invalid call was never executed"; + EXPECT_EQ(m->observations().first().status, "invalid_arguments"); + EXPECT_TRUE(planner->userPrompts[1].contains("not one of box|sphere")); +} + +TEST_F(AgentFixture, UnknownToolFromThePlannerIsRejectedWithoutExecution) +{ + Limits lim; lim.maxReplans = 0; m->setLimits(lim); + planner->replies << planJson({{"make_it_pretty", {}}}); + ASSERT_TRUE(m->startTask("pretty")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Failed); + EXPECT_TRUE(exec->calls.isEmpty()); +} + +TEST_F(AgentFixture, DestructiveStepWaitsForConfirmationUnlessTrusted) +{ + planner->replies << planJson({{"delete_entity", {{"entity_name", "Cube"}}}, {"create_primitive", {{"type", "box"}}}}); + ASSERT_TRUE(m->startTask("replace the cube")); + ASSERT_TRUE(pumpUntil(m, [this]() { return m->state() == State::AwaitingConfirmation; })); + EXPECT_TRUE(exec->calls.isEmpty()) << "nothing runs before the answer"; + EXPECT_TRUE(m->pendingConfirmation().contains("deletes 'Cube'")) << m->pendingConfirmation().toStdString(); + + m->confirmPendingStep(false); // deny → skipped, the rest continues + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(exec->calls, QStringList({"create_primitive"})); + EXPECT_EQ(m->plan().steps[0].status, Step::Skipped); + EXPECT_EQ(m->observations().first().status, "denied"); + EXPECT_EQ(m->state(), State::Completed); + + // approve path + AIAgentManager::kill(); SetUp(); + planner->replies << planJson({{"delete_entity", {{"entity_name", "Cube"}}}}); + ASSERT_TRUE(m->startTask("delete the cube")); + ASSERT_TRUE(pumpUntil(m, [this]() { return m->state() == State::AwaitingConfirmation; })); + m->confirmPendingStep(true); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(exec->calls, QStringList({"delete_entity"})); + EXPECT_EQ(m->state(), State::Completed); + + // trusted mode: no pause at all + AIAgentManager::kill(); SetUp(); + m->setTrustedMode(true); + planner->replies << planJson({{"delete_entity", {{"entity_name", "Cube"}}}}); + ASSERT_TRUE(m->startTask("delete the cube")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Completed); + EXPECT_EQ(exec->calls, QStringList({"delete_entity"})); + m->setTrustedMode(false); +} + +TEST_F(AgentFixture, PlannerCanAskForMoreCapabilitiesBeforePlanning) +{ + // "make it shiny" routes to materials/scene; the model asks for rigging docs first. + planner->replies << "{\"need_capabilities\": [\"rigging\"]}"; + planner->replies << planJson({{"auto_rig", {{"template", "humanoid"}}}}); + ASSERT_TRUE(m->startTask("make it shiny")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Completed); + ASSERT_EQ(planner->systemPrompts.size(), 2); + EXPECT_FALSE(planner->systemPrompts[0].contains("- auto_rig:")) << "first prompt: only routed capabilities"; + EXPECT_TRUE(planner->systemPrompts[1].contains("- auto_rig:")) << "second prompt: expanded on request"; + EXPECT_EQ(exec->calls, QStringList({"auto_rig"})); +} + +TEST_F(AgentFixture, QuestionIsAnsweredWithoutRunningTools) +{ + planner->replies << "{\"summary\": \"The scene holds one entity, Floor.\"}"; + ASSERT_TRUE(m->startTask("what is in the scene?")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Completed); + EXPECT_TRUE(exec->calls.isEmpty()); + EXPECT_EQ(m->lastSummary(), "The scene holds one entity, Floor."); + EXPECT_EQ(undo.count(), 0); +} + +TEST_F(AgentFixture, MalformedPlannerOutputIsRetriedThenFails) +{ + planner->replies << "I would love to help!" << "still no json" << "nope"; + ASSERT_TRUE(m->startTask("do things")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Failed); + EXPECT_EQ(planner->userPrompts.size(), 3) << "1 attempt + maxPlannerRetries"; + EXPECT_TRUE(planner->userPrompts[1].contains("previous reply was not valid")); + EXPECT_TRUE(exec->calls.isEmpty()); +} + +TEST_F(AgentFixture, RefusesToStartWithoutAModelOrWhileBusy) +{ + planner->isAvailable = false; + EXPECT_FALSE(m->startTask("anything")); + EXPECT_EQ(m->state(), State::Idle); + EXPECT_TRUE(transcript.last().contains("No AI model is loaded")); + + planner->isAvailable = true; + planner->replies << planJson({{"get_scene_info", {}}}); + ASSERT_TRUE(m->startTask("scene?")); + EXPECT_FALSE(m->startTask("another")) << "busy"; + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_EQ(m->state(), State::Completed); + EXPECT_EQ(undo.count(), 0) << "read-only steps never open an undo group"; +} + +TEST_F(AgentFixture, SceneContextIsInjectedIntoEveryPlannerPrompt) +{ + m->setContextProvider([]() { return QStringLiteral("Entities: Floor, Crate"); }); + planner->replies << planJson({{"get_scene_info", {}}}); + ASSERT_TRUE(m->startTask("x")); + ASSERT_TRUE(pumpToEnd(m)); + EXPECT_TRUE(planner->systemPrompts.first().contains("Scene state:\nEntities: Floor, Crate")); +} + +TEST(AIAgentObservation, ParsesFactsArtifactsAndErrorsFromToolText) +{ + Observation ob = observationFromToolResult(0, "get_mesh_info", + ok("Mesh Information for Wolf:\n- Vertices: 12,345\n- Triangles: 20000\n- Bones: 24\nHas skeleton: yes")); + EXPECT_EQ(ob.status, "success"); + EXPECT_DOUBLE_EQ(ob.facts["vertices"].toDouble(), 12345.0); + EXPECT_DOUBLE_EQ(ob.facts["triangles"].toDouble(), 20000.0); + EXPECT_DOUBLE_EQ(ob.facts["bones"].toDouble(), 24.0); + EXPECT_TRUE(ob.facts["hasSkeleton"].toBool()); + + ob = observationFromToolResult(1, "export_mesh", ok("Exported mesh to /tmp/out/wolf.glb (1 skin)")); + EXPECT_EQ(ob.artifacts, QStringList({"/tmp/out/wolf.glb"})); + + ob = observationFromToolResult(2, "apply_material", err("Error: material 'Gold' not found\nAvailable: Wood")); + EXPECT_EQ(ob.status, "error"); + EXPECT_EQ(ob.error, "Error: material 'Gold' not found"); + EXPECT_TRUE(ob.raw.contains("Available: Wood")) << "raw is kept for the transcript"; + EXPECT_FALSE(ob.toPromptLine().contains("Available: Wood")) << "but never copied into the prompt"; + + ob = observationFromToolResult(3, "auto_rig", ok("{\"applied\":true,\"boneCount\":19,\"fallbackReason\":\"UniRig unavailable\"}")); + EXPECT_EQ(ob.facts["boneCount"].toInt(), 19); + ASSERT_EQ(ob.warnings.size(), 1); + EXPECT_TRUE(ob.warnings.first().startsWith("fallback:")); +} diff --git a/src/AIAgentTypes.cpp b/src/AIAgentTypes.cpp new file mode 100644 index 000000000..bd36137b8 --- /dev/null +++ b/src/AIAgentTypes.cpp @@ -0,0 +1,207 @@ +#include "AIAgentTypes.h" + +#include +#include + +namespace AIAgent { + +QString Step::signature() const +{ + QJsonObject o{{"tool", tool}, {"arguments", arguments}}; + return QString::fromUtf8(QJsonDocument(o).toJson(QJsonDocument::Compact)); +} + +QJsonObject Step::toJson() const +{ + return { + {"tool", tool}, + {"arguments", arguments}, + {"why", why}, + {"status", statusName(status)}, + {"attempts", attempts}, + {"error", error}, + }; +} + +QJsonObject Observation::toJson(bool includeRaw) const +{ + QJsonObject o{ + {"step", stepIndex}, + {"tool", tool}, + {"status", status}, + {"artifacts", QJsonArray::fromStringList(artifacts)}, + {"facts", facts}, + {"warnings", QJsonArray::fromStringList(warnings)}, + }; + if (!error.isEmpty()) o["error"] = error; + if (includeRaw) o["raw"] = raw; + return o; +} + +QString Observation::toPromptLine() const +{ + QJsonObject o{{"step", stepIndex + 1}, {"tool", tool}, {"status", status}}; + if (!facts.isEmpty()) o["facts"] = facts; + if (!artifacts.isEmpty()) o["artifacts"] = QJsonArray::fromStringList(artifacts); + if (!error.isEmpty()) o["error"] = error.left(200); + return QString::fromUtf8(QJsonDocument(o).toJson(QJsonDocument::Compact)); +} + +int Plan::nextPendingIndex() const +{ + for (int i = 0; i < steps.size(); ++i) + if (steps[i].status == Step::Pending) return i; + return -1; +} + +bool Plan::allDone() const +{ + for (const Step& s : steps) + if (s.status == Step::Pending || s.status == Step::Running) return false; + return true; +} + +QJsonObject Plan::toJson() const +{ + QJsonArray arr; + for (const Step& s : steps) arr.append(s.toJson()); + return { + {"title", title}, + {"goal", goal}, + {"capabilities", QJsonArray::fromStringList(capabilities)}, + {"steps", arr}, + }; +} + +// --------------------------------------------------------------------------- + +Observation observationFromToolResult(int stepIndex, const QString& tool, + const QJsonObject& toolResult) +{ + Observation ob; + ob.stepIndex = stepIndex; + ob.tool = tool; + + QString text; + const QJsonArray content = toolResult["content"].toArray(); + for (const QJsonValue& v : content) { + const QJsonObject c = v.toObject(); + if (c["type"].toString() == QLatin1String("text") || c.contains("text")) { + if (!text.isEmpty()) text += '\n'; + text += c["text"].toString(); + } + } + if (text.isEmpty() && !content.isEmpty()) + text = QString::fromUtf8(QJsonDocument(content).toJson(QJsonDocument::Compact)); + if (text.isEmpty() && toolResult.contains("error")) + text = toolResult["error"].toString(); + ob.raw = text; + + const bool isError = toolResult["isError"].toBool() + || text.trimmed().startsWith(QLatin1String("Error"), Qt::CaseInsensitive); + ob.status = isError ? QStringLiteral("error") : QStringLiteral("success"); + if (isError) { + ob.error = text.section('\n', 0, 0).trimmed(); + if (ob.error.isEmpty()) ob.error = QStringLiteral("tool reported an error"); + } + + // ---- facts: "