From a4dae3eaf61864c0680748b3d8cbe24f0e6d6eb1 Mon Sep 17 00:00:00 2001 From: ketd <94940923+ketd@users.noreply.github.com> Date: Fri, 6 Nov 2026 06:03:32 -0500 Subject: [PATCH] =?UTF-8?q?feat(sdk):=20=E5=90=8C=E6=AD=A5=E6=A1=8C?= =?UTF-8?q?=E9=9D=A2=E7=AB=AF=20SDK=20=E5=B9=B6=E6=9B=B4=E6=96=B0=E5=88=9B?= =?UTF-8?q?=E4=BD=9C=E6=8C=87=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SDK 与 aoe-desktop dev 的 src/main/workbench/sdk 对齐：Agent 连续会话与 instructions、agentTool、 ASR 断线恢复与续传客户端、音频能力契约、页面主题声明、WorkbenchHostError 拆分；保留模板自有的 DEFAULT_LLM_MODEL（文本调用未指定模型时补 gemini-3.8-flash）。 - manifest 契约与构建扫描支持 agent-tool-details、agent.instructions 等特性；外部 Host 连接同步。 - 指引：Agent 连续会话与工具说明；平台尚未接入嵌入模型，暂不介绍 embed 与本地向量工具，检索只介绍 rerank； 媒体模型清单由 Floatboat 远端配置，media 恒为 available，暂无模型的种类需显式传 model。 Co-Authored-By: Claude Opus 5.5 --- .agents/skills/floatboat-workbench/SKILL.md | 2 +- .../floatboat-workbench/references/agent.md | 36 ++- .../references/engineering.md | 2 +- .../floatboat-workbench/references/llm.md | 23 +- .../references/workflows.md | 2 +- sdk/agent-contracts.mts | 70 ++++- sdk/asr-client.mts | 293 ++++++++++++++++++ sdk/asr-contracts.mts | 16 +- sdk/audio-contracts.mts | 49 +++ sdk/browser-asr.mts | 145 +++++++++ sdk/browser.mts | 229 +++++++++++++- sdk/host-contracts.mts | 51 ++- sdk/host-error.mts | 24 ++ sdk/host.mts | 71 +++-- sdk/runtime-contracts.mts | 6 + sdk/runtime.mts | 85 ++++- tests/host-connection.test.mjs | 56 ++++ tooling/compiler/host-operations.ts | 39 ++- tooling/contracts/workbenchManifest.ts | 19 +- tooling/contracts/workbenchOperations.ts | 13 +- tooling/host-connection.mjs | 45 ++- 21 files changed, 1174 insertions(+), 102 deletions(-) create mode 100644 sdk/asr-client.mts create mode 100644 sdk/audio-contracts.mts create mode 100644 sdk/browser-asr.mts create mode 100644 sdk/host-error.mts diff --git a/.agents/skills/floatboat-workbench/SKILL.md b/.agents/skills/floatboat-workbench/SKILL.md index da5f122..1d1357f 100644 --- a/.agents/skills/floatboat-workbench/SKILL.md +++ b/.agents/skills/floatboat-workbench/SKILL.md @@ -36,7 +36,7 @@ These topics are also available without installing dependencies through `node to | Browser connection, theme and Host contributions | [ui](references/ui.md) | | Business tables, migrations and transaction boundaries | [database](references/database.md) | | Workflow/Combo declarations and portable file resources | [resources](references/resources.md) | -| Model text, structured output, streaming, conversations, tool calls, reasoning, image input, speech, image generation, Jev decisions, embeddings, reranking or model list | [llm](references/llm.md) | +| Model text, structured output, streaming, conversations, tool calls, reasoning, image input, speech, image generation, Jev decisions, reranking or model list | [llm](references/llm.md) | | Agent tasks with tools and an explicit execution directory | [agent](references/agent.md) | | Streaming speech recognition | [asr](references/asr.md) | | Starting or observing a declared Workflow | [workflows](references/workflows.md) | diff --git a/.agents/skills/floatboat-workbench/references/agent.md b/.agents/skills/floatboat-workbench/references/agent.md index fd1be19..4b5cbde 100644 --- a/.agents/skills/floatboat-workbench/references/agent.md +++ b/.agents/skills/floatboat-workbench/references/agent.md @@ -1,6 +1,6 @@ # Agent tasks -Declare `agent` in manifest.capabilities. On the server, use `app.host.agent.start({ prompt, workingDirectory, idempotencyKey, model?, timeoutMs? })` for reasoning plus tool execution. workingDirectory is an explicit real absolute execution directory; it is not implicitly the workspace cwd. A declared resource directory can provide that path. The Host fixes workspace/session ownership; callers do not select sessionId or workspaceId. +Declare `agent` in manifest.capabilities. On the server, use `app.host.agent.start({ prompt, workingDirectory, idempotencyKey, model?, timeoutMs?, instructions? })` for reasoning plus tool execution. `instructions` (up to 30000 characters) is appended to the Agent system prompt: use it for tool usage notes and business rules. It is not a message, is not shown in Host chat and never appears in `conversations.messages`, so do not prepend such text to `prompt` or the first user turn. Omitting `model` uses the same default as llm calls (`gemini-3.8-flash`, or the first available platform model when that one is not allowed); a local model runs only when named explicitly. workingDirectory is an explicit real absolute execution directory; it is not implicitly the workspace cwd. A declared resource directory can provide that path. The Host fixes workspace/session ownership; callers do not select sessionId or workspaceId. Persist the business operation ID before start and reuse it as idempotencyKey. Save the returned runId, use get/list/findByIdempotencyKey to recover state, events({runId,afterSequence}) for progress, and cancel({runId}) to cancel. Same-key changed input is a conflict. On timeout or disconnection, query the original operation before retrying. @@ -14,11 +14,13 @@ The SDK resumes interrupted Agent event subscriptions from the last delivered cu Persist the last applied sequence with business state. If retry attempts are exhausted, read agent.get, show its partial text while running and reopen events using the saved cursor. For an expired cursor, reconcile the current snapshot and poll get for partial text and terminal status; do not reuse the expired cursor or leave the UI permanently loading. Task execution timeoutMs is separate from the subscription timeout. -Current Floatboat persists assistant text and tool calls/results during execution and displays them in the Host chat session. SDK events intentionally expose sanitized progress, not raw tool arguments/results or internal reasoning. The workbench owns its business history and loading/error states; Host persistence does not automatically render them in your page. +Current Floatboat persists assistant text and tool calls/results during execution and displays them in the Host chat session. By default SDK events carry sanitized progress: text, tool names with success/failure, status, and one `usage` event per turn with `promptTokens`/`cachedPromptTokens`/`completionTokens` for every model call plus the total. The usage is not split into system prompt, tool definitions and history. Internal reasoning is never exposed. The workbench owns its business history and loading/error states; Host persistence does not automatically render them in your page. + +When the page really needs tool records (for example a context inspector), declare both `agent` and `agent-tool-details` in manifest.capabilities. Then `tool-started` also carries `input` (JSON text, at most 4000 characters) and `tool-finished` carries `output` (at most 8000 characters) and `error`; `inputTruncated`/`outputTruncated` are true when text was cut. These values can contain workspace files and command output: show them only where needed and never log or send them elsewhere. The declaration is visible in review. ## Multi-turn conversations -For a chat UI, create one conversation and persist its conversationId in business SQLite. Use a new idempotencyKey for each user send and reuse that key only when retrying that same send. A new conversation starts a separate context; agent.start remains an independent task. +For a chat UI, create one conversation and persist its conversationId in business SQLite. Pass `instructions` to `conversations.create` when every turn needs the same background; it is fixed for the life of the conversation and changing it under the same idempotencyKey is a conflict. Use a new idempotencyKey for each user send and reuse that key only when retrying that same send. A new conversation starts a separate context; agent.start remains an independent task. ```ts const conversation = await app.host.agent.conversations.create({ @@ -39,3 +41,31 @@ for await (const event of app.host.agent.events(run)) { Host restores prior assistant and tool history before the next turn; do not concatenate old messages into prompt. Use conversations.get/list and conversations.messages({conversationId,offset,limit}) after refresh; nextOffset is null at the end. Messages expose only user/assistant text; complete tool details remain in Host chat. The directory is fixed at create; a different execution directory requires a new conversation. These APIs use the existing agent capability and workbench execution permission. While one turn is running, another new send returns AGENT_CONVERSATION_BUSY. Cancel with agent.cancel({runId}), wait for its terminal status, then send the next turn. Cancelling a subscription does not cancel the run. Account changes or Host shutdown interrupt the current run without erasing the owning account's conversation. Verify a follow-up referring to a prior fact and a tool result, refresh recovery, cancellation followed by another send, and the busy state. These APIs require a Floatboat build that advertises agent/conversationCreate and agent/conversationSend through capabilities.list; older clients must be upgraded. + +## Agent tools + +Let the Agent call this workbench's own business functions with native tools instead of reading notes and calling the app API through bash or HTTP: + +```ts +app.agentTool( + { + name: 'library_add', + description: 'Save a finished design into the library. Use after the user approves a draft.', + parameters: { + type: 'object', + properties: { title: { type: 'string' }, html: { type: 'string' } }, + required: ['title', 'html'], + additionalProperties: false, + }, + }, + async ({ input, runId, conversationId, signal }) => saveToLibrary(input, { runId, conversationId, signal }), +); +``` + +`name` starts with a lowercase letter, uses single underscores between lowercase letters/digits, and has at most 40 characters. `parameters` is a JSON Schema whose top level is `type: 'object'`. Register tools at startup, before `listen()`; invalid or duplicate definitions throw. + +Tools are offered only to Agent runs that this workbench starts (`agent.start` and conversation turns), and the model sees them as `mcp__aoe-host__app_`. The Host validates arguments against `parameters` first; invalid calls never reach the handler. A string return value goes to the model unchanged, other values are serialized as JSON, and an error with a `code` property reaches the model as that code and message. `signal` aborts when the run is cancelled. In events, `tool-started.appTool` names the workbench tool. Tools are available when the Floatboat Host runs the app (after open), not in standalone `pnpm dev` preview. + +The build records `agent.tools` and `agent.instructions` in `manifest.requires.operations` when the code uses them, so older Floatboat clients ask the user to upgrade. + +Skills that ship with a ported plugin belong in a bundled Combo dependency (`manifest.dependencies` with `kind: 'combo'`); the Host loads declared Combos before every Agent run. diff --git a/.agents/skills/floatboat-workbench/references/engineering.md b/.agents/skills/floatboat-workbench/references/engineering.md index 8c043da..76ec260 100644 --- a/.agents/skills/floatboat-workbench/references/engineering.md +++ b/.agents/skills/floatboat-workbench/references/engineering.md @@ -12,7 +12,7 @@ Read and update `app/ENGINEERING.md` for the actual entrypoints, module responsi Use strict TypeScript and unknown plus validation for external JSON. The compiler rejects explicit any, type-check suppression and server/Node imports from UI. Edit source and the declarative manifest/build configuration; `app/dist/`, `app/sdk/` and `.generated/` are generated. Application Vite/PostCSS configs, plugins and package scripts are not the Host build interface. -For an additional pure-JS runtime dependency, choose an exact compatible version and update both the root and `app/` package/lock files using pnpm (`pnpm add --save-exact package@version`, then the corresponding command from `app/`). The root toolchain supplies TypeScript, Vite and esbuild. Native modules are outside this application profile. Checking verifies the lock against installed versions; it does not repair dependencies through network access. +For an additional pure-JS runtime dependency, choose an exact compatible version and update both the root and `app/` package/lock files using pnpm (`pnpm add --save-exact package@version`, then the corresponding command from `app/`). The root toolchain supplies TypeScript, Vite and esbuild. Native modules are outside this application profile. Checking verifies the lock against installed versions; it does not repair dependencies through network access. Code cannot be downloaded or loaded by package name at runtime: the build rejects dynamic `import()`/`require` with non-literal paths and unbundled external dependencies, so review can see all code. When porting a plugin-based app, install the plugins you need as dependencies before building and import them with fixed paths. ## Metadata diff --git a/.agents/skills/floatboat-workbench/references/llm.md b/.agents/skills/floatboat-workbench/references/llm.md index e833797..0a33858 100644 --- a/.agents/skills/floatboat-workbench/references/llm.md +++ b/.agents/skills/floatboat-workbench/references/llm.md @@ -5,7 +5,7 @@ Declare `llm` in manifest.capabilities. Call the SDK on the server; the browser ## Text, structured output and streaming - `app.host.llm.generateText({ prompt, ...options })` returns text, model, requestId, finishReason, optional usage and an `assistant` message you can append to a conversation. -- `generateObject({ prompt, schema })` validates output against JSON Schema. Treat the returned object as unknown until checked against the business type. +- `generateObject({ prompt, schema })` validates output against JSON Schema. The Host first extracts JSON from a code fence or surrounding prose; text that still is not JSON fails with `LLM_OBJECT_INVALID` and a schema mismatch with `LLM_OBJECT_SCHEMA_MISMATCH` (both executed, so retry deliberately). Treat the returned object as unknown until checked against the business type. - `streamText({ prompt })` returns an async iterable with delta, tool-call, result and done events. Preserve incomplete/interrupted state if the stream ends without the required terminal result. - With `prompt`, all three accept `images` (up to 8, each at most 10 MiB): `{ resource: { resourceId, relativePath } }` for a file in a declared directory resource, `{ url: 'https://…' }` that the Host downloads, or a small `{ dataUrl: 'data:image/png;base64,…' }` (the whole request is limited to 2 MiB). Only PNG, JPEG, WebP and GIF are accepted. A model or protocol without image input returns `LLM_IMAGE_INPUT_UNSUPPORTED` before any charge; check `inputs` from `listModels` first. @@ -30,37 +30,40 @@ for (let round = 0; round < 5; round++) { ## Model catalog -`listModels({ kind? })` returns `models` (`id`, `kinds` of `text`/`speech`/`image`/`decision`/`embedding`/`rerank`, `inputs` of `text`/`image`, `source`), per-kind `defaults`, and `media`. Text models follow the same release, membership and local-model rules as the Floatboat chat model picker. Speech, image, decision, embedding and rerank models come from the platform catalog; when `media` is `unavailable` that part may be incomplete, so keep an explicit retry instead of treating it as "no models". Choose models from this result rather than hard-coding names. +`listModels({ kind? })` returns `models` (`id`, `kinds` of `text`/`speech`/`image`/`decision`/`rerank`, `inputs` of `text`/`image`, `source`), per-kind `defaults`, and `media`. Text models follow the same release, membership and local-model rules as the Floatboat chat model picker. `defaults.text` is the text default (`gemini-3.8-flash` when allowed); local models are listed with `source: 'local'` but are never a default. Speech, image, decision and rerank models come from a list Floatboat configures remotely (the first entry of each kind is its default), so `media` is always `available`; the platform may change these lists without an app update. Choose models from this result rather than hard-coding names. ## Speech and images Both write results into a manifest resource with `kind: 'directory'` and `access: 'read-write'`; pass a logical `output: { resourceId, relativePath }`, never an absolute path. Results return the resource reference, byte length and format/MIME only. -- `generateSpeech({ text, voice?, format?, speed?, instructions?, model?, output })` synthesizes up to 4096 characters. `format` is `mp3` (default), `wav`, `opus`, `aac`, `flac` or `pcm`. +- `generateSpeech({ text, voice?, format?, speed?, instructions?, model?, output })` synthesizes up to 4096 characters. `voice` is a provider voice ID (an ElevenLabs voice ID, or a Chirp name such as `cmn-CN-Chirp3-HD-Kore`); leave it empty for the model's default voice and do not pass OpenAI names like `alloy`. Platform speech models generally support `mp3` (default), `wav`, `opus` and `pcm` (24 kHz, 16-bit mono), and most reject `instructions`. - `generateImage({ prompt, size?, quality?, count?, images?, model?, output })` creates 1–4 images. With `images` it edits from those references. For `count > 1`, files get `-1`, `-2` before the extension. The default request deadline is 330 seconds; show progress instead of blocking the page. -Omitting `model` uses the catalog default for that kind; if the catalog is unavailable the call returns `LLM_MODEL_REQUIRED`. +Omitting `model` uses the catalog default for that kind; a kind with no models yet (rerank today) has no default, so the call returns `LLM_MODEL_REQUIRED` until you pass an explicit `model`. ## Decisions (Jev) -`decide({ state, questions, model? })` asks the platform decision model (default `jev-latest`) structured questions about `state` (string, object or array). `questions` holds 1–32 entries keyed by `[A-Za-z0-9_-]{1,64}`, each `{ type: 'noul' | 'choice' | 'score', instructions, criteria? }`; `choice` and `score` require `criteria`. It returns the raw `answers`. Use it for yes/no, option and scoring judgements; it is cheaper and steadier than prompting a text model, and does not generate prose. +`decide({ state, questions, model? })` asks the platform decision model (default `jev-latest`) structured questions about `state` (string, object or array). `questions` holds 1–32 entries keyed by `[A-Za-z0-9_-]{1,64}`, each `{ type: 'noul' | 'choice' | 'score', instructions, criteria? }`. For `choice`, `criteria` is an object mapping each option name to its description (1–255 entries); for `score`, it is an ordered array of levels scored from 0 (1–10 entries); `noul` optionally takes `{ true, false }` descriptions. Other shapes return `LLM_DECISION_CRITERIA_INVALID` before any charge. It returns the raw `answers`. Use it for yes/no, option and scoring judgements; it is cheaper and steadier than prompting a text model, and does not generate prose. ```ts const result = await app.host.llm.decide({ state: { message: ticket.body }, - questions: { urgent: { type: 'noul', instructions: 'Does the customer need a reply within an hour?' } }, + questions: { + urgent: { type: 'noul', instructions: 'Does the customer need a reply within an hour?' }, + topic: { type: 'choice', instructions: 'What is the message about?', criteria: { billing: 'Invoices or charges', bug: 'Something is broken' } }, + tone: { type: 'score', instructions: 'How upset is the customer?', criteria: ['Calm', 'Annoyed', 'Angry'] }, + }, }); ``` -## Retrieval: embeddings and reranking +## Retrieval: reranking -- `embed({ model, input, dimensions? })` requires an explicit model (pick one from `listModels({ kind: 'embedding' })`) and takes up to 256 texts. Vectors from different models are not comparable, so store the model ID next to each vector. - `rerank({ query, documents, topN?, model? })` returns `{ index, relevanceScore }` sorted by score, with `index` pointing into `documents` (up to 1000). -- The Host does not store vectors. Keep them in your app's SQLite as BLOBs and compare them with `sdk/vectors.mts`: `vectorToBlob`, `vectorFromBlob`, `cosineSimilarity` and `topK`. A full scan of 10,000 vectors at 1536 dimensions takes about 20 ms; use `topK` for recall, then `rerank` the shortlist. +- For a small candidate set, rerank all documents directly and show the matching originals as sources. ## Required Host operations -`pnpm workbench:check` and Floatboat builds scan the server code and write Host operations newer than the baseline (for example `llm.embed`, or `llm.messages` / `llm.tools` / `llm.reasoning` / `llm.images` when text calls use those fields) to `manifest.requires.operations`. Older Floatboat clients then ask the user to update instead of failing at runtime. Commit the field with your code; the build only adds entries, so remove one by hand only when the code no longer uses it. +`pnpm workbench:check` and Floatboat builds scan the server code and write Host operations newer than the baseline (for example `llm.rerank`, or `llm.messages` / `llm.tools` / `llm.reasoning` / `llm.images` when text calls use those fields) to `manifest.requires.operations`. Older Floatboat clients then ask the user to update instead of failing at runtime. Commit the field with your code; the build only adds entries, so remove one by hand only when the code no longer uses it. ## Errors and billing diff --git a/.agents/skills/floatboat-workbench/references/workflows.md b/.agents/skills/floatboat-workbench/references/workflows.md index 103b178..baad48e 100644 --- a/.agents/skills/floatboat-workbench/references/workflows.md +++ b/.agents/skills/floatboat-workbench/references/workflows.md @@ -16,7 +16,7 @@ const started = await app.host.workflows.start({ const run = await app.host.workflows.get({ runId: started.runId }); ``` -The operation ID, account-to-workspace association and document value above come from real business records. New applications use dependency, not a hardcoded local Workflow name/ID. Legacy name is compatible only when dependencies was never declared. A nested Workflow uses the child's logical name, declares it in requires, and permits that call in its own `capabilities.workflow.allowedWorkflowNames`. The Host carries the fixed ID/content digest through the declared graph. +The operation ID, account-to-workspace association and document value above come from real business records. New applications use dependency, not a hardcoded local Workflow name/ID. Legacy name is compatible only when dependencies was never declared; an empty `dependencies: []` still counts as declared, and start by name then fails with `WORKBENCH_DEPENDENCY_REFERENCE_REQUIRED` (not executed). A name that matches no installed Workflow fails with `WORKFLOW_NOT_FOUND` (not executed); that idempotencyKey is not consumed, so the same key can be retried once the Workflow is installed. A nested Workflow uses the child's logical name, declares it in requires, and permits that call in its own `capabilities.workflow.allowedWorkflowNames`. The Host carries the fixed ID/content digest through the declared graph. `workspaceId` optionally selects another persistent workspace on the same computer under the current Floatboat account. Omission uses the app's original workspace, independent of foreground UI switches; the source app/session ownership is retained. Use [workspaces](workspaces.md) to create an isolated website account and open its login page. Required missing, changed or untrusted dependencies prevent ready execution. Store-protected Workflows retain native installation, execution authorization and billing. diff --git a/sdk/agent-contracts.mts b/sdk/agent-contracts.mts index e815f47..449d304 100644 --- a/sdk/agent-contracts.mts +++ b/sdk/agent-contracts.mts @@ -1,4 +1,4 @@ -// ABOUTME: 工作台直接 Agent 任务的输入、运行事实与公开进度。 +// ABOUTME: 工作台直接 Agent 任务的输入、运行事实、公开进度与工作台自有 Agent 工具。 export interface AgentTaskInput { prompt: string; @@ -6,6 +6,24 @@ export interface AgentTaskInput { idempotencyKey: string; model?: string; timeoutMs?: number; + /** 追加到 Agent 系统提示的工作台说明，最长 30000 字符；不进入会话消息，也不在 Host 聊天中显示。 */ + instructions?: string; +} + +export interface AgentTokenUsage { + /** 含缓存命中与缓存写入部分的全部输入 token。 */ + promptTokens?: number; + cachedPromptTokens?: number; + completionTokens?: number; + totalTokens?: number; +} + +/** 一次模型调用（工具循环中的一步）的用量。 */ +export interface AgentUsageStep { + model?: string; + promptTokens: number; + cachedPromptTokens: number; + completionTokens: number; } export interface AgentTaskRun { @@ -19,14 +37,35 @@ export interface AgentTaskRun { /** 运行中为已接收文本快照，终态为最终结果或保留的部分输出。 */ text: string; errorCode: string | null; - usage?: { promptTokens?: number; completionTokens?: number; totalTokens?: number }; + usage?: AgentTokenUsage; } +/** + * input/output/error 只在 manifest.capabilities 声明 agent-tool-details 时出现，均为截断后的文本； + * 被截断时对应的 *Truncated 为 true。 + */ export type AgentTaskProgress = | { type: 'text'; text: string } - | { type: 'tool-started'; toolId: string; toolName: string } - | { type: 'tool-finished'; toolId: string; failed: boolean } - | { type: 'status'; status: string }; + | { + type: 'tool-started'; + toolId: string; + toolName: string; + /** 调用的是本工作台 server.agentTool 注册的工具时为其名称。 */ + appTool?: string; + input?: string; + inputTruncated?: boolean; + } + | { + type: 'tool-finished'; + toolId: string; + failed: boolean; + output?: string; + outputTruncated?: boolean; + error?: string; + } + | { type: 'status'; status: string } + /** 每轮结束时一条：本轮每次模型调用的用量及合计。 */ + | { type: 'usage'; steps: AgentUsageStep[]; total: AgentTokenUsage }; export type AgentTaskEvent = | { type: 'event'; cursor: number; event: AgentTaskProgress } @@ -46,6 +85,8 @@ export interface AgentConversationCreate { workingDirectory: string; model?: string; idempotencyKey: string; + /** 本会话每一轮都追加到系统提示的说明，创建后不可修改；不出现在 messages 中。 */ + instructions?: string; } export interface AgentConversationSend { conversationId: string; @@ -59,3 +100,22 @@ export interface AgentConversationMessage { role: 'user' | 'assistant'; text: string; } + +/** 工作台服务端提供给本工作台 Agent 运行的工具。 */ +export interface AgentToolDefinition { + /** 小写字母开头，只含小写字母、数字和单个下划线（不连续、不结尾），最长 40 字符。 */ + name: string; + /** 告诉模型何时、如何使用该工具，最长 4000 字符。 */ + description: string; + /** JSON Schema，顶层必须是 type: 'object'；Host 先按它校验模型参数再转交处理函数。 */ + parameters: Record; +} + +export interface AgentToolCall { + /** 已按 parameters 校验的模型参数。 */ + input: Record; + runId: string; + conversationId?: string; + /** Host 取消运行或放弃等待时中止。 */ + signal: AbortSignal; +} diff --git a/sdk/asr-client.mts b/sdk/asr-client.mts new file mode 100644 index 0000000..7415edd --- /dev/null +++ b/sdk/asr-client.mts @@ -0,0 +1,293 @@ +// ABOUTME: 在稳定 ASR 会话内串行发送并有限重试音频，临时断开的事件订阅按游标持续恢复。 + +import type { AsrEvent, CallOptions, HostClient } from './host-contracts.mjs'; +import { WorkbenchHostError } from './host-error.mjs'; + +type AsrClient = HostClient['asr']; +type FrameInput = Parameters[0]; +type FrameReceipt = Awaited>; +const MAX_RETRIES = 8; +const MAX_PENDING_FRAMES = 128; +const MAX_PENDING_BYTES = 8 * 1024 * 1024; + +interface QueuedFrame { + input: FrameInput; + options: CallOptions; + signal: AbortSignal; + settled: boolean; + finish(error?: unknown, receipt?: FrameReceipt): void; +} + +interface FrameQueue { + frames: QueuedFrame[]; + pending: Set; + bytes: number; + controller: AbortController; + stopping: boolean; + terminal?: Awaited>; + draining?: Promise; + stop?: ReturnType; +} + +function transient(error: unknown): boolean { + return ( + error instanceof WorkbenchHostError && + (['HOST_TRANSPORT_FAILED', 'HOST_STREAM_INTERRUPTED'].includes(error.code) || + (error.code === 'HOST_REQUEST_FAILED' && [502, 503, 504].includes(error.status ?? 0))) + ); +} + +function scope(options: CallOptions, lifetime?: AbortSignal, defaultTimeoutMs?: number) { + const controller = new AbortController(); + const timeoutMs = options.timeoutMs ?? defaultTimeoutMs; + const timer = + timeoutMs === undefined + ? undefined + : setTimeout(() => controller.abort(new DOMException('ASR 调用超时', 'TimeoutError')), timeoutMs); + const sources = [options.signal, lifetime].filter((value): value is AbortSignal => value !== undefined); + const listeners = sources.map((signal) => { + const abort = () => controller.abort(signal.reason); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + return () => signal.removeEventListener('abort', abort); + }); + return { + signal: controller.signal, + dispose() { + clearTimeout(timer); + for (const remove of listeners) remove(); + }, + }; +} + +function backoff(retry: number, signal: AbortSignal): Promise { + signal.throwIfAborted(); + return new Promise((resolve, reject) => { + const abort = () => { + clearTimeout(timer); + signal.removeEventListener('abort', abort); + reject(signal.reason); + }; + const timer = setTimeout( + () => { + signal.removeEventListener('abort', abort); + resolve(); + }, + Math.min(400 * 2 ** (retry - 1), 6000), + ); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + }); +} + +function untilAborted(pending: Promise, signal: AbortSignal): Promise { + signal.throwIfAborted(); + return new Promise((resolve, reject) => { + const abort = () => { + signal.removeEventListener('abort', abort); + reject(signal.reason); + }; + signal.addEventListener('abort', abort, { once: true }); + pending.then(resolve, reject).finally(() => signal.removeEventListener('abort', abort)); + if (signal.aborted) abort(); + }); +} + +export function createResumableAsrClient(transport: AsrClient): AsrClient { + const queues = new Map(); + const closedStreams = new Set(); + function queueFor(streamId: string): FrameQueue { + let queue = queues.get(streamId); + if (!queue) { + queue = { frames: [], pending: new Set(), bytes: 0, controller: new AbortController(), stopping: false }; + queues.set(streamId, queue); + } + return queue; + } + function close( + streamId: string, + error: unknown = new WorkbenchHostError({ code: 'ASR_SESSION_NOT_RUNNING' }), + terminal?: FrameQueue['terminal'], + ) { + closedStreams.add(streamId); + if (closedStreams.size > 100) closedStreams.delete(closedStreams.values().next().value!); + const queue = queues.get(streamId); + if (!queue) return; + queues.delete(streamId); + queue.terminal = terminal; + queue.controller.abort(error); + queue.frames.length = 0; + } + + async function send(task: QueuedFrame): Promise { + for (let failures = 0; ; ) { + task.signal.throwIfAborted(); + try { + return await untilAborted( + transport.sendFrame(task.input, { ...task.options, signal: task.signal }), + task.signal, + ); + } catch (error) { + task.signal.throwIfAborted(); + if (!transient(error) || ++failures > MAX_RETRIES) throw error; + await backoff(failures, task.signal); + } + } + } + + function drain(streamId: string, queue: FrameQueue): void { + if (queue.draining) return; + queue.draining = (async () => { + while (queue.frames.length && !queue.controller.signal.aborted) { + const task = queue.frames.shift()!; + if (task.settled) continue; + try { + task.finish(undefined, await send(task)); + } catch (error) { + task.finish(error); + // 前帧结果未知时不能继续发送后续序号；由调用方明确恢复该帧或结束会话。 + if (queues.get(streamId) === queue) { + queues.delete(streamId); + queue.controller.abort(error); + } + queue.frames.length = 0; + } + } + })().finally(() => { + queue.draining = undefined; + // 写回执先唤醒调用方；它可能在本轮 drain 的 finally 前提交下一帧。 + if (queue.frames.length && !queue.controller.signal.aborted) drain(streamId, queue); + }); + } + + function sendFrame(input: FrameInput, options: CallOptions = {}): Promise { + if (closedStreams.has(input.streamId)) + return Promise.reject(new WorkbenchHostError({ code: 'ASR_SESSION_NOT_RUNNING' })); + const queue = queueFor(input.streamId); + if (queue.stopping) return Promise.reject(new WorkbenchHostError({ code: 'ASR_SESSION_NOT_RUNNING' })); + const padding = input.pcmBase64.endsWith('==') ? 2 : input.pcmBase64.endsWith('=') ? 1 : 0; + const bytes = Math.max(0, Math.floor((input.pcmBase64.length * 3) / 4) - padding); + if (queue.pending.size >= MAX_PENDING_FRAMES || queue.bytes + bytes > MAX_PENDING_BYTES) + return Promise.reject(new WorkbenchHostError({ code: 'ASR_FRAME_QUEUE_FULL' })); + const lifetime = scope(options, queue.controller.signal, 120_000); + return new Promise((resolve, reject) => { + const abort = () => task.finish(lifetime.signal.reason); + const task: QueuedFrame = { + // 调用方复用参数对象时，网络重试仍必须发送原序号与原音频。 + input: { ...input }, + options, + signal: lifetime.signal, + settled: false, + finish(error, receipt) { + if (task.settled) return; + task.settled = true; + lifetime.signal.removeEventListener('abort', abort); + lifetime.dispose(); + queue.pending.delete(task); + queue.bytes -= bytes; + const index = queue.frames.indexOf(task); + if (index >= 0) queue.frames.splice(index, 1); + if (error !== undefined) reject(error); + else resolve(receipt!); + }, + }; + queue.bytes += bytes; + queue.pending.add(task); + queue.frames.push(task); + lifetime.signal.addEventListener('abort', abort, { once: true }); + if (lifetime.signal.aborted) abort(); + drain(input.streamId, queue); + }); + } + + async function* events( + input: Parameters[0], + options: CallOptions = {}, + ): AsyncIterable { + const lifetime = scope(options); + let cursor = input.afterSequence ?? 0; + let failures = 0; + let resultReceived = false; + try { + while (true) { + lifetime.signal.throwIfAborted(); + try { + for await (const event of transport.events( + { ...input, afterSequence: cursor }, + { ...options, signal: lifetime.signal }, + )) { + lifetime.signal.throwIfAborted(); + if (event.type === 'event') { + if (event.cursor <= cursor) continue; + cursor = event.cursor; + failures = 0; + } + if (event.type === 'result') { + resultReceived = !['running', 'stopping'].includes(event.session.status); + if (resultReceived) close(input.streamId, undefined, event.session); + } + // done 只结束本次订阅；没有业务终态时仍需从原游标恢复。 + if (event.type === 'done' && !resultReceived) + throw new WorkbenchHostError({ code: 'HOST_STREAM_INTERRUPTED', execution: 'unknown' }); + yield event; + if (event.type === 'done') return; + } + if (resultReceived) return; + throw new WorkbenchHostError({ code: 'HOST_STREAM_INTERRUPTED', execution: 'unknown' }); + } catch (error) { + lifetime.signal.throwIfAborted(); + if (resultReceived) return; + if (!transient(error)) throw error; + // 字幕连接暂不可用不代表本机收音失败；持续恢复直到真正终态或调用方取消。 + failures = Math.min(failures + 1, 5); + await backoff(failures, lifetime.signal); + } + } + } finally { + lifetime.dispose(); + } + } + + return { + async start(input, options) { + const result = await transport.start(input, options); + closedStreams.delete(result.streamId); + return result; + }, + async get(input, options) { + const session = await transport.get(input, options); + if (!['running', 'stopping'].includes(session.status)) close(input.streamId, undefined, session); + return session; + }, + sendFrame, + events, + stop(input, options = {}) { + const queue = queueFor(input.streamId); + queue.stopping = true; + queue.stop ??= (async () => { + const lifetime = scope(options, undefined, 120_000); + try { + await untilAborted(queue.draining ?? Promise.resolve(), lifetime.signal); + if (queue.terminal) return queue.terminal; + queue.controller.signal.throwIfAborted(); + const session = await untilAborted( + transport.stop(input, { ...options, signal: lifetime.signal }), + lifetime.signal, + ); + if (!['running', 'stopping'].includes(session.status)) close(input.streamId, undefined, session); + return session; + } catch (error) { + if (queues.get(input.streamId) === queue) close(input.streamId); + throw error; + } finally { + lifetime.dispose(); + } + })(); + return queue.stop; + }, + cancel(input, options) { + close(input.streamId); + return transport.cancel(input, options); + }, + }; +} diff --git a/sdk/asr-contracts.mts b/sdk/asr-contracts.mts index 25d05d6..3376aaa 100644 --- a/sdk/asr-contracts.mts +++ b/sdk/asr-contracts.mts @@ -16,16 +16,30 @@ export interface AsrUtterance { endTimeMs?: number; } +export interface AsrRecovery { + state: 'recovering' | 'ready'; + attempt: number; + /** null 表示音频已在本机保存，网络恢复前持续退避。 */ + maxAttempts: number | null; + bufferedAudioMs: number; + // 只由已确定分句的时间边界推进，发送回执不代表识别完成。 + confirmedAudioMs: number; + retryAfterMs?: number; +} + export interface AsrSession { streamId: string; status: 'running' | 'stopping' | 'completed' | 'failed' | 'cancelled'; text: string; errorCode: string | null; nextSequence: number; + utterances?: AsrUtterance[]; + recovery?: AsrRecovery; } export interface AsrProgress { - type: 'started' | 'partial' | 'final' | 'stopped' | 'error'; + type: 'started' | 'partial' | 'final' | 'stopped' | 'error' | 'recovering' | 'recovered'; + recovery?: AsrRecovery; text?: string; speakerId?: string | number; utterances?: AsrUtterance[]; diff --git a/sdk/audio-contracts.mts b/sdk/audio-contracts.mts new file mode 100644 index 0000000..3e107b8 --- /dev/null +++ b/sdk/audio-contracts.mts @@ -0,0 +1,49 @@ +// ABOUTME: 工作台文件音频能力的公开类型，使用逻辑资源引用隔离本机路径。 + +export interface AudioResourceReference { + resourceId: string; + relativePath: string; +} + +export type AudioInputFormat = 'mp3' | 'm4a' | 'aac' | 'wav' | 'flac' | 'ogg' | 'webm'; + +export interface AudioProbe { + format: AudioInputFormat; + durationMs: number; + encodedByteCount: number; + sampleRate: number; + channelCount: number; +} + +export type AudioJobKind = 'denoise' | 'transcribeFile'; +export type AudioJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled'; + +export interface AudioUtterance { + text: string; + speakerId?: string | number; + startTimeMs?: number; + endTimeMs?: number; +} + +export interface AudioJob { + jobId: string; + kind: AudioJobKind; + status: AudioJobStatus; + processedMs: number; + durationMs: number; + text: string; + utterances: AudioUtterance[]; + output: (AudioResourceReference & { format: 'm4a' | 'wav' | 'mp3' }) | null; + errorCode: string | null; +} + +export type AudioProgress = + | { type: 'progress'; stage: 'probe' | 'denoise' | 'decode' | 'transcribe'; processedMs: number; durationMs: number } + | { type: 'partial' | 'final'; text: string; utterances?: AudioUtterance[] } + | { type: 'error'; errorCode: string }; + +/** 与 ASR、Agent 事件流一致，流末尾固定追加一条 done 帧。 */ +export type AudioEvent = + | { type: 'event'; cursor: number; event: AudioProgress } + | { type: 'result'; cursor: number; job: AudioJob } + | { type: 'done'; requestId: string }; diff --git a/sdk/browser-asr.mts b/sdk/browser-asr.mts new file mode 100644 index 0000000..19a3130 --- /dev/null +++ b/sdk/browser-asr.mts @@ -0,0 +1,145 @@ +// ABOUTME: 浏览器通过同源工作台路由消费 ASR，复用公共 SDK 的帧重试与游标续订。 + +import { createResumableAsrClient } from './asr-client.mjs'; +import { WorkbenchHostError } from './host-error.mjs'; +import type { WorkbenchPage } from './browser.mjs'; +import type { AsrEvent, CallOptions, HostClient } from './host-contracts.mjs'; + +export function createWorkbenchAsrClient( + page: Pick, + options: { basePath?: string } = {}, +): HostClient['asr'] { + const basePath = options.basePath ?? '/api/asr'; + if (!/^\/api\/[a-zA-Z0-9/_-]+$/u.test(basePath)) throw new WorkbenchHostError({ code: 'ASR_ROUTE_INVALID' }); + + async function response(path: string, init: RequestInit, call: CallOptions = {}): Promise { + const signal = + call.timeoutMs === undefined + ? call.signal + : call.signal + ? AbortSignal.any([call.signal, AbortSignal.timeout(call.timeoutMs)]) + : AbortSignal.timeout(call.timeoutMs); + let result: Response; + try { + result = await page.fetch(`${basePath}${path}`, { ...init, signal }); + } catch { + // 单次请求超时可重发原帧；调用者取消和整个操作超时必须立即终止。 + call.signal?.throwIfAborted(); + throw new WorkbenchHostError({ code: 'HOST_TRANSPORT_FAILED', execution: 'unknown' }); + } + if (!result.ok) { + const body = (await result.json().catch(() => null)) as { error?: { code?: string; message?: string } } | null; + throw new WorkbenchHostError( + { + code: body?.error?.code ?? 'HOST_REQUEST_FAILED', + message: body?.error?.message, + execution: 'unknown', + }, + result.status, + ); + } + return result; + } + + async function json(path: string, body?: unknown, call: CallOptions = {}): Promise { + const result = await response( + path, + body === undefined + ? {} + : { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }, + { timeoutMs: 30_000, ...call }, + ); + try { + return (await result.json()) as T; + } catch (error) { + call.signal?.throwIfAborted(); + if (error instanceof SyntaxError) throw new WorkbenchHostError({ code: 'HOST_RESPONSE_INVALID' }); + throw new WorkbenchHostError({ code: 'HOST_TRANSPORT_FAILED', execution: 'unknown' }); + } + } + + async function* events( + input: { streamId: string; afterSequence?: number }, + call: CallOptions = {}, + ): AsyncIterable { + const controller = new AbortController(); + const signal = call.signal ? AbortSignal.any([call.signal, controller.signal]) : controller.signal; + let idle: ReturnType | undefined; + const resetIdle = () => { + clearTimeout(idle); + idle = setTimeout(() => controller.abort(), 120_000); + }; + let reader: ReadableStreamDefaultReader | undefined; + resetIdle(); + try { + const result = await response( + `/${encodeURIComponent(input.streamId)}/events?after=${input.afterSequence ?? 0}`, + { headers: { accept: 'text/event-stream' } }, + { ...call, signal }, + ); + if (!result.body) throw new WorkbenchHostError({ code: 'HOST_STREAM_INTERRUPTED' }); + reader = result.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + for (;;) { + const chunk = await reader.read(); + resetIdle(); + buffer += decoder.decode(chunk.value, { stream: !chunk.done }); + if (buffer.length > 2_000_000) throw new WorkbenchHostError({ code: 'HOST_STREAM_TOO_LARGE' }); + let boundary: RegExpExecArray | null; + while ((boundary = /\r?\n\r?\n/u.exec(buffer))) { + const frame = buffer.slice(0, boundary.index); + buffer = buffer.slice(boundary.index + boundary[0].length); + const data = frame + .split(/\r?\n/u) + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(5).trimStart()) + .join('\n'); + if (!data) continue; + let event: AsrEvent | { type: 'error'; error?: { code?: string; message?: string } }; + try { + event = JSON.parse(data); + } catch { + throw new WorkbenchHostError({ code: 'HOST_STREAM_INVALID' }); + } + if (!event || typeof event !== 'object') throw new WorkbenchHostError({ code: 'HOST_STREAM_INVALID' }); + if (event.type === 'error') + throw new WorkbenchHostError({ + code: event.error?.code ?? 'HOST_STREAM_FAILED', + message: event.error?.message, + }); + if (event.type !== 'event' && event.type !== 'result' && event.type !== 'done') + throw new WorkbenchHostError({ code: 'HOST_STREAM_INVALID' }); + if (event.type !== 'done' && (!Number.isSafeInteger(event.cursor) || event.cursor < 0)) + throw new WorkbenchHostError({ code: 'HOST_STREAM_INVALID' }); + yield event; + if (event.type === 'done') return; + } + if (chunk.done) throw new WorkbenchHostError({ code: 'HOST_STREAM_INTERRUPTED', execution: 'unknown' }); + } + } catch (error) { + call.signal?.throwIfAborted(); + if (error instanceof WorkbenchHostError) throw error; + throw new WorkbenchHostError({ code: 'HOST_STREAM_INTERRUPTED', execution: 'unknown' }); + } finally { + clearTimeout(idle); + controller.abort(); + await reader?.cancel().catch(() => undefined); + reader?.releaseLock(); + } + } + + return createResumableAsrClient({ + start: (input, call) => json('/start', input, call), + sendFrame: ({ streamId, sequence, pcmBase64 }, call) => + json(`/${encodeURIComponent(streamId)}/frames`, { sequence, pcmBase64 }, call), + get: ({ streamId }, call) => json(`/${encodeURIComponent(streamId)}`, undefined, call), + events, + stop: ({ streamId }, call) => json(`/${encodeURIComponent(streamId)}/stop`, {}, call), + cancel: ({ streamId }, call) => json(`/${encodeURIComponent(streamId)}/cancel`, {}, call), + }); +} diff --git a/sdk/browser.mts b/sdk/browser.mts index 91a4b32..c03ff1e 100644 --- a/sdk/browser.mts +++ b/sdk/browser.mts @@ -1,29 +1,236 @@ -// ABOUTME: 页面可直接连接本地工作台；父窗口消息只用于主题和语言同步。 +// ABOUTME: 页面直接连接本地工作台；宿主主题仅用于未自行声明明暗方案的页面。 +export { createWorkbenchAsrClient } from './browser-asr.mjs'; +export interface WorkbenchThemeDeclaration { + brand?: string; + toolbar?: string; + accent?: string; +} + export interface WorkbenchPage { fetch(input: string | URL, init?: RequestInit): Promise; + announceTheme(theme: WorkbenchThemeDeclaration): void; + /** 页面主动卸载时释放主题观察；重复连接共享同一实例。 */ + dispose(): void; +} + +const COLOR_FUNCTION = /^(?:rgb|rgba|hsl|hsla|hwb|lab|lch|oklab|oklch|color)\(/iu; +const HEX_COLOR = /^#(?:[0-9a-f]{3,4}|[0-9a-f]{6}|[0-9a-f]{8})$/iu; +const NAMED_COLOR = /^[a-z][a-z0-9-]*$/iu; +const THEME_TOKENS = ['background', 'text', 'border', 'muted', 'primary'] as const; +let activePage: WorkbenchPage | undefined; +let pendingDeclaration: WorkbenchThemeDeclaration = {}; + +function isSafeColor(value: unknown): value is string { + if (typeof value !== 'string') return false; + const normalized = value.trim(); + if (!normalized || normalized.length > 96 || /[;{}"']/u.test(normalized)) return false; + if (/^(?:transparent|currentcolor|inherit|initial|unset|revert|revert-layer)$/iu.test(normalized)) return false; + if (!HEX_COLOR.test(normalized) && !COLOR_FUNCTION.test(normalized) && !NAMED_COLOR.test(normalized)) return false; + return typeof CSS === 'undefined' || typeof CSS.supports !== 'function' || CSS.supports('color', normalized); +} + +function normalizeTheme(theme: WorkbenchThemeDeclaration): WorkbenchThemeDeclaration { + return Object.fromEntries( + Object.entries(theme) + .filter(([name]) => name === 'brand' || name === 'toolbar' || name === 'accent') + .filter(([, value]) => isSafeColor(value)) + .map(([name, value]) => [name, value.trim()]), + ) as WorkbenchThemeDeclaration; +} + +function rgba(color: string): number[] | null { + const match = /^rgba?\(([^)]+)\)$/iu.exec(color); + if (match) { + const parts = match[1].split(/[\s,/]+/u).filter(Boolean); + const alpha = parts[3] ?? '1'; + return [ + ...parts.slice(0, 3).map((part) => Number.parseFloat(part) * (part.endsWith('%') ? 2.55 : 1)), + Number.parseFloat(alpha) / (alpha.endsWith('%') ? 100 : 1), + ]; + } + if (typeof CanvasRenderingContext2D !== 'undefined' && isSafeColor(color)) { + const canvas = document.createElement('canvas'); + canvas.width = canvas.height = 1; + const context = canvas.getContext('2d', { willReadFrequently: true }); + if (context) { + context.fillStyle = color; + context.fillRect(0, 0, 1, 1); + const [red, green, blue, alpha] = context.getImageData(0, 0, 1, 1).data; + return [red, green, blue, alpha / 255]; + } + } + return null; } + +function toolbarSurface(element: HTMLElement): string | undefined { + const color = [0, 0, 0]; + let alpha = 0; + for (let ancestor: HTMLElement | null = element; ancestor; ancestor = ancestor.parentElement) { + const style = getComputedStyle(ancestor); + if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity || 1) < 0.95) return; + if (alpha >= 0.999) continue; + if (style.backgroundImage && style.backgroundImage !== 'none') return; + const layer = rgba(style.backgroundColor); + if (!layer) continue; + const weight = layer[3] * (1 - alpha); + for (let index = 0; index < 3; index++) color[index] += layer[index] * weight; + alpha += weight; + } + // 没有确定底色时不能把透明黑当作工作台颜色。 + if (alpha < 0.999) return; + return `rgb(${color.map(Math.round).join(', ')})`; +} + +function getToolbarColor(): string | undefined { + const candidates = Array.from( + document.querySelectorAll( + '[data-workbench-toolbar], [data-toolbar], [role="toolbar"], header, nav, [class*="toolbar" i], [class*="topbar" i], [class*="top-bar" i]', + ), + ) + .map((element) => ({ element, rect: element.getBoundingClientRect() })) + .filter( + ({ rect }) => + rect.top >= -1 && + rect.top <= 80 && + rect.height > 0 && + rect.height <= 160 && + rect.width >= window.innerWidth * 0.55, + ) + .sort((left, right) => left.rect.top - right.rect.top || right.rect.width - left.rect.width); + for (const { element } of candidates) { + const color = toolbarSurface(element); + if (color) return color; + } + return undefined; +} + +export function announceWorkbenchTheme(theme: WorkbenchThemeDeclaration): void { + if (activePage) activePage.announceTheme(theme); + else { + pendingDeclaration = normalizeTheme(theme); + if (window.parent !== window) + window.parent.postMessage({ type: 'floatboat:workbench-theme', theme: pendingDeclaration }, '*'); + } +} + export function connectWorkbenchPage(_legacyOptions?: unknown): Promise { + if (activePage) return Promise.resolve(activePage); const parameters = new URLSearchParams(location.hash.slice(1)); if (parameters.has('workbench-page')) { parameters.delete('workbench-page'); history.replaceState(null, '', `${location.pathname}${location.search}${parameters.size ? `#${parameters}` : ''}`); } - window.addEventListener('message', (event) => { + let declared: WorkbenchThemeDeclaration = pendingDeclaration; + let hostTheme: Record = {}; + let lastPublished = ''; + let timer: ReturnType | undefined; + let disposed = false; + const originalTokens = new Map(); + const projectedTokens = new Map(); + const root = document.documentElement; + const applyHostTheme = () => { + // 单一明暗方案意味着页面拥有自己的调色板，混入宿主 token 会造成深底深字。 + const schemes = getComputedStyle(root).colorScheme.split(/\s+/u); + const ownsPalette = schemes.includes('dark') !== schemes.includes('light'); + for (const name of THEME_TOKENS) { + const token = `--workbench-${name}`; + if (ownsPalette) { + if (projectedTokens.has(token) && root.style.getPropertyValue(token) === projectedTokens.get(token)) { + const original = originalTokens.get(token); + if (original?.value) root.style.setProperty(token, original.value, original.priority); + else root.style.removeProperty(token); + } + projectedTokens.delete(token); + } else if (typeof hostTheme[name] === 'string') { + if (!originalTokens.has(token)) + originalTokens.set(token, { + value: root.style.getPropertyValue(token), + priority: root.style.getPropertyPriority(token), + }); + if (root.style.getPropertyValue(token) !== hostTheme[name]) root.style.setProperty(token, hostTheme[name]); + projectedTokens.set(token, hostTheme[name]); + } + } + }; + const publish = () => { + if (disposed || window.parent === window) return; + applyHostTheme(); + const toolbar = getToolbarColor(); + const theme = { ...(toolbar ? { toolbar } : {}), ...declared }; + const serialized = JSON.stringify(theme); + if (serialized === lastPublished) return; + lastPublished = serialized; + window.parent.postMessage({ type: 'floatboat:workbench-theme', theme }, '*'); + }; + const schedule = () => { + if (timer !== undefined || disposed) return; + timer = setTimeout(() => { + timer = undefined; + publish(); + }, 120); + }; + const receive = (event: MessageEvent) => { if (window.parent === window || event.source !== window.parent || event.data?.type !== 'floatboat:workbench') return; - const theme = event.data.theme ?? {}; - for (const name of ['background', 'text', 'border', 'muted', 'primary']) { - if (typeof theme[name] === 'string') - document.documentElement.style.setProperty(`--workbench-${name}`, theme[name]); - } - if (typeof event.data.language === 'string') document.documentElement.lang = event.data.language; + hostTheme = event.data.theme ?? {}; + applyHostTheme(); + if (typeof event.data.language === 'string') root.lang = event.data.language; + schedule(); + }; + const observer = new MutationObserver((records) => { + if ( + records.some( + (record) => + record.type === 'attributes' || + record.type === 'childList' || + record.target.parentElement?.tagName === 'STYLE', + ) + ) + schedule(); }); - if (window.parent !== window) window.parent.postMessage({ type: 'floatboat:workbench-ready' }, '*'); - return Promise.resolve({ + const media = window.matchMedia?.('(prefers-color-scheme: dark)'); + const page: WorkbenchPage = { fetch(input, init) { const target = new URL(input, location.href); if (target.origin !== location.origin) throw new Error('WORKBENCH_PAGE_TARGET_INVALID'); return fetch(target, init); }, - }); + announceTheme(theme) { + declared = normalizeTheme(theme); + publish(); + }, + dispose() { + if (disposed) return; + disposed = true; + clearTimeout(timer); + observer.disconnect(); + window.removeEventListener('message', receive); + window.removeEventListener('resize', schedule); + window.removeEventListener('pagehide', page.dispose); + document.removeEventListener('load', schedule, true); + document.removeEventListener('transitionend', schedule); + media?.removeEventListener('change', schedule); + if (activePage === page) activePage = undefined; + pendingDeclaration = {}; + }, + }; + activePage = page; + if (window.parent !== window) { + window.addEventListener('message', receive); + window.addEventListener('resize', schedule); + window.addEventListener('pagehide', page.dispose); + document.addEventListener('load', schedule, true); + document.addEventListener('transitionend', schedule); + media?.addEventListener('change', schedule); + observer.observe(root, { + attributes: true, + attributeFilter: ['class', 'style', 'hidden', 'data-theme'], + childList: true, + characterData: true, + subtree: true, + }); + window.parent.postMessage({ type: 'floatboat:workbench-ready' }, '*'); + schedule(); + } + return Promise.resolve(page); } diff --git a/sdk/host-contracts.mts b/sdk/host-contracts.mts index dd957fe..07d7570 100644 --- a/sdk/host-contracts.mts +++ b/sdk/host-contracts.mts @@ -10,8 +10,10 @@ import type { AgentConversationMessage, } from './agent-contracts.mjs'; import type { AsrStartInput, AsrSession, AsrEvent } from './asr-contracts.mjs'; +import type { AudioEvent, AudioJob, AudioProbe, AudioResourceReference } from './audio-contracts.mjs'; export type * from './agent-contracts.mjs'; export type * from './asr-contracts.mjs'; +export type * from './audio-contracts.mjs'; export type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; @@ -117,7 +119,9 @@ export type SpeechFormat = 'mp3' | 'wav' | 'opus' | 'aac' | 'flac' | 'pcm'; export interface SpeechInput { text: string; model?: string; + /** 供应商音色 ID：ElevenLabs 为 voice ID，Chirp 为 cmn-CN-Chirp3-HD-Kore 形式；留空使用该模型的默认音色。 */ voice?: string; + /** 平台语音模型通常只支持 mp3、wav、opus、pcm（24 kHz 16 位单声道）。 */ format?: SpeechFormat; speed?: number; instructions?: string; @@ -156,9 +160,12 @@ export interface ImageGenerationResult { type DecisionInstructions = string | Json[] | { [key: string]: Json }; export type DecisionQuestion = - | { type: 'noul'; instructions: DecisionInstructions; criteria?: Json } - | { type: 'choice'; instructions: DecisionInstructions; criteria: Json } - | { type: 'score'; instructions: DecisionInstructions; criteria: Json }; + /** criteria 可选：{ true?, false? } 分别描述什么算“是”与“否”。回答为 { type:'noul', noul: 0–1 }。 */ + | { type: 'noul'; instructions: DecisionInstructions; criteria?: { true?: Json; false?: Json } } + /** criteria 为「选项名 → 说明」对象，1–255 项。回答含 choice、confidence 与各选项 probabilities。 */ + | { type: 'choice'; instructions: DecisionInstructions; criteria: { [choice: string]: Json } } + /** criteria 为按分值从 0 起排列的等级数组，1–10 项。回答的 score 为概率加权分，可为小数，并附 legend。 */ + | { type: 'score'; instructions: DecisionInstructions; criteria: Json[] }; export interface DecisionInput { /** 默认 jev-latest。 */ @@ -360,11 +367,41 @@ export interface HostClient { }; asr: { start(input: AsrStartInput, options?: CallOptions): Promise<{ streamId: string }>; - sendFrame(input: { streamId: string; sequence: number; pcmBase64: string }): Promise<{ nextSequence: number }>; - get(input: { streamId: string }): Promise; + /** 回执确认 Host 已持久保存该帧，不代表云端已识别；网络故障重试原序号。 */ + sendFrame( + input: { streamId: string; sequence: number; pcmBase64: string }, + options?: CallOptions, + ): Promise<{ nextSequence: number }>; + get(input: { streamId: string }, options?: CallOptions): Promise; + /** 断流按游标恢复；游标过期明确报错，不能用 get 的最新文本替代历史。 */ events(input: { streamId: string; afterSequence?: number }, options?: CallOptions): AsyncIterable; - stop(input: { streamId: string }): Promise; - cancel(input: { streamId: string }): Promise; + /** 停止接收新音频；stopping 表示本机音频仍待补识别，最终结果继续从 events 获取。 */ + stop(input: { streamId: string }, options?: CallOptions): Promise; + cancel(input: { streamId: string }, options?: CallOptions): Promise; + }; + audio: { + probe(input: AudioResourceReference): Promise; + denoise( + input: { + input: AudioResourceReference; + output: AudioResourceReference; + outputFormat?: 'm4a' | 'wav' | 'mp3'; + idempotencyKey: string; + }, + options?: CallOptions, + ): Promise<{ jobId: string }>; + transcribeFile( + input: { + input: AudioResourceReference; + idempotencyKey: string; + preprocess?: { denoise?: boolean }; + speakerDiarizationEnabled?: boolean; + }, + options?: CallOptions, + ): Promise<{ jobId: string }>; + get(input: { jobId: string }): Promise; + events(input: { jobId: string; afterSequence?: number }, options?: CallOptions): AsyncIterable; + cancel(input: { jobId: string }): Promise; }; capabilities: { list(input?: { domain?: string; operation?: string }): Promise<{ diff --git a/sdk/host-error.mts b/sdk/host-error.mts new file mode 100644 index 0000000..7909a38 --- /dev/null +++ b/sdk/host-error.mts @@ -0,0 +1,24 @@ +// ABOUTME: Host SDK 的公开错误契约，可由 Node 与浏览器传输适配器共同使用。 + +export interface HostFailure { + code: string; + message?: string; + requestId?: string; + execution?: 'not-executed' | 'executed' | 'unknown'; +} + +export class WorkbenchHostError extends Error { + readonly code: string; + readonly requestId?: string; + readonly execution: 'not-executed' | 'executed' | 'unknown'; + readonly status?: number; + + constructor(error: HostFailure, status?: number) { + super(error.message ?? error.code ?? 'WORKBENCH_HOST_ERROR'); + this.name = 'WorkbenchHostError'; + this.code = error.code ?? 'WORKBENCH_HOST_ERROR'; + this.requestId = error.requestId; + this.execution = error.execution ?? 'not-executed'; + this.status = status; + } +} diff --git a/sdk/host.mts b/sdk/host.mts index e0d2851..d4b6ee2 100644 --- a/sdk/host.mts +++ b/sdk/host.mts @@ -17,8 +17,14 @@ import type { WorkflowRun, AgentTaskEvent, AsrEvent, + AudioEvent, + AudioJob, + AudioProbe, } from './host-contracts.mjs'; export type * from './host-contracts.mjs'; +import { createResumableAsrClient } from './asr-client.mjs'; +import { WorkbenchHostError, type HostFailure } from './host-error.mjs'; +export { WorkbenchHostError } from './host-error.mjs'; /** Model used when an llm call names none; an explicit `model` always wins. */ export const DEFAULT_LLM_MODEL = 'gemini-3.8-flash'; @@ -26,32 +32,12 @@ const withDefaultModel = (input: T): T => input !== null && typeof input === 'object' && !Array.isArray(input) && !(input as { model?: unknown }).model ? { ...input, model: DEFAULT_LLM_MODEL } : input; - -interface HostFailure { - code: string; - message?: string; - requestId?: string; - execution?: 'not-executed' | 'executed' | 'unknown'; -} interface HostResponse { error?: HostFailure; data: T; } type TrackTask = (operation: () => Promise, kind?: string) => Promise; -export class WorkbenchHostError extends Error { - readonly code: string; - readonly requestId?: string; - readonly execution: 'not-executed' | 'executed' | 'unknown'; - constructor(error: HostFailure) { - super(error.message ?? error.code ?? 'WORKBENCH_HOST_ERROR'); - this.name = 'WorkbenchHostError'; - this.code = error.code ?? 'WORKBENCH_HOST_ERROR'; - this.requestId = error.requestId; - this.execution = error.execution ?? 'not-executed'; - } -} - export function createHostClient( connection: HostConnection | null | (() => HostConnection | null | undefined), track: TrackTask = (operation) => operation(), @@ -93,12 +79,31 @@ export function createHostClient( ): Promise { return track(async () => { const response = await transmit('/v1/call', { domain, operation, input, requestId: options.requestId }, options); - const result = (await response.json()) as HostResponse; + const result = + domain === 'asr' && operation === 'sendFrame' + ? await asrResponse(response) + : ((await response.json()) as HostResponse); if (!response.ok || result.error) throw new WorkbenchHostError(result.error ?? { code: 'HOST_REQUEST_FAILED' }); return result.data; }, `${domain}.${operation}`); } + async function asrResponse(response: Response): Promise> { + let result: HostResponse; + try { + result = (await response.json()) as HostResponse; + } catch (error) { + if (!response.ok) throw new WorkbenchHostError({ code: 'HOST_REQUEST_FAILED' }, response.status); + throw new WorkbenchHostError({ + code: error instanceof SyntaxError ? 'HOST_RESPONSE_INVALID' : 'HOST_TRANSPORT_FAILED', + execution: 'unknown', + }); + } + if (!response.ok || result.error) + throw new WorkbenchHostError(result.error ?? { code: 'HOST_REQUEST_FAILED' }, response.status); + return result; + } + async function* events( domain: string, operation: string, @@ -125,6 +130,7 @@ export function createHostClient( options, resumable ? subscriptionSignal : undefined, ); + if (!response.ok && domain === 'asr') await asrResponse(response); if (!response.ok) throw new WorkbenchHostError( ((await response.json()) as HostResponse).error ?? { code: 'HOST_REQUEST_FAILED' }, @@ -162,6 +168,8 @@ export function createHostClient( if (!terminal) throw new WorkbenchHostError({ code: 'HOST_STREAM_INTERRUPTED', execution: 'unknown' }); } catch (error) { if (error instanceof WorkbenchHostError) throw error; + if (domain === 'asr' && error instanceof SyntaxError) + throw new WorkbenchHostError({ code: 'HOST_STREAM_INVALID', execution: 'unknown' }); throw new WorkbenchHostError({ code: 'HOST_STREAM_INTERRUPTED', execution: 'unknown' }); } finally { await reader.cancel().catch(() => undefined); @@ -257,13 +265,21 @@ export function createHostClient( events: (input, options) => agentEvents(input, options), cancel: (input) => request('agent', 'cancel', input), }, - asr: { + asr: createResumableAsrClient({ start: (input, options) => request('asr', 'start', input, options), - sendFrame: (input) => request('asr', 'sendFrame', input), - get: (input) => request('asr', 'get', input), - events: (input, options) => events('asr', 'events', input, options), - stop: (input) => request('asr', 'stop', input), - cancel: (input) => request('asr', 'cancel', input), + sendFrame: (input, options) => request('asr', 'sendFrame', input, options), + get: (input, options) => request('asr', 'get', input, options), + events: (input, options) => events('asr', 'events', input, options, true), + stop: (input, options) => request('asr', 'stop', input, options), + cancel: (input, options) => request('asr', 'cancel', input, options), + }), + audio: { + probe: (input) => request('audio', 'probe', input), + denoise: (input, options) => request('audio', 'denoise', input, options), + transcribeFile: (input, options) => request('audio', 'transcribeFile', input, options), + get: (input) => request('audio', 'get', input), + events: (input, options) => events('audio', 'events', input, options, true), + cancel: (input) => request('audio', 'cancel', input), }, capabilities: { list: (input = {}) => request('capabilities', 'list', input) }, llm: { @@ -271,7 +287,6 @@ export function createHostClient( generateObject: (input, options) => request('llm', 'generateObject', withDefaultModel(input), options), streamText: (input, options) => events('llm', 'streamText', withDefaultModel(input), options), - // 媒体与决策模型的默认值由 Host 模型目录决定，不套用文本默认模型。 listModels: (input = {}) => request('llm', 'listModels', input), generateSpeech: (input, options = {}) => request('llm', 'generateSpeech', input, { timeoutMs: 150_000, ...options }), diff --git a/sdk/runtime-contracts.mts b/sdk/runtime-contracts.mts index 804e18d..7deea88 100644 --- a/sdk/runtime-contracts.mts +++ b/sdk/runtime-contracts.mts @@ -7,6 +7,7 @@ import type { WorkbenchResources, WorkbenchResourceDeclaration } from './resourc import type { DatabaseSync } from 'node:sqlite'; import type { HostClient } from './host.mjs'; +import type { AgentToolCall, AgentToolDefinition } from './agent-contracts.mjs'; export interface WorkbenchManifest { appId: string; @@ -40,6 +41,11 @@ export interface WorkbenchServer { handler: (context: RouteContext) => unknown | Promise, ): void; action(name: string, handler: (input: unknown) => unknown | Promise): void; + /** + * 注册本工作台 Agent 运行可调用的工具；需要 manifest 声明 agent。 + * 返回值交给模型：字符串原样传递，其他值序列化为 JSON。抛错时模型收到错误码与消息。 + */ + agentTool(definition: AgentToolDefinition, handler: (call: AgentToolCall) => unknown | Promise): void; trackTask(operation: () => Promise, kind?: string): Promise; tasks(): Array<{ id: string; kind: string; state: string; updated_at: string; error_code: string | null }>; close(): Promise; diff --git a/sdk/runtime.mts b/sdk/runtime.mts index a7806f2..b4f87de 100644 --- a/sdk/runtime.mts +++ b/sdk/runtime.mts @@ -15,6 +15,7 @@ import { import type { IncomingMessage, ServerResponse } from 'node:http'; import type { HostConnection } from './host.mjs'; import type { RouteContext, WorkbenchManifest, WorkbenchServer } from './runtime-contracts.mjs'; +import type { AgentToolCall, AgentToolDefinition } from './agent-contracts.mjs'; import { errorFields } from './errors.mjs'; import { readWorkbenchBrowserPolicy, workbenchBrowserHeaders, WORKBENCH_MIME_TYPES } from './browser-policy.mjs'; import { createWorkbenchBrowserDiagnostics } from './runtime-diagnostics.mjs'; @@ -74,6 +75,10 @@ export async function createWorkbenchServer( } const routes: Route[] = []; const actions = new Map unknown | Promise>(); + const agentTools = new Map< + string, + { definition: AgentToolDefinition; handler: (call: AgentToolCall) => unknown | Promise } + >(); let url: string | undefined; let closing = false; let closed = false; @@ -180,6 +185,46 @@ export async function createWorkbenchServer( if (!action) return send(response, { error: { code: 'APP_ACTION_NOT_FOUND' } }, 404); return send(response, await trackTask(async () => action(body.input), `action:${body.action}`)); } + // 只供 Host 在 Agent 运行时列出与执行工作台工具；与 action 一样用进程 nonce 鉴权，页面无法直接调用。 + if (requestUrl.pathname === '/__workbench/agent-tools' && request.method === 'GET') { + if (!equalToken(request.headers.authorization?.replace(/^Bearer /, ''), config.nonce)) + return send(response, {}, 401); + return send(response, { tools: [...agentTools.values()].map((tool) => tool.definition) }); + } + if (requestUrl.pathname === '/__workbench/agent-tool' && request.method === 'POST') { + if (!equalToken(request.headers.authorization?.replace(/^Bearer /, ''), config.nonce)) + return send(response, {}, 401); + const body = await readBody(request); + const tool = typeof body.name === 'string' ? agentTools.get(body.name) : undefined; + if (!tool) return send(response, { error: { code: 'APP_AGENT_TOOL_NOT_FOUND' } }, 404); + const input = body.input; + if (!input || typeof input !== 'object' || Array.isArray(input) || typeof body.runId !== 'string') + return send(response, { error: { code: 'APP_AGENT_TOOL_REQUEST_INVALID' } }, 400); + const controller = new AbortController(); + response.once('close', () => { + if (!response.writableFinished) controller.abort(); + }); + try { + const result = await trackTask( + async () => + tool.handler({ + input: input as Record, + runId: body.runId as string, + ...(typeof body.conversationId === 'string' ? { conversationId: body.conversationId } : {}), + signal: controller.signal, + }), + `agent-tool:${tool.definition.name}`, + ); + return send(response, { result: result ?? null }); + } catch (error) { + const failure = errorFields(error); + return send( + response, + { error: { code: failure.code ?? 'APP_AGENT_TOOL_FAILED', message: failure.message.slice(0, 2000) } }, + 500, + ); + } + } if (requestUrl.pathname.startsWith('/api/')) { if (request.headers.origin && request.headers.origin !== url) return send(response, { error: { code: 'APP_ORIGIN_INVALID' } }, 403); @@ -199,12 +244,14 @@ export async function createWorkbenchServer( } if (request.method !== 'GET') return send(response, {}, 405); const uiRoot = resolve(config.sourceDirectory, manifest.ui); - const sdkAsset = - requestUrl.pathname === '/__workbench/browser.mjs' - ? 'browser.mjs' - : requestUrl.pathname === '/__workbench/diagnostics.mjs' - ? 'browser-diagnostics.mjs' - : null; + const browserSdkAssets: Record = { + '/__workbench/browser.mjs': 'browser.mjs', + '/__workbench/diagnostics.mjs': 'browser-diagnostics.mjs', + '/__workbench/browser-asr.mjs': 'browser-asr.mjs', + '/__workbench/asr-client.mjs': 'asr-client.mjs', + '/__workbench/host-error.mjs': 'host-error.mjs', + }; + const sdkAsset = browserSdkAssets[requestUrl.pathname] ?? null; let target: string; try { target = sdkAsset @@ -310,6 +357,32 @@ export async function createWorkbenchServer( if (actions.has(name)) throw new Error('APP_ACTION_DUPLICATE'); actions.set(name, handler); }, + agentTool(definition, handler) { + const name = definition?.name; + if (typeof name !== 'string' || name.length > 40 || !/^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$/u.test(name)) + throw new Error('APP_AGENT_TOOL_NAME_INVALID'); + if (agentTools.has(name)) throw new Error('APP_AGENT_TOOL_DUPLICATE'); + if ( + typeof definition.description !== 'string' || + !definition.description.trim() || + definition.description.length > 4000 + ) + throw new Error('APP_AGENT_TOOL_DESCRIPTION_INVALID'); + const parameters = definition.parameters; + if ( + !parameters || + typeof parameters !== 'object' || + Array.isArray(parameters) || + parameters.type !== 'object' || + JSON.stringify(parameters).length > 50_000 + ) + throw new Error('APP_AGENT_TOOL_PARAMETERS_INVALID'); + if (typeof handler !== 'function') throw new Error('APP_AGENT_TOOL_HANDLER_REQUIRED'); + agentTools.set(name, { + definition: { name, description: definition.description, parameters: structuredClone(parameters) }, + handler, + }); + }, tasks() { return database .prepare('SELECT id,kind,state,updated_at,error_code FROM _workbench_tasks ORDER BY updated_at DESC LIMIT 100') diff --git a/tests/host-connection.test.mjs b/tests/host-connection.test.mjs index 980bf9f..ecf1a51 100644 --- a/tests/host-connection.test.mjs +++ b/tests/host-connection.test.mjs @@ -83,6 +83,62 @@ test("客户端后启动、租约续期、重启重连和退出释放；只在 s assert.equal(disconnects, 2); }); +test("心跳瞬时失败在宽限期内保留租约，持续失败超过宽限期才重连", async () => { + const directory = await mkdtemp(join(tmpdir(), "floatboat-heartbeat-")); + await chmod(directory, 0o700); + let connects = 0; + let dropHeartbeats = false; + const server = createServer(async (req, res) => { + for await (const _chunk of req); + // 模拟 Host 主进程短暂繁忙：连接被直接断开，客户端拿不到任何 Host 错误码。 + if (req.url === "/heartbeat" && dropHeartbeats) return req.socket.destroy(); + res.setHeader("content-type", "application/json"); + if (req.url === "/status") res.end('{"protocol":2}'); + else if (req.url === "/connect") { + connects++; + res.end(JSON.stringify({ connection: { url: `${url}/workbenches/test/${connects}` }, lease: "c".repeat(64), resources: {} })); + } else res.end("{}"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const url = `http://127.0.0.1:${server.address().port}`; + await writeFile(join(directory, "test.json"), JSON.stringify({ protocol: 2, instance: "test", url }), { mode: 0o600 }); + const changes = []; + const options = { directory, intervalMs: 15, onStatus() {}, onChange: (value) => changes.push(value) }; + try { + const tolerant = await connectLocalHost({}, { ...options, leaseGraceMs: 60_000 }); + try { + assert.equal(connects, 1); + dropHeartbeats = true; + await delay(200); + assert.equal(connects, 1); + assert.equal(tolerant.current().connection.url, `${url}/workbenches/test/1`); + assert.ok(!changes.includes(null)); + } finally { + dropHeartbeats = false; + await tolerant.close(); + } + const strict = await connectLocalHost({}, { ...options, leaseGraceMs: 50 }); + try { + assert.equal(connects, 2); + dropHeartbeats = true; + for (let i = 0; i < 300 && connects < 3; i++) { + await delay(15); + if (strict.current() === null) dropHeartbeats = false; + } + assert.equal(connects, 3); + } finally { + dropHeartbeats = false; + await strict.close(); + } + } finally { + await new Promise((resolve) => { + server.close(resolve); + server.closeAllConnections(); + }); + await rm(directory, { recursive: true, force: true }); + } +}); + test("不使用可被其他用户改写的发现描述文件", async () => { if (!process.getuid) return; const directory = await mkdtemp(join(tmpdir(), "floatboat-discovery-mode-")); diff --git a/tooling/compiler/host-operations.ts b/tooling/compiler/host-operations.ts index f2bc704..ab76709 100644 --- a/tooling/compiler/host-operations.ts +++ b/tooling/compiler/host-operations.ts @@ -6,15 +6,34 @@ import type tsModule from 'typescript'; type TypeScript = typeof tsModule; const CONTRACTS = /(?:^|\/)sdk\/host-contracts(?:\.d)?\.mts$/u; -const TEXT_OPERATIONS = new Set(['llm.generateText', 'llm.streamText', 'llm.generateObject']); +const RUNTIME_CONTRACTS = /(?:^|\/)sdk\/runtime-contracts(?:\.d)?\.mts$/u; /** 旧客户端严格校验会拒收的文本调用字段。 */ -const FEATURE_FIELDS: Readonly> = { +const TEXT_FEATURE_FIELDS: Readonly> = { images: 'llm.images', messages: 'llm.messages', tools: 'llm.tools', toolChoice: 'llm.tools', reasoning: 'llm.reasoning', }; +/** 旧客户端严格校验会拒收的 Agent 调用字段。 */ +const AGENT_FEATURE_FIELDS: Readonly> = { instructions: 'agent.instructions' }; +const FEATURE_FIELDS_BY_OPERATION: Readonly>>> = { + 'llm.generateText': TEXT_FEATURE_FIELDS, + 'llm.streamText': TEXT_FEATURE_FIELDS, + 'llm.generateObject': TEXT_FEATURE_FIELDS, + 'agent.start': AGENT_FEATURE_FIELDS, + 'agent.conversations.create': AGENT_FEATURE_FIELDS, +}; +/** WorkbenchServer 上旧客户端不会调用的方法：旧 Host 不读取其注册结果，功能会静默缺失。 */ +const SERVER_FEATURES: Readonly> = { agentTool: 'agent.tools' }; + +function serverFeature(ts: TypeScript, declaration: tsModule.Declaration): string | undefined { + if (!ts.isMethodSignature(declaration) || !ts.isIdentifier(declaration.name)) return undefined; + if (!RUNTIME_CONTRACTS.test(declaration.getSourceFile().fileName.replaceAll('\\', '/'))) return undefined; + const parent = declaration.parent; + if (!ts.isInterfaceDeclaration(parent) || parent.name.text !== 'WorkbenchServer') return undefined; + return Object.hasOwn(SERVER_FEATURES, declaration.name.text) ? SERVER_FEATURES[declaration.name.text] : undefined; +} /** 方法声明位于 HostClient 内时返回点分路径，如 agent.conversations.create。 */ function operationPath(ts: TypeScript, declaration: tsModule.Declaration): string | undefined { @@ -45,6 +64,8 @@ export function collectWorkbenchHostOperations( const fromSymbol = (symbol: tsModule.Symbol | undefined): string | undefined => { let operation: string | undefined; for (const declaration of symbol?.declarations ?? []) { + const feature = serverFeature(ts, declaration); + if (feature) found.add(feature); const path = operationPath(ts, declaration); if (path) { found.add(path); @@ -88,25 +109,24 @@ export function collectWorkbenchHostOperations( return parts.some((part) => !(part.flags & (ts.TypeFlags.Undefined | ts.TypeFlags.Never))); }); - const recordFeatures = (argument: tsModule.Expression | undefined) => { + const recordFeatures = (argument: tsModule.Expression | undefined, fields: Readonly>) => { if (!argument) return; if (ts.isObjectLiteralExpression(argument)) { for (const property of argument.properties) { if (ts.isSpreadAssignment(property)) { const type = checker.getTypeAtLocation(property.expression); - for (const [field, feature] of Object.entries(FEATURE_FIELDS)) + for (const [field, feature] of Object.entries(fields)) if (typeUses(type, field, property)) found.add(feature); continue; } const name = property.name; - if (name && (ts.isIdentifier(name) || ts.isStringLiteral(name)) && Object.hasOwn(FEATURE_FIELDS, name.text)) - found.add(FEATURE_FIELDS[name.text]!); + if (name && (ts.isIdentifier(name) || ts.isStringLiteral(name)) && Object.hasOwn(fields, name.text)) + found.add(fields[name.text]!); } return; } const type = checker.getTypeAtLocation(argument); - for (const [field, feature] of Object.entries(FEATURE_FIELDS)) - if (typeUses(type, field, argument)) found.add(feature); + for (const [field, feature] of Object.entries(fields)) if (typeUses(type, field, argument)) found.add(feature); }; const visit = (node: tsModule.Node): void => { @@ -116,7 +136,8 @@ export function collectWorkbenchHostOperations( else if (ts.isBindingElement(node)) fromSymbol(bindingProperty(node)); if (ts.isCallExpression(node)) { const operation = resolveCallee(node.expression); - if (operation && TEXT_OPERATIONS.has(operation)) recordFeatures(node.arguments[0]); + const fields = operation ? FEATURE_FIELDS_BY_OPERATION[operation] : undefined; + if (fields) recordFeatures(node.arguments[0], fields); } ts.forEachChild(node, visit); }; diff --git a/tooling/contracts/workbenchManifest.ts b/tooling/contracts/workbenchManifest.ts index d97a554..cd8e6e5 100644 --- a/tooling/contracts/workbenchManifest.ts +++ b/tooling/contracts/workbenchManifest.ts @@ -80,8 +80,21 @@ export const workbenchManifestSchema = z ui: relativePath, migrations: relativePath, capabilities: z - .array(z.enum(['llm', 'agent', 'asr', 'workflows', 'connected-services', 'calendar', 'automation', 'workspaces'])) - .max(8) + .array( + z.enum([ + 'llm', + 'agent', + 'asr', + 'workflows', + 'connected-services', + 'calendar', + 'automation', + 'workspaces', + // 在 Agent 进度事件中提供截断后的工具参数与结果；需同时声明 agent。 + 'agent-tool-details', + ]), + ) + .max(9) .refine((values) => new Set(values).size === values.length, '能力声明不能重复') .default([]), dependencies: z.array(workbenchDependencySchema).max(64).optional(), @@ -109,6 +122,8 @@ export const workbenchManifestSchema = z validateWorkbenchAssetDeclarations(manifest, context); if (manifest.connectedServices?.length && !manifest.capabilities.includes('connected-services')) context.addIssue({ code: 'custom', path: ['connectedServices'], message: '服务声明需要 connected-services 能力' }); + if (manifest.capabilities.includes('agent-tool-details') && !manifest.capabilities.includes('agent')) + context.addIssue({ code: 'custom', path: ['capabilities'], message: 'agent-tool-details 需要同时声明 agent 能力' }); const services = manifest.connectedServices?.map(item => item.id) ?? []; if (new Set(services).size !== services.length) context.addIssue({ code: 'custom', path: ['connectedServices'], message: '服务逻辑标识不能重复' }); diff --git a/tooling/contracts/workbenchOperations.ts b/tooling/contracts/workbenchOperations.ts index 720a502..5767b0e 100644 --- a/tooling/contracts/workbenchOperations.ts +++ b/tooling/contracts/workbenchOperations.ts @@ -78,10 +78,17 @@ export const WORKBENCH_SDK_OPERATIONS = [ ] as const; /** - * 不新增方法、但旧客户端会因严格入参校验拒绝的能力特性。 - * 构建扫描在 llm 文本调用的参数中发现对应字段时记录。 + * 不新增 HostClient 方法、但旧客户端不支持的能力特性。 + * 构建扫描在 llm 文本调用或 Agent 调用的参数中发现对应字段，或服务端注册 Agent 工具时记录。 */ -export const WORKBENCH_SDK_FEATURES = ['llm.images', 'llm.messages', 'llm.tools', 'llm.reasoning'] as const; +export const WORKBENCH_SDK_FEATURES = [ + 'llm.images', + 'llm.messages', + 'llm.tools', + 'llm.reasoning', + 'agent.instructions', + 'agent.tools', +] as const; export const workbenchRequirementNameSchema = z .string() diff --git a/tooling/host-connection.mjs b/tooling/host-connection.mjs index 6eeaa2b..5306b07 100644 --- a/tooling/host-connection.mjs +++ b/tooling/host-connection.mjs @@ -36,6 +36,8 @@ export async function connectLocalHost( instance = process.env.FLOATBOAT_HOST_INSTANCE, offline = process.env.FLOATBOAT_HOST_MODE === "offline", intervalMs = 3000, + // 须小于 Host 租约有效期（默认 30 秒），宽限期内 Host 侧租约仍有效。 + leaseGraceMs = 20000, onChange = () => {}, onStatus = (code) => process.stdout.write(`[Floatboat] ${code}\n`), } = {}, @@ -43,6 +45,7 @@ export async function connectLocalHost( let selected = instance, descriptor, lease, + renewedAt = 0, timer, running, stopping = false, @@ -112,13 +115,35 @@ export async function connectLocalHost( ); return candidates.filter(Boolean); } + async function drop(error) { + if (lease) + await post(descriptor, "/disconnect", { lease }).catch(() => {}); + lease = undefined; + if (value) publish(null); + if (!stopping) + report( + /^[A-Z][A-Z_]+$/.test(error.message) + ? error.message + : "HOST_NOT_CONNECTED", + ); + } + async function renew() { + try { + await post(descriptor, "/heartbeat", { lease }); + renewedAt = Date.now(); + } catch (error) { + if (stopping) return; + // Host 返回错误码表示租约已失效，立即重连；超时或连接中断可能只是 Host 主进程短暂繁忙， + // 宽限期内保留租约继续续期，避免中断正在进行的调用。 + const rejected = /^[A-Z][A-Z_]+$/.test(error.message); + if (!rejected && Date.now() - renewedAt < leaseGraceMs) return; + await drop(error); + } + } async function tick() { if (stopping || offline) return; + if (lease) return renew(); try { - if (lease) { - await post(descriptor, "/heartbeat", { lease }); - return; - } const hosts = await discover(); if (hosts.length > 1) { report("HOST_INSTANCE_REQUIRED: set FLOATBOAT_HOST_INSTANCE"); @@ -134,6 +159,7 @@ export async function connectLocalHost( throw new Error("HOST_CONNECTION_INVALID"); safeAddress(result.connection.url, true); lease = result.lease; + renewedAt = Date.now(); selected = descriptor.instance; if (!stopping) { publish(result); @@ -141,16 +167,7 @@ export async function connectLocalHost( } } catch (error) { if (stopping) return; - if (lease) - await post(descriptor, "/disconnect", { lease }).catch(() => {}); - lease = undefined; - if (value) publish(null); - if (!stopping) - report( - /^[A-Z][A-Z_]+$/.test(error.message) - ? error.message - : "HOST_NOT_CONNECTED", - ); + await drop(error); } } const poll = () => {