diff --git a/.agents/skills/floatboat-workbench/SKILL.md b/.agents/skills/floatboat-workbench/SKILL.md index 6c437ea..c7e3b1f 100644 --- a/.agents/skills/floatboat-workbench/SKILL.md +++ b/.agents/skills/floatboat-workbench/SKILL.md @@ -36,7 +36,7 @@ These topics are also available without installing dependencies through `node to | Browser connection, theme and Host contributions | [ui](references/ui.md) | | Business tables, migrations and transaction boundaries | [database](references/database.md) | | Workflow/Combo declarations and portable file resources | [resources](references/resources.md) | -| Model text, structured output or streaming | [llm](references/llm.md) | +| Model text, structured output, streaming, image input, speech, image generation, Jev decisions or model list | [llm](references/llm.md) | | Agent tasks with tools and an explicit execution directory | [agent](references/agent.md) | | Streaming speech recognition | [asr](references/asr.md) | | Starting or observing a declared Workflow | [workflows](references/workflows.md) | diff --git a/.agents/skills/floatboat-workbench/references/llm.md b/.agents/skills/floatboat-workbench/references/llm.md index ba3a5f9..535fcb3 100644 --- a/.agents/skills/floatboat-workbench/references/llm.md +++ b/.agents/skills/floatboat-workbench/references/llm.md @@ -2,10 +2,41 @@ Declare `llm` in manifest.capabilities. Call the SDK on the server; the browser uses an app API route. With Floatboat running, standalone preview uses the real Host model service. Handle `HOST_NOT_CONNECTED` while offline and verify real output; see repository `docs/platform.md`. +## Text, structured output and streaming + - `app.host.llm.generateText({ prompt, ...options })` returns text, model, requestId, finishReason and optional usage. - `generateObject({ prompt, schema })` validates output against JSON Schema. Treat the returned object as unknown until checked against the business type. - `streamText({ prompt })` returns an async iterable with delta, result and done events. Preserve incomplete/interrupted state if the stream ends without the required terminal result. - -Use `app.host.capabilities.list({ domain: 'llm' })` to inspect the connected contract. The platform owns model routing and credentials; there is no app-supplied provider API key field. Read `sdk/host-contracts.mts` (or generated `app/sdk/host-contracts.d.mts`) for supported options. Match business UI loading/cancel/error behavior to actual returned state rather than fabricated text. +- All three accept `images` (up to 8, each at most 10 MiB): `{ resource: { resourceId, relativePath } }` for a file in a declared directory resource, `{ url: 'https://…' }` that the Host downloads, or a small `{ dataUrl: 'data:image/png;base64,…' }` (the whole request is limited to 2 MiB). Only PNG, JPEG, WebP and GIF are accepted. A model or protocol without image input returns `LLM_IMAGE_INPUT_UNSUPPORTED` before any charge; check `inputs` from `listModels` first. Calls that name no model use `DEFAULT_LLM_MODEL` (`gemini-3.8-flash`, exported by `sdk/host.mts`); omit `model` unless the product needs a different one. To choose one, pass a Host-supported model ID, for example `generateText({ model: 'gemini-3.8-flash', prompt: 'Hello' })`. The Host resolves the model catalog and selects Chat Completions, Responses, Anthropic or Gemini protocol, endpoint and authorization automatically. Configured `local/` IDs use their registered endpoint. Do not infer a provider from a model-name prefix or supply your own endpoint. Text, streaming and JSON Schema generation follow the same routing. Real verification of one model does not prove every upstream account/model is available. + +## Model catalog + +`listModels({ kind? })` returns `models` (`id`, `kinds` of `text`/`speech`/`image`/`decision`, `inputs` of `text`/`image`, `source`), per-kind `defaults`, and `media`. Text models follow the same release, membership and local-model rules as the Floatboat chat model picker. Speech, image and decision models come from the platform catalog; when `media` is `unavailable` that part may be incomplete, so keep an explicit retry instead of treating it as "no models". Choose models from this result rather than hard-coding names. + +## Speech and images + +Both write results into a manifest resource with `kind: 'directory'` and `access: 'read-write'`; pass a logical `output: { resourceId, relativePath }`, never an absolute path. Results return the resource reference, byte length and format/MIME only. + +- `generateSpeech({ text, voice?, format?, speed?, instructions?, model?, output })` synthesizes up to 4096 characters. `format` is `mp3` (default), `wav`, `opus`, `aac`, `flac` or `pcm`. +- `generateImage({ prompt, size?, quality?, count?, images?, model?, output })` creates 1–4 images. With `images` it edits from those references. For `count > 1`, files get `-1`, `-2` before the extension. The default request deadline is 330 seconds; show progress instead of blocking the page. + +Omitting `model` uses the catalog default for that kind; if the catalog is unavailable the call returns `LLM_MODEL_REQUIRED`. + +## Decisions (Jev) + +`decide({ state, questions, model? })` asks the platform decision model (default `jev-latest`) structured questions about `state` (string, object or array). `questions` holds 1–32 entries keyed by `[A-Za-z0-9_-]{1,64}`, each `{ type: 'noul' | 'choice' | 'score', instructions, criteria? }`; `choice` and `score` require `criteria`. It returns the raw `answers`. Use it for yes/no, option and scoring judgements; it is cheaper and steadier than prompting a text model, and does not generate prose. + +```ts +const result = await app.host.llm.decide({ + state: { message: ticket.body }, + questions: { urgent: { type: 'noul', instructions: 'Does the customer need a reply within an hour?' } }, +}); +``` + +## Errors and billing + +The platform owns model routing, credentials and billing; there is no app-supplied provider API key field. When credits run out every call returns `HOST_CREDITS_EXHAUSTED` with `execution: 'not-executed'`, and Floatboat shows its own "out of credits" prompt in the main window. Show a retryable state in the app and do not loop retries. Other upstream failures return `LLM_HTTP_`. + +Use `app.host.capabilities.list({ domain: 'llm' })` to inspect the connected contract; older Hosts report the newer operations as unavailable. Read `sdk/host-contracts.mts` (or generated `app/sdk/host-contracts.d.mts`) for supported options. Match business UI loading/cancel/error behavior to actual returned state rather than fabricated text. diff --git a/sdk/host-contracts.mts b/sdk/host-contracts.mts index 5645c0c..3739a1b 100644 --- a/sdk/host-contracts.mts +++ b/sdk/host-contracts.mts @@ -25,12 +25,114 @@ export interface CallOptions { timeoutMs?: number; } +/** manifest 声明的目录资源中的相对路径；Host 不向工作台暴露本机绝对路径。 */ +export interface ResourceReference { + resourceId: string; + relativePath: string; +} + +/** 图片输入三选一：声明资源文件、https 地址（由 Host 下载）或图片 data URL；单张不超过 10 MiB。 */ +export type ImageInput = { resource: ResourceReference } | { url: string } | { dataUrl: string }; + export interface ModelInput { prompt: string; model?: string; system?: string; maxOutputTokens?: number; temperature?: number; + /** 最多 8 张；模型或其协议不支持图片时返回 LLM_IMAGE_INPUT_UNSUPPORTED 且不计费。 */ + images?: ImageInput[]; +} + +export type ModelKind = 'text' | 'speech' | 'image' | 'decision'; + +export interface ModelCatalogEntry { + id: string; + kinds: ModelKind[]; + inputs: Array<'text' | 'image'>; + source: 'platform' | 'local'; +} + +export interface ModelCatalog { + models: ModelCatalogEntry[]; + /** 各种类在未指定 model 时使用的模型。 */ + defaults: Partial>; + /** 平台媒体目录状态；unavailable 时语音、图片与决策列表可能不完整。 */ + media: 'available' | 'unavailable'; +} + +export interface ModelUsage { + inputTokens: number; + outputTokens: number; + totalTokens: number; +} + +export interface StoredMedia extends ResourceReference { + byteLength: number; +} + +export type SpeechFormat = 'mp3' | 'wav' | 'opus' | 'aac' | 'flac' | 'pcm'; + +export interface SpeechInput { + text: string; + model?: string; + voice?: string; + format?: SpeechFormat; + speed?: number; + instructions?: string; + /** 必须是 access 为 read-write 的目录资源。 */ + output: ResourceReference; +} + +export interface SpeechResult { + model: string; + output: StoredMedia & { format: SpeechFormat }; + requestId: string; +} + +export interface ImageGenerationInput { + prompt: string; + model?: string; + /** 如 1024x1024，或 auto。 */ + size?: string; + quality?: 'auto' | 'low' | 'medium' | 'high' | 'standard' | 'hd'; + /** 1–4 张；多张时在 output 文件名后追加 -1、-2。 */ + count?: number; + /** 参考图；提供时使用图片编辑接口。 */ + images?: ImageInput[]; + /** 必须是 access 为 read-write 的目录资源。 */ + output: ResourceReference; +} + +export interface ImageGenerationResult { + model: string; + images: Array; + revisedPrompt?: string; + usage?: ModelUsage; + requestId: string; +} + +type DecisionInstructions = string | Json[] | { [key: string]: Json }; + +export type DecisionQuestion = + | { type: 'noul'; instructions: DecisionInstructions; criteria?: Json } + | { type: 'choice'; instructions: DecisionInstructions; criteria: Json } + | { type: 'score'; instructions: DecisionInstructions; criteria: Json }; + +export interface DecisionInput { + /** 默认 jev-latest。 */ + model?: string; + state: string | Json[] | { [key: string]: Json }; + /** 1–32 个问题，键为 [A-Za-z0-9_-]{1,64}。 */ + questions: Record; +} + +export interface DecisionResult { + model: string; + /** Jev 原样返回的回答，键与 questions 对应。 */ + answers: Record; + usage?: ModelUsage; + requestId: string; } export interface ModelResult { @@ -209,6 +311,11 @@ export interface HostClient { options?: CallOptions, ): Promise; streamText(input: ModelInput, options?: CallOptions): AsyncIterable; + listModels(input?: { kind?: ModelKind }): Promise; + generateSpeech(input: SpeechInput, options?: CallOptions): Promise; + /** 生成较慢，默认请求期限 330 秒。 */ + generateImage(input: ImageGenerationInput, options?: CallOptions): Promise; + decide(input: DecisionInput, options?: CallOptions): Promise; }; workflows: { start(input: WorkflowStart, options?: CallOptions): Promise<{ runId: string; idempotencyKey: string }>; diff --git a/sdk/host.mts b/sdk/host.mts index 4e19929..df9e838 100644 --- a/sdk/host.mts +++ b/sdk/host.mts @@ -7,6 +7,10 @@ import type { HostConnection, ModelEvent, ModelResult, + ModelCatalog, + SpeechResult, + ImageGenerationResult, + DecisionResult, WorkflowEvent, WorkflowRun, AgentTaskEvent, @@ -265,6 +269,13 @@ export function createHostClient( generateObject: (input, options) => request('llm', 'generateObject', withDefaultModel(input), options), streamText: (input, options) => events('llm', 'streamText', withDefaultModel(input), options), + // 媒体与决策模型的默认值由 Host 模型目录决定，不套用文本默认模型。 + listModels: (input = {}) => request('llm', 'listModels', input), + generateSpeech: (input, options = {}) => + request('llm', 'generateSpeech', input, { timeoutMs: 150_000, ...options }), + generateImage: (input, options = {}) => + request('llm', 'generateImage', input, { timeoutMs: 330_000, ...options }), + decide: (input, options) => request('llm', 'decide', input, options), }, workflows: { start: (input, options) => request('workflows', 'start', input, options),