From f398ab5a976841674ec148df81e9148772ab1e9f Mon Sep 17 00:00:00 2001 From: Amaan Date: Tue, 15 Sep 2026 20:00:27 +0530 Subject: [PATCH] feat/endpoint-commands: add endpoint commands: responses, chat_completions, moderations, embeddings Four thin commands that POST to the API endpoint they are named for, with whatever model the caller passes. They keep no model list, so a new or renamed model works without a CLI release. The Claude Code and OpenClaw plugins will call these instead of the per-task commands, which makes them independent of CLI model updates. - -m/--model (required), sent as-is; the API decides whether it exists - text from the positional argument, or stdin when there isn't one - responses/chat_completions: -i, --metadata, --raw; print the model text - moderations/embeddings: print the full response - --body on all four for extra top-level request fields - savings recorded like every other command - version 3.7.2 -> 3.8.0 Co-Authored-By: Claude Opus 5 (1M context) --- .claude/skills/model-sync/SKILL.md | 2 + README.md | 62 +++++++ docs/ADDING_COMMANDS.md | 3 + docs/DOCUMENTATION.md | 165 +++++++++++++++++++ package-lock.json | 4 +- package.json | 2 +- src/cli.ts | 8 + src/commands/chatCompletions.ts | 80 +++++++++ src/commands/embeddings.ts | 42 +++++ src/commands/moderations.ts | 40 +++++ src/commands/responses.ts | 67 ++++++++ src/lib/request.ts | 101 ++++++++++++ tests/endpointCommands.test.ts | 256 +++++++++++++++++++++++++++++ 13 files changed, 829 insertions(+), 3 deletions(-) create mode 100644 src/commands/chatCompletions.ts create mode 100644 src/commands/embeddings.ts create mode 100644 src/commands/moderations.ts create mode 100644 src/commands/responses.ts create mode 100644 src/lib/request.ts create mode 100644 tests/endpointCommands.test.ts diff --git a/.claude/skills/model-sync/SKILL.md b/.claude/skills/model-sync/SKILL.md index e0bae85..044dadb 100644 --- a/.claude/skills/model-sync/SKILL.md +++ b/.claude/skills/model-sync/SKILL.md @@ -135,6 +135,8 @@ There is no exemption list. A model the API does not return is not a ZeroGPU mod | `Text Generation` | pricing, `chat --model`, the `chat` Models tables and routing sentences | | every other task — `Summarization`, `Text Classification`, `Text Moderation`, `PII`, `Text Embedding`, and any new one | pricing only; a command calls it only when its `MODEL` constant already names it | +The endpoint commands — `responses`, `chat_completions`, `moderations`, `embeddings` — take the model from `-m` and hold no model list. That is deliberate: it is what keeps the agent plugins working across model changes without a CLI release. Never add a model list, route, or validation to them, and never delete them in [loop 3](#3-remove-what-is-gone); they have no model to lose. Model ids in their doc examples follow the normal rename and removal rules. + ## Never invent API-sourced facts only: id, task, `maxTokens`, input/output price, parameter count, use cases. Architecture details (`MoE`, `13B active`), language counts, and provider comparisons may be carried over from an existing notes cell while still true, or taken from `pricing.description` — never generated. Comparisons that follow from the payload's own prices ("its priciest") are allowed. Never invent a command, a flag, an example output, or a route. diff --git a/README.md b/README.md index 8b2df78..9ce192b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,11 @@ The official command-line interface for [ZeroGPU](https://zerogpu.ai) — run fa - [`extract_json`](#extract_json) - [`extract_pii`](#extract_pii) - [`redact_pii`](#redact_pii) + - [Endpoints](#endpoints) + - [`responses`](#responses) + - [`chat_completions`](#chat_completions) + - [`moderations`](#moderations) + - [`embeddings`](#embeddings) - [Environment Variables](#environment-variables) - [Output](#output) - [Troubleshooting](#troubleshooting) @@ -301,6 +306,63 @@ zerogpu redact_pii "Call Sarah at 415-555-0100 or email sarah@acme.com." --- +### Endpoints + +Call a ZeroGPU API endpoint directly, with any model. These commands keep no model list: the model you pass is sent as-is, so a model the platform adds or renames works without a CLI update. They are the stable surface the ZeroGPU agent plugins build on. + +Text comes from the positional argument, or from stdin when there isn't one, which keeps very large prompts off the command line. + +| Option | Applies to | Description | +|---|---|---| +| `-m, --model ` | all | **Required.** Model id, sent exactly as given. | +| `-i, --instructions ` | `responses`, `chat_completions` | `responses`: the `instructions` field. `chat_completions`: a system message ahead of the text. | +| `--metadata ` | `responses`, `chat_completions` | JSON object sent as `metadata` — the per-model options such as `usecase`, `labels`, `schema`, `threshold`. | +| `--body ` | all | Extra top-level request fields, e.g. `'{"max_output_tokens":256}'`. Fields set by the other options take precedence. | +| `--raw` | `responses`, `chat_completions` | Print the full API response instead of only the model's text. | + +`responses` and `chat_completions` print the model's text, pretty-printed when it is JSON. `moderations` and `embeddings` print the full response. Every call is recorded for `cost_savings`, like the task commands. + +#### `responses` + +POST to `/v1/responses`. + +```bash +zerogpu responses "Email John Smith at john@acme.com." -m gliner-multi-pii-v1 \ + --metadata '{"usecase":"redact","mask":"label"}' + +# Long input from a file, on stdin +zerogpu responses -m gpt-oss-120b -i "Summarize this report." < report.txt +``` + +#### `chat_completions` + +POST to `/v1/chat/completions`. Also available as `chat-completions`. + +```bash +zerogpu chat_completions "Explique la mise en cache en une phrase." -m qwen3-30b-a3b-fp8 + +zerogpu chat_completions "The app uses Python 3.11 and PostgreSQL 15." -m gliner2-base-v1 \ + --metadata '{"usecase":"ner","labels":["programming language","database"],"threshold":0.3}' +``` + +#### `moderations` + +POST to `/v1/moderations`. + +```bash +zerogpu moderations "Screen this comment before we publish it." -m zlm-v1-moderation-edge +``` + +#### `embeddings` + +POST to `/v1/embeddings`. + +```bash +zerogpu embeddings "ZeroGPU runs small models at the edge." -m all-minilm-l6-v2 +``` + +--- + ## Environment Variables | Variable | Purpose | diff --git a/docs/ADDING_COMMANDS.md b/docs/ADDING_COMMANDS.md index d4d77e1..b0af261 100644 --- a/docs/ADDING_COMMANDS.md +++ b/docs/ADDING_COMMANDS.md @@ -2,6 +2,8 @@ This guide explains how to add a new CLI command to the ZeroGPU CLI. +**First, check whether you need one.** The endpoint commands — `responses`, `chat_completions`, `moderations`, and `embeddings` — already reach every model on those endpoints: they send whatever `--model`, `--metadata`, and `--body` the caller gives. The Claude Code and OpenClaw plugins call them, so a new model needs no CLI change for the plugins to use it. Add a task command only when a dedicated, discoverable command is worth having for people using the CLI directly. Keep the endpoint commands free of model lists; that is what keeps the plugins independent of CLI releases. + ## Layout - `src/commands/` — one file per command, each exporting a `registerCommand(program)` function. @@ -9,6 +11,7 @@ This guide explains how to add a new CLI command to the ZeroGPU CLI. - `src/lib/responses.ts` — shared `RESPONSES_ENDPOINT`, `ResponsesApiResponse`, and the `extractOutputText` / `extractReasoningText` helpers for `/v1/responses` calls. - `src/lib/chatCompletions.ts` — the same for `/v1/chat/completions`, used by models the platform serves only there (currently `qwen3-30b-a3b-fp8`, `glm-5.2`, and `deepseek-v4-flash`), plus `toResponsesUsage` to normalize token counts for savings tracking. - `src/lib/auth.ts` — `getApiKey()` for authenticated requests. +- `src/lib/request.ts` — plumbing for the endpoint commands: `requireApiKey`, `resolveInput` (argument or stdin), `parseJsonObject`, `postJson`, and the print helpers. ## Steps diff --git a/docs/DOCUMENTATION.md b/docs/DOCUMENTATION.md index c610d31..e26e8bc 100644 --- a/docs/DOCUMENTATION.md +++ b/docs/DOCUMENTATION.md @@ -104,6 +104,12 @@ The CLI exposes the following commands: | [`summarize`](#413-summarize) | Summarize text with `llama-3.1-8b-instruct-fast` | | [`generate_followups`](#414-generate_followups) | Generate follow-up questions | | [`classify_domain`](#415-classify_domain) | Domain-level IAB classification | +| [`responses`](#416-responses) | Call `/v1/responses` with any model | +| [`chat_completions`](#417-chat_completions) | Call `/v1/chat/completions` with any model | +| [`moderations`](#418-moderations) | Call `/v1/moderations` with any model | +| [`embeddings`](#419-embeddings) | Call `/v1/embeddings` with any model | + +Commands 4.1–4.15 each wrap one task and pin its model. The endpoint commands, 4.16–4.19, take the model from `--model` and keep no model list, so a model the platform adds or renames works with them without a CLI release. ### Common exit codes | Code | Meaning | @@ -674,8 +680,167 @@ zerogpu classify_domain nytimes.com --- +### 4.16 `responses` + +Call the Responses API with any model. The model id is sent exactly as given — the CLI does not check it against a list, so the API decides whether it exists. + +**Synopsis** +``` +zerogpu responses [text] -m [-i ] [--metadata ] [--body ] [--raw] +``` + +**Parameters** + +| Name | Type | Required | Description | +|---|---|---|---| +| `text` (positional) | string | optional | Input text. When omitted, read from stdin, with trailing newlines dropped. | +| `-m`, `--model ` | string | **yes** | Model id. | +| `-i`, `--instructions ` | string | optional | Sent as `instructions`. | +| `--metadata ` | JSON object | optional | Sent as `metadata` — the model's options, e.g. `{"usecase":"redact","mask":"label"}`. | +| `--body ` | JSON object | optional | Extra top-level request fields. `model`, `input`, `instructions`, and `metadata` from the other options take precedence. | +| `--raw` | boolean | optional | Print the full response instead of the output text. | + +**Request body** +```jsonc +{ /* ...--body */ "model": "<--model>", "input": "", "instructions": "<-i>", "metadata": { /* --metadata */ } } +``` + +**Example** +```bash +zerogpu responses "Email John Smith at john@acme.com about invoice 12345." \ + -m gliner-multi-pii-v1 --metadata '{"usecase":"redact","mask":"label"}' + +zerogpu responses -m zlm-v1-iab-classify-edge < article.txt +``` + +**Expected output** +The output text (the `output_text` part of the `message` item, which skips any reasoning item), pretty-printed when it parses as JSON. With `--raw`, the whole response as JSON. +``` +Email [PERSON] at [EMAIL] about invoice 12345. +``` + +**Outcomes** + +| Outcome | Exit | +|---|---| +| Success — output printed | `0` | +| Not signed in | `1` | +| `--model` missing | `1` — `error: required option '-m, --model ' not specified` | +| `--metadata` or `--body` is not a JSON object | `1` — `Invalid --metadata JSON: ` or `--metadata must be a JSON object.` | +| No text in the argument or on stdin | `1` — `No input text. Pass it as an argument or pipe it on stdin.` | +| Network error (fetch threw) | `1` — `Request failed: ` | +| HTTP non-2xx, including an unknown model | `1` — `Request failed with status .` + body | +| Response missing output text (without `--raw`) | `1` — `Response did not contain any output text.` + raw JSON dump | + +--- + +### 4.17 `chat_completions` + +Call the Chat Completions API with any model. Alias: `chat-completions`. + +**Synopsis** +``` +zerogpu chat_completions [text] -m [-i ] [--metadata ] [--body ] [--raw] +``` + +**Parameters** + +| Name | Type | Required | Description | +|---|---|---|---| +| `text` (positional) | string | optional | Sent as the `user` message. When omitted, read from stdin, with trailing newlines dropped. | +| `-m`, `--model ` | string | **yes** | Model id. | +| `-i`, `--instructions ` | string | optional | Sent as a `system` message ahead of the user message. | +| `--metadata ` | JSON object | optional | Sent as `metadata`, e.g. `{"usecase":"ner","labels":["database"],"threshold":0.3}`. | +| `--body ` | JSON object | optional | Extra top-level request fields. `model`, `messages`, and `metadata` from the other options take precedence. | +| `--raw` | boolean | optional | Print the full response instead of the message content. | + +**Request body** +```jsonc +{ /* ...--body */ "model": "<--model>", "messages": [{ "role": "system", "content": "<-i>" }, { "role": "user", "content": "" }], "metadata": { /* --metadata */ } } +``` + +**Example** +```bash +zerogpu chat_completions "The application is built with Python 3.11 and uses PostgreSQL 15." \ + -m gliner2-base-v1 \ + --metadata '{"usecase":"ner","labels":["programming language","database"],"threshold":0.3}' +``` + +**Expected output** +`choices[0].message.content`, pretty-printed when it parses as JSON. The reasoning trace is not printed; use `--raw` to see it. +```json +{ + "entities": { + "programming language": ["Python 3.11"], + "database": ["PostgreSQL 15"] + } +} +``` + +**Outcomes** — as for [`responses`](#416-responses), except a response with no message content exits `1` with `Response did not contain any message content.` + raw JSON dump. + +--- + +### 4.18 `moderations` + +Call the Moderations API with any model and print the response. + +**Synopsis** +``` +zerogpu moderations [text] -m [--body ] +``` + +**Parameters** + +| Name | Type | Required | Description | +|---|---|---|---| +| `text` (positional) | string | optional | Text to screen. When omitted, read from stdin. | +| `-m`, `--model ` | string | **yes** | Model id, e.g. `zlm-v1-moderation-edge`. | +| `--body ` | JSON object | optional | Extra top-level request fields. `model` and `input` take precedence. | + +**Example** +```bash +zerogpu moderations "Screen this comment before we publish it." -m zlm-v1-moderation-edge +``` + +**Expected output** — the moderations response as JSON: `results[].flagged`, `results[].categories`, and `results[].category_scores`. + +**Outcomes** — as for [`responses`](#416-responses); there is no content to extract, so a successful response is always printed. + +--- + +### 4.19 `embeddings` + +Call the Embeddings API with any model and print the response. + +**Synopsis** +``` +zerogpu embeddings [text] -m [--body ] +``` + +**Parameters** + +| Name | Type | Required | Description | +|---|---|---|---| +| `text` (positional) | string | optional | Text to embed. When omitted, read from stdin. | +| `-m`, `--model ` | string | **yes** | Model id, e.g. `all-minilm-l6-v2` or `bge-small-en-v1.5`. | +| `--body ` | JSON object | optional | Extra top-level request fields. `model` and `input` take precedence. | + +**Example** +```bash +zerogpu embeddings "ZeroGPU runs small models at the edge." -m all-minilm-l6-v2 +``` + +**Expected output** — the embeddings response as JSON: `data[].embedding` holds the vector and `usage` the input tokens. + +**Outcomes** — as for [`moderations`](#418-moderations). + +--- + ## 5. Network & API Contract +The endpoint commands (4.16–4.19) POST to the endpoint they are named for — `/v1/responses`, `/v1/chat/completions`, `/v1/moderations`, `/v1/embeddings` — with the headers below. The rest of this section describes the task commands. + All inference commands POST to: ``` diff --git a/package-lock.json b/package-lock.json index 5517750..42cdbfb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "zerogpu-cli", - "version": "3.7.2", + "version": "3.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "zerogpu-cli", - "version": "3.7.2", + "version": "3.8.0", "license": "MIT", "dependencies": { "commander": "^12.1.0", diff --git a/package.json b/package.json index 6d0ae33..aa11087 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "zerogpu-cli", - "version": "3.7.2", + "version": "3.8.0", "description": "Command-line interface for ZeroGPU.", "type": "module", "bin": { diff --git a/src/cli.ts b/src/cli.ts index 86bcc6d..06e7b00 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url"; import { Command } from "commander"; import updateNotifier from "update-notifier"; import { registerChatCommand } from "./commands/chat.js"; +import { registerChatCompletionsCommand } from "./commands/chatCompletions.js"; import { registerChatThinkingCommand } from "./commands/chatThinking.js"; import { registerClassifyDomainCommand } from "./commands/classifyDomain.js"; import { registerClassifyIabCommand } from "./commands/classifyIab.js"; @@ -11,12 +12,15 @@ import { registerClassifyIabEnrichedCommand } from "./commands/classifyIabEnrich import { registerClassifyStructuredCommand } from "./commands/classifyStructured.js"; import { registerClassifyZeroShotCommand } from "./commands/classifyZeroShot.js"; import { registerCostSavingsCommand } from "./commands/costSavings.js"; +import { registerEmbeddingsCommand } from "./commands/embeddings.js"; import { registerExtractEntitiesCommand } from "./commands/extractEntities.js"; import { registerExtractJsonCommand } from "./commands/extractJson.js"; import { registerExtractPiiCommand } from "./commands/extractPii.js"; import { registerGenerateFollowupsCommand } from "./commands/generateFollowups.js"; import { registerLoginCommand } from "./commands/login.js"; +import { registerModerationsCommand } from "./commands/moderations.js"; import { registerRedactPiiCommand } from "./commands/redactPii.js"; +import { registerResponsesCommand } from "./commands/responses.js"; import { registerStatusCommand } from "./commands/status.js"; import { registerSummarizeCommand } from "./commands/summarize.js"; @@ -107,6 +111,10 @@ export function buildProgram(): Command { registerSummarizeCommand(program); registerChatCommand(program); registerChatThinkingCommand(program); + registerResponsesCommand(program); + registerChatCompletionsCommand(program); + registerModerationsCommand(program); + registerEmbeddingsCommand(program); registerCostSavingsCommand(program); return program; diff --git a/src/commands/chatCompletions.ts b/src/commands/chatCompletions.ts new file mode 100644 index 0000000..bcd4714 --- /dev/null +++ b/src/commands/chatCompletions.ts @@ -0,0 +1,80 @@ +import { Command } from "commander"; +import { + CHAT_COMPLETIONS_ENDPOINT, + toResponsesUsage, + type ChatCompletionsApiResponse, +} from "../lib/chatCompletions.js"; +import { + fail, + parseJsonObject, + postJson, + printContent, + printJson, + requireApiKey, + resolveInput, +} from "../lib/request.js"; +import { recordAndMaybeNotify } from "../lib/savings.js"; + +interface ChatCompletionsOptions { + model: string; + instructions?: string; + metadata?: string; + body?: string; + raw?: boolean; +} + +export function registerChatCompletionsCommand(program: Command): void { + program + .command("chat_completions [text]") + .alias("chat-completions") + .description( + "Call the Chat Completions API (/v1/chat/completions) with any model. Reads text from stdin when no argument is given.", + ) + .requiredOption("-m, --model ", "Model id, sent exactly as given.") + .option("-i, --instructions ", "System message sent ahead of the text.") + .option( + "--metadata ", + 'JSON object sent as `metadata`, e.g. \'{"usecase":"ner","labels":["person"],"threshold":0.3}\'', + ) + .option( + "--body ", + "JSON object of extra top-level request fields. Fields set by other options take precedence.", + ) + .option("--raw", "Print the full API response instead of only the message content.") + .action(async (text: string | undefined, opts: ChatCompletionsOptions) => { + const apiKey = requireApiKey(); + const extra = parseJsonObject(opts.body, "--body"); + const metadata = parseJsonObject(opts.metadata, "--metadata"); + const input = await resolveInput(text); + + const body: Record = { + ...extra, + model: opts.model, + messages: [ + ...(opts.instructions !== undefined + ? [{ role: "system", content: opts.instructions }] + : []), + { role: "user", content: input }, + ], + }; + if (metadata) body.metadata = metadata; + + const data = (await postJson( + CHAT_COMPLETIONS_ENDPOINT, + apiKey, + body, + )) as ChatCompletionsApiResponse; + + if (opts.raw) { + printJson(data); + } else { + const content = data.choices?.[0]?.message?.content; + if (!content) { + fail("Response did not contain any message content.", JSON.stringify(data, null, 2)); + } + printContent(content); + } + + recordAndMaybeNotify({ model: opts.model, usage: toResponsesUsage(data.usage) }); + }); +} diff --git a/src/commands/embeddings.ts b/src/commands/embeddings.ts new file mode 100644 index 0000000..fab22de --- /dev/null +++ b/src/commands/embeddings.ts @@ -0,0 +1,42 @@ +import { Command } from "commander"; +import { toResponsesUsage, type ChatCompletionsUsage } from "../lib/chatCompletions.js"; +import { + parseJsonObject, + postJson, + printJson, + requireApiKey, + resolveInput, +} from "../lib/request.js"; +import { recordAndMaybeNotify } from "../lib/savings.js"; + +export const EMBEDDINGS_ENDPOINT = "https://api.zerogpu.ai/v1/embeddings"; + +export function registerEmbeddingsCommand(program: Command): void { + program + .command("embeddings [text]") + .description( + "Call the Embeddings API (/v1/embeddings) with any model and print the response. Reads text from stdin when no argument is given.", + ) + .requiredOption("-m, --model ", "Model id, sent exactly as given.") + .option( + "--body ", + "JSON object of extra top-level request fields. Fields set by other options take precedence.", + ) + .action(async (text: string | undefined, opts: { model: string; body?: string }) => { + const apiKey = requireApiKey(); + const extra = parseJsonObject(opts.body, "--body"); + const input = await resolveInput(text); + + const data = (await postJson(EMBEDDINGS_ENDPOINT, apiKey, { + ...extra, + model: opts.model, + input, + })) as { usage?: ChatCompletionsUsage }; + + printJson(data); + + // Embedding models bill input tokens only; the API reports no completion + // tokens, so the output side is estimated and priced at zero. + recordAndMaybeNotify({ model: opts.model, usage: toResponsesUsage(data.usage) }); + }); +} diff --git a/src/commands/moderations.ts b/src/commands/moderations.ts new file mode 100644 index 0000000..5cab269 --- /dev/null +++ b/src/commands/moderations.ts @@ -0,0 +1,40 @@ +import { Command } from "commander"; +import { toResponsesUsage, type ChatCompletionsUsage } from "../lib/chatCompletions.js"; +import { + parseJsonObject, + postJson, + printJson, + requireApiKey, + resolveInput, +} from "../lib/request.js"; +import { recordAndMaybeNotify } from "../lib/savings.js"; + +export const MODERATIONS_ENDPOINT = "https://api.zerogpu.ai/v1/moderations"; + +export function registerModerationsCommand(program: Command): void { + program + .command("moderations [text]") + .description( + "Call the Moderations API (/v1/moderations) with any model and print the response. Reads text from stdin when no argument is given.", + ) + .requiredOption("-m, --model ", "Model id, sent exactly as given.") + .option( + "--body ", + "JSON object of extra top-level request fields. Fields set by other options take precedence.", + ) + .action(async (text: string | undefined, opts: { model: string; body?: string }) => { + const apiKey = requireApiKey(); + const extra = parseJsonObject(opts.body, "--body"); + const input = await resolveInput(text); + + const data = (await postJson(MODERATIONS_ENDPOINT, apiKey, { + ...extra, + model: opts.model, + input, + })) as { usage?: ChatCompletionsUsage }; + + printJson(data); + + recordAndMaybeNotify({ model: opts.model, usage: toResponsesUsage(data.usage) }); + }); +} diff --git a/src/commands/responses.ts b/src/commands/responses.ts new file mode 100644 index 0000000..52a83af --- /dev/null +++ b/src/commands/responses.ts @@ -0,0 +1,67 @@ +import { Command } from "commander"; +import { + fail, + parseJsonObject, + postJson, + printContent, + printJson, + requireApiKey, + resolveInput, +} from "../lib/request.js"; +import { + RESPONSES_ENDPOINT, + extractOutputText, + type ResponsesApiResponse, +} from "../lib/responses.js"; +import { recordAndMaybeNotify } from "../lib/savings.js"; + +interface ResponsesOptions { + model: string; + instructions?: string; + metadata?: string; + body?: string; + raw?: boolean; +} + +export function registerResponsesCommand(program: Command): void { + program + .command("responses [text]") + .description( + "Call the Responses API (/v1/responses) with any model. Reads text from stdin when no argument is given.", + ) + .requiredOption("-m, --model ", "Model id, sent exactly as given.") + .option("-i, --instructions ", "System instructions.") + .option( + "--metadata ", + 'JSON object sent as `metadata`, e.g. \'{"usecase":"redact","mask":"label"}\'', + ) + .option( + "--body ", + "JSON object of extra top-level request fields. Fields set by other options take precedence.", + ) + .option("--raw", "Print the full API response instead of only the output text.") + .action(async (text: string | undefined, opts: ResponsesOptions) => { + const apiKey = requireApiKey(); + const extra = parseJsonObject(opts.body, "--body"); + const metadata = parseJsonObject(opts.metadata, "--metadata"); + const input = await resolveInput(text); + + const body: Record = { ...extra, model: opts.model, input }; + if (opts.instructions !== undefined) body.instructions = opts.instructions; + if (metadata) body.metadata = metadata; + + const data = (await postJson(RESPONSES_ENDPOINT, apiKey, body)) as ResponsesApiResponse; + + if (opts.raw) { + printJson(data); + } else { + const content = extractOutputText(data); + if (!content) { + fail("Response did not contain any output text.", JSON.stringify(data, null, 2)); + } + printContent(content); + } + + recordAndMaybeNotify({ model: opts.model, usage: data.usage }); + }); +} diff --git a/src/lib/request.ts b/src/lib/request.ts new file mode 100644 index 0000000..c866bdc --- /dev/null +++ b/src/lib/request.ts @@ -0,0 +1,101 @@ +import { getApiKey } from "./auth.js"; + +// Shared plumbing for the endpoint commands — `responses`, `chat_completions`, +// `moderations`, and `embeddings`. They send whatever model the caller names +// and keep no model list, so a new or renamed model never needs a CLI release. + +export function fail(...lines: string[]): never { + for (const line of lines) console.error(line); + process.exit(1); +} + +export function requireApiKey(): string { + const resolved = getApiKey(); + if (!resolved) { + fail("You're not fully signed in yet. Run 'zerogpu login' to set your API key."); + } + return resolved.apiKey; +} + +export interface InputStream extends AsyncIterable { + isTTY?: boolean; +} + +// Text comes from the positional argument, or from stdin when it is piped, which +// keeps very large prompts off the command line. Trailing newlines from stdin are +// dropped, the same way shell `$(...)` drops them. +export async function resolveInput( + text: string | undefined, + stdin: InputStream = process.stdin, +): Promise { + let input = text; + if (input === undefined && !stdin.isTTY) { + const chunks: Buffer[] = []; + for await (const chunk of stdin) chunks.push(Buffer.from(chunk)); + input = Buffer.concat(chunks).toString("utf8").replace(/[\r\n]+$/, ""); + } + if (!input) fail("No input text. Pass it as an argument or pipe it on stdin."); + return input; +} + +export function parseJsonObject( + value: string | undefined, + flag: string, +): Record | undefined { + if (value === undefined) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + fail(`Invalid ${flag} JSON: ${message}`); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + fail(`${flag} must be a JSON object.`); + } + return parsed as Record; +} + +export async function postJson( + endpoint: string, + apiKey: string, + body: Record, +): Promise { + let response: Response; + try { + response = await fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": apiKey, + }, + body: JSON.stringify(body), + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + fail(`Request failed: ${message}`); + } + + if (!response.ok) { + const errBody = await response.text(); + fail(`Request failed with status ${response.status}.`, ...(errBody ? [errBody] : [])); + } + + return response.json(); +} + +// Pretty-print model output that is JSON; print anything else verbatim. +export function printContent(content: string): void { + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch { + console.log(content); + return; + } + console.log(JSON.stringify(parsed, null, 2)); +} + +export function printJson(payload: unknown): void { + console.log(JSON.stringify(payload, null, 2)); +} diff --git a/tests/endpointCommands.test.ts b/tests/endpointCommands.test.ts new file mode 100644 index 0000000..c279f17 --- /dev/null +++ b/tests/endpointCommands.test.ts @@ -0,0 +1,256 @@ +import { describe, it, expect, beforeEach, afterEach, vi, type MockInstance } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { buildProgram } from "../src/cli.js"; +import { CHAT_COMPLETIONS_ENDPOINT } from "../src/lib/chatCompletions.js"; +import { resolveInput } from "../src/lib/request.js"; +import { RESPONSES_ENDPOINT } from "../src/lib/responses.js"; +import { readSavings } from "../src/lib/savings.js"; + +let tmpHome: string; +let originalHome: string | undefined; +let originalApiKey: string | undefined; +let fetchMock: ReturnType; +let logSpy: MockInstance; +let errorSpy: MockInstance; + +function respondWith(payload: unknown, status = 200): void { + fetchMock.mockResolvedValue( + new Response(typeof payload === "string" ? payload : JSON.stringify(payload), { status }), + ); +} + +function sentRequest(): { url: string; headers: Record; body: Record } { + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + return { + url, + headers: init.headers as Record, + body: JSON.parse(init.body as string) as Record, + }; +} + +function printed(): string { + return logSpy.mock.calls.map((args) => args.join(" ")).join("\n"); +} + +async function zerogpu(...args: string[]): Promise { + await buildProgram().parseAsync(["node", "zerogpu", ...args]); +} + +beforeEach(() => { + tmpHome = mkdtempSync(join(tmpdir(), "zerogpu-endpoint-test-")); + originalHome = process.env["HOME"]; + originalApiKey = process.env["ZEROGPU_API_KEY"]; + process.env["HOME"] = tmpHome; + process.env["ZEROGPU_API_KEY"] = "zgpu-api-test"; + + fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + vi.spyOn(process, "exit").mockImplementation((code) => { + throw new Error(`exit ${code}`); + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + if (originalHome === undefined) delete process.env["HOME"]; + else process.env["HOME"] = originalHome; + if (originalApiKey === undefined) delete process.env["ZEROGPU_API_KEY"]; + else process.env["ZEROGPU_API_KEY"] = originalApiKey; + rmSync(tmpHome, { recursive: true, force: true }); +}); + +describe("responses", () => { + const payload = { + output: [ + { type: "reasoning", content: [{ type: "reasoning_text", text: "thinking" }] }, + { type: "message", content: [{ type: "output_text", text: '{"label":"sports"}' }] }, + ], + usage: { input_tokens: 12, output_tokens: 4 }, + }; + + it("posts the model, input, instructions, and metadata, and prints the output text", async () => { + respondWith(payload); + await zerogpu( + "responses", + "The Lakers won.", + "-m", + "gliner2-base-v1", + "-i", + "Be brief.", + "--metadata", + '{"usecase":"ner","labels":["team"]}', + ); + + const req = sentRequest(); + expect(req.url).toBe(RESPONSES_ENDPOINT); + expect(req.headers["x-api-key"]).toBe("zgpu-api-test"); + expect(req.body).toEqual({ + model: "gliner2-base-v1", + input: "The Lakers won.", + instructions: "Be brief.", + metadata: { usecase: "ner", labels: ["team"] }, + }); + expect(printed()).toBe(JSON.stringify({ label: "sports" }, null, 2)); + }); + + it("sends a model id no list in the CLI knows about", async () => { + respondWith(payload); + await zerogpu("responses", "hi", "-m", "some-model-released-tomorrow"); + expect(sentRequest().body.model).toBe("some-model-released-tomorrow"); + }); + + it("merges --body fields, with option-set fields taking precedence", async () => { + respondWith(payload); + await zerogpu( + "responses", + "hi", + "-m", + "gpt-oss-120b", + "--body", + '{"max_output_tokens":64,"model":"ignored"}', + ); + expect(sentRequest().body).toEqual({ + max_output_tokens: 64, + model: "gpt-oss-120b", + input: "hi", + }); + }); + + it("prints the whole response with --raw", async () => { + respondWith(payload); + await zerogpu("responses", "hi", "-m", "gpt-oss-120b", "--raw"); + expect(printed()).toBe(JSON.stringify(payload, null, 2)); + }); + + it("records savings under the requested model", async () => { + respondWith(payload); + await zerogpu("responses", "hi", "-m", "gliner2-base-v1"); + const savings = readSavings(); + expect(savings.totalRequests).toBe(1); + expect(savings.totalTokens).toBe(16); + expect(savings.byModel["gliner2-base-v1"]?.requests).toBe(1); + }); + + it("exits 1 when the response has no output text", async () => { + respondWith({ output: [] }); + await expect(zerogpu("responses", "hi", "-m", "gpt-oss-120b")).rejects.toThrow("exit 1"); + expect(errorSpy).toHaveBeenCalledWith("Response did not contain any output text."); + }); +}); + +describe("chat_completions", () => { + const payload = { + choices: [{ message: { content: "Hello there.", reasoning: "hidden" } }], + usage: { prompt_tokens: 20, completion_tokens: 3, total_tokens: 23 }, + }; + + it("sends instructions as a system message plus metadata, and prints the content", async () => { + respondWith(payload); + await zerogpu( + "chat_completions", + "Say hi.", + "-m", + "qwen3-30b-a3b-fp8", + "-i", + "You are terse.", + "--metadata", + '{"usecase":"ner"}', + ); + + const req = sentRequest(); + expect(req.url).toBe(CHAT_COMPLETIONS_ENDPOINT); + expect(req.body).toEqual({ + model: "qwen3-30b-a3b-fp8", + messages: [ + { role: "system", content: "You are terse." }, + { role: "user", content: "Say hi." }, + ], + metadata: { usecase: "ner" }, + }); + expect(printed()).toBe("Hello there."); + }); + + it("is reachable by its dashed alias and maps usage for savings", async () => { + respondWith(payload); + await zerogpu("chat-completions", "Say hi.", "-m", "glm-5.2"); + expect(sentRequest().body.messages).toEqual([{ role: "user", content: "Say hi." }]); + expect(readSavings().totalTokens).toBe(23); + }); +}); + +describe("moderations and embeddings", () => { + it("posts to /v1/moderations and prints the envelope", async () => { + const payload = { results: [{ flagged: false }], usage: { prompt_tokens: 5 } }; + respondWith(payload); + await zerogpu("moderations", "hello", "-m", "zlm-v1-moderation-edge"); + expect(sentRequest().url).toBe("https://api.zerogpu.ai/v1/moderations"); + expect(sentRequest().body).toEqual({ model: "zlm-v1-moderation-edge", input: "hello" }); + expect(printed()).toBe(JSON.stringify(payload, null, 2)); + }); + + it("posts to /v1/embeddings and prints the envelope", async () => { + const payload = { data: [{ index: 0, embedding: [0.1, 0.2] }], usage: { prompt_tokens: 2 } }; + respondWith(payload); + await zerogpu("embeddings", "hello", "-m", "all-minilm-l6-v2"); + expect(sentRequest().url).toBe("https://api.zerogpu.ai/v1/embeddings"); + expect(printed()).toBe(JSON.stringify(payload, null, 2)); + expect(readSavings().byModel["all-minilm-l6-v2"]?.requests).toBe(1); + }); +}); + +describe("shared failures", () => { + it("exits 1 with the status and body on a non-2xx response", async () => { + respondWith('{"error":"invalid_api_key"}', 401); + await expect(zerogpu("embeddings", "hi", "-m", "all-minilm-l6-v2")).rejects.toThrow("exit 1"); + expect(errorSpy).toHaveBeenCalledWith("Request failed with status 401."); + expect(errorSpy).toHaveBeenCalledWith('{"error":"invalid_api_key"}'); + expect(readSavings().totalRequests).toBe(0); + }); + + it("rejects invalid --metadata before sending anything", async () => { + await expect( + zerogpu("responses", "hi", "-m", "gliner2-base-v1", "--metadata", "[1,2]"), + ).rejects.toThrow("exit 1"); + expect(errorSpy).toHaveBeenCalledWith("--metadata must be a JSON object."); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("exits 1 when not signed in", async () => { + delete process.env["ZEROGPU_API_KEY"]; + await expect(zerogpu("moderations", "hi", "-m", "zlm-v1-moderation-edge")).rejects.toThrow( + "exit 1", + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe("resolveInput", () => { + function stream(text: string, isTTY = false) { + return Object.assign( + (async function* () { + yield Buffer.from(text); + })(), + { isTTY }, + ); + } + + it("reads piped stdin and drops trailing newlines", async () => { + expect(await resolveInput(undefined, stream("line one\nline two\n\n"))).toBe( + "line one\nline two", + ); + }); + + it("prefers the positional argument over stdin", async () => { + expect(await resolveInput("from argv", stream("from stdin"))).toBe("from argv"); + }); + + it("exits 1 when there is no text at all", async () => { + await expect(resolveInput(undefined, stream("", true))).rejects.toThrow("exit 1"); + }); +});