diff --git a/.env.example b/.env.example index 33f9879..da6c33e 100644 --- a/.env.example +++ b/.env.example @@ -3,9 +3,11 @@ # ora.ai host and need none of these — `ax audit ` works with zero # configuration, and so does `ax webmcp-audit http://localhost:3000`. -# Secret API key (looks like ora_sk_...). Required by `journey` and by -# `audit --tunnel ora` (needs the tunnels:write + tunnels:connect scopes), which -# exchange it for a short-lived bearer token. +# Front API key (looks like ora_sk_...; made in your Front account at app.ora.ai). +# Required by `journey` and `audit --tunnel ora`. A set of journeys needs the +# journeys:write, journeys:read, runs:read, runs:write and intents:read scopes; +# the tunnel needs tunnels:write + tunnels:connect. Each exchanges the key for a +# short-lived bearer token. ORA_API_KEY= # Platform API base. Defaults to https://api.agentfront.sh (production) if unset. diff --git a/README.md b/README.md index 4b73936..4ebe8f9 100644 --- a/README.md +++ b/README.md @@ -18,18 +18,17 @@ npx ax audit https://stripe.com ``` -**No account, no API key.** Everything below works out of the box. The exceptions are [`journey`](#for-ora-partners) and `audit --tunnel ora`, which need an ora platform key, and `deep-journey --task`, which needs a partner key. +**No account, no API key** for `audit`, `webmcp-audit`, `journey --domain ` on one of ora's curated tasks, `skill` and `generate-files`. The exceptions are [`journey`](#journey) on a goal of your own and `audit --tunnel ora`, which need a Front API key from your Front account at app.ora.ai.

ax audit https://stripe.com — live progress, then the layered report and ora's ranked top fixes

-Six commands: +Five commands: - **`audit `** — run ora's hosted agent-readiness audit against a site: live progress, then a layered report of what passed, what's broken, and ora's ranked list of the highest-impact fixes (or `--json` for the raw contract payload). No account or API key needed. Gate CI with `--min-score`. - **`webmcp-audit `** — audit a page's `document.modelContext` tools in a real browser. Works on any http(s) URL, localhost included, so you can check a surface before you publish it. Measures the page the way ora's capture worker does and has ora score it; nothing is stored, published, or ranked. -- **`deep-journey `** — run a real AI agent at a site on one of ora's curated tasks through the public journey API: no key, no workspace. A partner API key unlocks free-text tasks (`--task`) and a larger allowance. -- **`journey ""`** — send a real AI agent (claude-code, codex, …) at a site and watch its navigation live as a boxed node-graph, then get the scored insight, tokens, and cost. Partner-only: needs an `ORA_API_KEY` (see [For ora partners](#for-ora-partners)). +- **`journey`** - send a real AI agent at a site and watch it live. With no goal of your own (`ax journey --domain stripe.com`) it runs one of ora's curated tasks through ora's public journey API: no key, no workspace. With a goal (`ax journey "" --domain stripe.com`) it sends a real agent (claude-code, codex, …) on that goal and draws its navigation live as a graph of boxed nodes, then the scored insight, tokens, and cost. Ask for several goals or agents, `--repeat`, or levels of help with `--levels`, and it runs a set of journeys instead and counts how often they finish, which shows where agents lose your site. Runs on your Front account at app.ora.ai and needs a Front API key in `ORA_API_KEY` (see [Get a key](#get-a-key)). - **`skill [name]`** — list, print, or install ora's agent skills, digest-verified from the public registry. - **`generate-files `** — draft the files agents look for on a site (`ard.json`, `ai-catalog.json`, `poppy.json`, `llms.txt`, a `robots.txt` edit, an API catalog, MCP server cards), filled in from ora's last scan of the domain. @@ -97,7 +96,7 @@ Budget notes: ora allows 30 scans + 6 `--force` scans per rolling 24h per IP (pl A local dev server only exists on your machine, so ora can't reach it. The simplest option is to audit a publicly reachable deployment of the same code (e.g. a preview URL). To audit localhost itself, run it through a tunnel. The result is stored as **ephemeral** either way: excluded from ora's rankings and leaderboard, served with `Cache-Control: no-store`, never indexed, and deleted after 7 days. -**ora's own tunnel.** With an ora platform key (`ora_sk_…`, scopes `tunnels:write` + `tunnels:connect`) in `ORA_API_KEY`, `ax` creates a tunnel, connects it, audits the public hostname, and deletes the tunnel when done — nothing to install, nothing left behind: +**ora's own tunnel.** With a Front API key (`ora_sk_…`, scopes `tunnels:write` + `tunnels:connect`) in `ORA_API_KEY`, `ax` creates a tunnel, connects it, audits the public hostname, and deletes the tunnel when done - nothing to install, nothing left behind: ``` ax audit localhost:3000 --tunnel ora @@ -211,17 +210,35 @@ page did not load, or a capture ora would refuse) · `3` API, a failed capture, returned. The gate reads the score, not the grade — a withheld grade would otherwise let everything through. -## deep-journey +## journey ``` -ax deep-journey [--intent id | --task text] [--agent id] [--api-key k] [--no-stream] [--json] +npx ax journey --domain stripe.com --intent pricing # free, no key +npx ax journey "Find the API docs and how to authenticate" --domain stripe.com # a goal of your own ``` +`ax journey` sends a real AI agent at a site and shows you whether it finished and where it got stuck. It runs on one of two systems, picked by what you type: + +- **No goal of your own:** one of ora's curated tasks, on ora.ai's public journeys. No key, no account. See [Free, on a curated task](#free-on-a-curated-task). +- **A goal of your own:** on your Front account at app.ora.ai, with a Front API key in `ORA_API_KEY`. One goal and one agent, with no `--repeat` and no levels: ax runs one journey and draws it live. Ask for more (several goals or agents, `--repeat`, or levels with `--levels`, `--docs`, `--exact` or `--brand`) and ax runs a set of journeys: it previews the set, asks before launching, waits for the platform's verdicts and counts them. [Setup](#setup) covers one journey, [Get a key](#get-a-key) the scopes a set needs. + +The flags of the two kinds cannot be mixed: `--intent`, `--agent`, `--api-key` and `--no-stream` are for the free path, and `--harness`, `--model`, `--agents`, `--levels`, `--brand`, `--docs`, `--exact`, `--repeat`, `--yes` and `--wait` for your Front account. A goal with one of the free flags runs on ora's public journeys, which takes a partner key; a goal alone, with only a partner key set and no `ORA_API_KEY`, does too. + + + +### Free, on a curated task + ``` -ax deep-journey stripe.com --intent pricing -ax deep-journey zapier.com --task "Find how to build a Slack → Sheets zap" # needs a partner key +ax journey --domain [--intent id] [--agent id] [--api-key k] [--no-stream] [--json] ``` +``` +ax journey --domain stripe.com --intent pricing +ax journey "Find how to build a Slack → Sheets zap" --domain zapier.com --agent cas-haiku # needs a partner key +``` + +`ax deep-journey ` is the older name for this path. It still works, with the same flags (`--task` included) and exit codes, and prints the `ax journey` call it now stands for; it will be removed in a future major release. + Runs a real AI agent at a site through ora's **public** journey API — no key, no workspace needed for curated tasks. The agent's trajectory streams live as progress lines; the terminal output is ora's verdict, the billable step count, and the generated insight: ``` @@ -238,22 +255,244 @@ Runs a real AI agent at a site through ora's **public** journey API — no key, Anonymous callers run curated intents (`pricing`, `signup`, `api-docs`, `integrate`, `support`, …) — list them with `GET https://ora.ai/api/journey/intents`, and the publicly offered agents with `GET https://ora.ai/api/journey/agents`. The public caps are 100 runs per rolling 24h per target and task, and 20 runs per 24h per IP (plus a 20/min burst limit and a shared ceiling of 3,000 anonymous runs a day across all callers); a target at its cap answers with the most recent stored run (`rate_limited: true`) instead of an error, so repeat CI invocations are cheap. Verdict and step count come from ora's contract as-is — the CLI never re-judges a run. -**Keyed tier:** an ora-issued partner API key (`--api-key`, or `ORA_PARTNER_API_KEY` / `ORA_SCAN_API_KEY` in the environment) unlocks `--task` — a free-text task of 4–300 characters that replaces the curated intent — and moves the caller to an allowance of 1000 runs per rolling 24h per key with no per-target cap and no burst guard. Keys are issued manually by ora (no self-serve signup). A wrong or unrecognized key with a *curated* intent silently degrades to the anonymous tier; `--task` without a recognized key is an error. +**Keyed tier:** an ora-issued partner API key (`--api-key`, or `ORA_PARTNER_API_KEY` / `ORA_SCAN_API_KEY` in the environment) unlocks a goal of your own on this path — free text of 4–300 characters that replaces the curated intent — and moves the caller to an allowance of 1000 runs per rolling 24h per key with no per-target cap and no burst guard. Keys are issued manually by ora (no self-serve signup). A wrong or unrecognized key with a *curated* intent silently degrades to the anonymous tier; a goal of your own without a recognized key is an error. The CLI always follows the run's stream to the end, `--no-stream` included, because ora saves a finished run from inside the stream. If the stream drops or goes quiet (no frames for 120s), the CLI reopens it, which resumes the same run without re-running the agent. A stream `429` is waited out (up to 60s) before reopening; any other HTTP error from the stream (a `404`, an auth error, a `5xx`) ends the command at once. Only after five failed reopens does it fall back to polling the run detail, within a 15-minute budget shared with the stream. | Flag | Effect | |---|---| -| `--intent ` | Curated task id (server default when omitted) | -| `--task ` | Free-text task — needs a partner API key; mutually exclusive with `--intent` | +| `--domain ` | Site the agent works on (required) | +| `--intent ` | Curated task id (server default when omitted); cannot be combined with a goal of your own | | `--api-key ` | ora partner API key; also read from `ORA_PARTNER_API_KEY` (then `ORA_SCAN_API_KEY`) | | `--agent ` | Agent from the public roster (default: `cas-haiku`, Claude Code on the Haiku model ora's roster lists) | | `--no-stream` | Hide the live trajectory view and print only the final result | | `--json` | Print the terminal run detail (verdict, step_count, result) as JSON | -deep-journey exit codes: `0` run succeeded (any verdict) · `1` run failed · `2` usage error · `3` API unreachable / rate-limited. +Exit codes on this path: `0` run succeeded (any verdict) · `1` run failed · `2` usage error · `3` API unreachable / rate-limited. + +### One journey + +``` +ax journey "" [--domain d] [--harness h] [--model m] [--json] +``` + +Triggers a real agent run on ora's platform and renders the journey live. The whole graph repaints in place as the agent moves: + +``` + ▶ stripe.com · claude-code + + 9 steps · 3 reasoning · 1 search + + ╭────────────────────────────────────────────────────────────────────────────────╮ + │ ◆ Find the API docs and how to authenticate. Fetch the actual pages to verify. │ + ╰─┬──────────────────────────────────────────────────────────────────────────────╯ + │ (…) The user is asking me to find the API documentation for Stripe… + │ ╭──────────────────────────────────────────────╮ + ╰──┤ 🔍 "Stripe API documentation authentication" │ + ╰─┬────────────────────────────────────────────╯ + │ ╭─────────────────────────────────────────╮ + ├──┤ 🌐 docs.stripe.com/api/authentication ✓ │ + │ ╰─────────────────────────────────────────╯ + │ ╭───────────────────────────╮ + ╰──┤ 🌐 docs.stripe.com/apis ✓ │ + ╰───────────────────────────╯ + + ──────────────────────────────────────────────────── + ✓ completed 9 turns · 30.8s · 119.1k tokens · $0.031 + + Visibility █████████████████████░ 95/100 ✓ intent satisfied +``` + +Edges come from ora's own attribution of each step (a followed link nests under the page that linked it; searches hang off the goal). `(…)` markers are the agent's reasoning, placed above the action it led to. + +| Flag | Effect | +|---|---| +| `` | What the agent should do, quoted | +| `--domain ` | Site the agent targets (e.g. `stripe.com`) | +| `--harness ` | Agent harness: `claude-code` (default), `codex`, `openclaw`, `hermess`, `claude-agent`, `eve`, … | +| `--model ` | Model override (e.g. `claude-haiku-4-5-20251001`, cheap and good for demos) | +| `--agents ` | The agent written the way a set names it. One entry on its own still runs one journey. Cannot be combined with `--harness` or `--model` | +| `--json` | Full result as JSON: run metadata, usage, insight, and the raw trajectory | + +`--yes` and `--wait` apply to a set; one journey ignores them. + +Note: results really do vary between runs. Agents sometimes answer from search snippets without fetching, which scores differently. Run a few journeys before drawing conclusions. + +| Code | Meaning | +|---|---| +| `0` | The run's outcome is `success` | +| `1` | The run ended with another outcome | +| `2` | Any error | + +#### Setup + +One journey needs a Front API key with the scopes `runs:read` and `runs:write`. To run sets of journeys too, create it with the scopes listed under [Get a key](#get-a-key): + +```sh +export ORA_API_KEY=ora_sk_... # your shell, CI secret store, etc. +ax journey "Find the pricing page" --domain example.com +``` + +For local development, copy `.env.example` to `.env`. The CLI reads a `.env` in the working directory on startup (exported variables win over the file). + +### A set of journeys + +``` +npx ax journey "Find how to create an issue from the API" --domain linear.app --agents claude-code,codex --exact https://linear.app/developers --repeat 3 +``` + +A set sends real agents at one site on goals you write and counts how often the platform judges each goal done. With levels on, each run gets a different amount of help, and the gap between two levels points at the layer where agents lose your site: Discovery, or Access and Usability. With levels off, every run gets the goal as you wrote it, so the set compares agents and repeats on one prompt. + +The example above is 1 goal at 3 levels (the goal names no site, so Need only runs; there is no `--docs`, so Docs linked does not), with 2 agents side by side and 3 repeats: 18 runs. The preview says so before anything launches. A bare harness such as `codex` takes the first model your plan offers for it, and the report names each column `harness/model`: + +``` +linear.app set j-4f1c9a27be + +Goal 1: Find how to create an issue from the API + Level claude-code/claude-sonnet-5-5 codex/gpt-5.6-luna + Need only 1 of 3 2 of 3 + Brand named 3 of 3 2 of 3 + Exact target 3 of 3 3 of 3 + + linear.app loses agents at the Discovery layer: 3 of 6 runs satisfied at Need only, 5 of 6 at Brand named. + No --docs URL, so the docs level is skipped. Add --docs to include it. +``` + +That report is shortened: a real one goes on to the Need only note and the list of journeys. + +A set needs `--domain`. It is one site, 1 to 5 goals, up to 4 levels of help, 1 to 4 agents and 1 to 10 repeats, and never more than 200 runs. ax counts runs per cell (a goal, a level and an agent) and shows a rate from 5 gradable runs. Below that it shows the counts alone. + +### Get a key + +`ax journey` runs on your Front account. Sign up at app.ora.ai with a work email (personal addresses are not accepted), then create an API key in your account settings with these scopes: `journeys:write`, `journeys:read`, `runs:read`, `runs:write`, `intents:read`. Set it as `ORA_API_KEY`. The Free plan includes a monthly number of runs; before a set of journeys launches, its preview shows what is left. + +A partner key raises limits on ora.ai's single runs; a Front API key runs journeys on your Front account. + +### Commands + +``` +ax journey "" ["" ...] --domain [--agents ,...] [--levels] [--brand ] [--docs ] [--exact ] [--repeat N] [--yes] [--wait ] [--json] +ax journey view [--wait ] [--json] +ax journey list [--json] +ax journey cancel [--json] +``` + +When the first goal is exactly `view`, `list` or `cancel`, ax runs that command instead, so a goal cannot be just one of those words. + +| Flag | Effect | +|---|---| +| `` | A goal for the agents, quoted. Pass 1 to 5 | +| `--domain ` | The site, required for a set. A domain or an http(s) URL. A leading `www.` is dropped (`https://www.linear.app` is `linear.app`); other subdomains stay | +| `--agents ` | 1 to 4 agents as `harness` or `harness/model`, comma separated. Omit it and the set uses the Claude Code agent your plan offers, or the first agent it offers when it has none; the preview names it | +| `--harness `, `--model ` | One agent for the whole set, the same as `--agents h/m` (`--model` alone runs on `claude-code`). Neither can be combined with `--agents` | +| `--levels` | Run each goal at the levels of help below | +| `--brand ` | The brand named in the brand and docs prompts. Turns levels on. Default: the domain's name with a capital, so `linear.app` gives `Linear`. The preview names the brand it used | +| `--docs ` | The documentation URL. Turns levels on and adds Docs linked | +| `--exact ` | The page the agent starts at. Turns levels on and adds Exact target | +| `--repeat ` | Runs per goal, level and agent, 1 to 10 (default 1) | +| `--yes` | Launch without asking | +| `--wait ` | How many minutes to follow the runs before stopping, a whole number of 1 or more (default 60). `view` takes it too | +| `--json` | Print a JSON document on stdout instead of the report | + +The report goes to stdout. The preview, the prompt, progress and the other lines for a person go to stderr, so `ax journey ... --json` pipes clean. + +### Levels + +Levels are on with `--levels`, and with any of `--docs`, `--exact` or `--brand`. Every level is one prompt template. No model writes a prompt. Before a level adds its own sentence, the goal gets a period at the end unless it already ends in `.`, `?` or `!`. + +| Level | The prompt | Runs | +|---|---|---| +| Need only | The goal as you wrote it | When the goal does not name the site. A goal that contains the domain, or the brand as a whole word, skips this level and the preview says which term it found | +| Brand named | The goal, then `Use {brand}.` | Always | +| Docs linked | The goal, then `Use {brand}. Its documentation is at {docs}.` | When you pass `--docs` | +| Exact target | The goal, then `Start at {exact}.` | When you pass `--exact` | + +A skipped level prints a line saying why and how to include it. The levels climb in that order, from least help to most. + +With levels off, a set runs one level, As written: every run gets the goal as you wrote it. Its key in `--json` is `written`. With no second level to compare it to, the report prints no gap line. + +### The preview + +Before anything runs, a set reads your key's scopes, the agents your plan offers and what is left of your plan's runs this month, then prints: + +- the run total, as goals x levels x agents x repeats; +- the agent it chose, when you did not name one; +- the brand it will use, and any level it skipped, when levels are on; +- how many runs your plan has left this month, and the reset date when the platform gives one (a plan with no monthly limit says so); +- that it adds the site to your Front account's tracked sites if it is not there yet (with a daily ora score scan, which does not count against your plan), and one journey per goal, level and repeat to your journeys list. + +Then it asks `Launch N runs? [y/N]`. `--yes` skips the question. With no terminal to ask on and no `--yes`, it launches nothing and exits `2`. + +The preview refuses, and launches nothing, when: + +- the key is missing a scope (the message names which, and repeats the key steps above); +- an agent name is unknown, unavailable to your account, or listed twice under different names for the same agent; +- the plan offers no agent; +- the set would use more than 200 runs; +- the set needs more runs than your plan has left (it names the largest `--repeat` that fits, when one does). + +### The report + +One block per goal: levels down, agents across. Each cell reads `60% (3 of 5)` from 5 gradable runs, and `3 of 4` below that. Under a cell, its other runs are counted on their own lines: excluded, never started, in flight, not graded yet and not gradable. The two flags below are counted there too. + +- The platform grades every run. ax never does. A partial verdict counts as gradable, not as satisfied. +- A run that never started, failed, errored, timed out, was canceled or was lost is excluded. Excluded runs never count against your site. +- A line like `linear.app loses agents at the Discovery layer: 1 of 5 runs satisfied at Need only, 4 of 5 at Brand named.` appears when two neighboring levels that ran differ clearly. It compares levels, summed over agents, never one agent against another. A gap into Brand named or Docs linked points at the Discovery layer, a gap into Exact target at the Access and Usability layers. With fewer than 5 gradable runs at either level it is marked an early signal. When a level does worse than the one below it, the line says so and names no layer. +- At the need level the prompt does not name your site, so an agent can reach the goal without using it. + +Two flags sit beside the verdicts as counts. They never change a verdict. + +- **blocked**: your site answered the agent's first request with a 401, 403 or 429, or answered more than half of its requests that way. ax uses the rule ora.ai uses for its own journey runs, copied from it, so the flag means the same thing in both. A 404 never counts. +- **never visited**: the agent never opened a page on your site. ora.ai also flags a run that answered from the model's memory. A run in a set carries no signal for that, so `never visited` is raised in its place. + +If a run's steps cannot be read, the run is left unflagged and ax says so on stderr. + +The report ends with a Next line: `ax journey view ` while runs are still going or waiting for a grade, otherwise `ax audit `. + +When launching stopped at your plan's run limit, the wait ran out, reads kept failing or a run has no grade yet, the report prints what it has and says in one line why it is not complete. `--json` carries the same fact. + +### JSON + +A set, `ax journey view` and `ax journey list` print a document ax builds, because no single platform payload exists for a set. Statuses, verdicts and reasons are the platform's words. A rate is a fraction from 0 to 1, or `null` below 5 gradable runs. The Next line goes to stderr. + +A set and `view` print: -This differs from `ax journey` (below), which drives the authenticated platform with free-text goals and renders the full trajectory graph; `deep-journey` is the zero-setup public path. +| Field | What it holds | +|---|---| +| `setId`, `site`, `createdAt` | The set's id, its site and when it was recorded | +| `complete` | `true` only when launching ran to the end, following settled, and no run is going or waiting for a grade | +| `levels` | Per goal: its index, its text and the levels that ran (`written` alone when levels are off) | +| `skippedLevels` | The levels left out, each with the note the preview printed | +| `agents` | The report's columns, as `harness/model` | +| `cells` | One entry per goal, level and agent: the counts, the rate, the excluded runs by status, the flag counts | +| `gaps` | The gap lines as data | +| `shortLevels` | Levels that launched fewer runs than planned | +| `journeys` | Each journey's id, goal index, level and repeat | +| `notLaunched`, `stoppedReason` | Runs that never started, and why launching stopped (`quota`, `platform_down`, `api_error`, `interrupted`), or `null` | +| `ended` | How following ended: `settled`, `settle_limit`, `wait_limit`, `detached` or `read_failed` | + +`list --json` prints `{ "records": [...], "skipped": [...] }`. `cancel --json` prints the platform's answer unchanged. + +### The local record + +Each set writes one file, `~/.ax/journeys/.json`, before it creates the first journey, and adds to it as each journey is created. An id looks like `j-0123456789`. The file holds the spec, the agents, the planned levels and each journey's id and run ids. It never holds your key or a token. + +- `ax journey view ` picks a set up: it follows the runs, reads them and prints the same report, including from another terminal. Closing the terminal does not stop the runs. +- `ax journey list` shows the sets recorded on this machine, newest first. It reads that folder and asks no server. +- `ax journey cancel ` asks the platform to stop every run of the set, then prints how many it canceled, how many had already finished and how many it skipped. Run `ax journey view ` afterward for what finished. + +Known gap: if ax crashes after the platform answers a create and before the record is written, that one journey's id is lost. Its runs still happen and still show in your account's journeys list. + +### Exit codes + +For a set of journeys and for `view`, `list` and `cancel` (one journey's codes are under [One journey](#one-journey)): + +| Code | Meaning | +|---|---| +| `0` | A report printed, complete or not. A launch stopped at your plan's run limit, or a wait that ran out, still exits `0` with the report and its reason. `list`, `cancel` and a declined prompt exit `0` too | +| `2` | Usage: a bad flag or spec (a set with no `--domain` included), a missing key, a key made for another product, an unknown set id, or a refusal before launch | +| `3` | The platform failed and there is no report (or `cancel` could not reach it): the key exchange failed, launching stopped on an error or because the platform stopped answering, or reads kept failing before any run was read. The Next lines name `ax journey view ` and, once a run started, `ax journey cancel ` | +| `130` | Ctrl-C, after printing the `ax journey view ` line (and `ax journey cancel ` when it hit during launching) | ## skill @@ -302,7 +541,7 @@ console.log(result.score, result.grade, result.topFixes); `result` is the raw versioned audit payload (`AuditScanResult` / `AuditScoreResult`, generated from ora's OpenAPI spec). `fetchSkill` / `fetchSkillIndex` expose the digest-verified skill registry, and `fetchAgentFiles` the agent files generator. -The public journey client is exported too — the same no-key path as `ax deep-journey`: +The public journey client is exported too — the same no-key path as `ax journey --domain `: ```ts import { deepJourney, fetchJourneyAgents } from "ax"; @@ -321,68 +560,9 @@ console.log(detail.verdict, detail.step_count, detail.result?.insight?.summary); ## For ora partners -Everything above runs without an account. The commands in this section are for teams working with ora directly, and there is no self-serve signup: keys are issued by ora (contact [hello@ora.ai](mailto:hello@ora.ai)). Two kinds of key exist. `journey` (and `audit --tunnel ora`) needs an ora platform key (`ora_sk_…`). A separate partner or scan key, passed with `--api-key`, lifts the anonymous rate limits on `audit` and `deep-journey`, and the daily limit on `webmcp-audit`. +Everything above runs without an account, except [`journey`](#journey) on a goal of your own and `audit --tunnel ora`. Two kinds of key exist. A Front API key (`ora_sk_…`) is self-serve: [Get a key](#get-a-key) shows how to get one, and `journey` and `audit --tunnel ora` use it. A separate partner or scan key, passed with `--api-key`, lifts the anonymous rate limits on `audit` and the free `journey`, and the daily limit on `webmcp-audit`. Partner and scan keys are still issued by ora (contact [hello@ora.ai](mailto:hello@ora.ai)). -### journey - -``` -ax journey "" [--domain d] [--harness h] [--model m] [--json] -``` - -``` -ax journey "Find the API docs and how to authenticate" --domain stripe.com -``` - -Triggers a real agent run on ora's platform and renders the journey live — the whole graph repaints in place as the agent moves: - -``` - ▶ stripe.com · claude-code - - 9 steps · 3 reasoning · 1 search - - ╭────────────────────────────────────────────────────────────────────────────────╮ - │ ◆ Find the API docs and how to authenticate. Fetch the actual pages to verify. │ - ╰─┬──────────────────────────────────────────────────────────────────────────────╯ - │ (…) The user is asking me to find the API documentation for Stripe… - │ ╭──────────────────────────────────────────────╮ - ╰──┤ 🔍 "Stripe API documentation authentication" │ - ╰─┬────────────────────────────────────────────╯ - │ ╭─────────────────────────────────────────╮ - ├──┤ 🌐 docs.stripe.com/api/authentication ✓ │ - │ ╰─────────────────────────────────────────╯ - │ ╭───────────────────────────╮ - ╰──┤ 🌐 docs.stripe.com/apis ✓ │ - ╰───────────────────────────╯ - - ──────────────────────────────────────────────────── - ✓ completed 9 turns · 30.8s · 119.1k tokens · $0.031 - - Visibility █████████████████████░ 95/100 ✓ intent satisfied -``` - -Edges come from ora's own per-step attribution (link-follows nest under the page that linked them; searches hang off the intent). `(…)` markers are the agent's reasoning, placed above the action it led to. - -| Flag | Effect | -|---|---| -| `--domain ` | Site the agent targets (e.g. `stripe.com`) | -| `--harness ` | Agent harness: `claude-code` (default), `codex`, `openclaw`, `hermess`, `claude-agent`, `eve`, … | -| `--model ` | Model override (e.g. `claude-haiku-4-5-20251001` — cheap and good for demos) | -| `--json` | Full result as JSON: run metadata, usage, insight, and the raw trajectory | - -Note: run-to-run variance is real — agents sometimes answer from search snippets without fetching, which scores differently. Run a few journeys before drawing conclusions. - -Journey exit codes: `0` run outcome `success` · `1` non-success outcome · `2` any error. - -#### Setup - -`journey` needs an ora platform API key (scopes `runs:read` + `runs:write`): - -```sh -export ORA_API_KEY=ora_sk_... # your shell, CI secret store, etc. -ax journey "Find the pricing page" --domain example.com -``` - -For local development, copy `.env.example` to `.env` — the CLI reads a `.env` in the working directory on startup (exported variables win over the file). +A partner key raises limits on ora.ai's single runs; a Front API key runs journeys on your Front account. ## Environment variables @@ -392,16 +572,18 @@ A `.env` in the working directory is read on startup — copy `.env.example` and | Var | Used by | Default | Purpose | |---|---|---|---| -| `ORA_API_URL` | audit, webmcp-audit, deep-journey, skill, generate-files | `https://ora.ai` | Public API base (no auth); also the ingest base for `webmcp-audit` | +| `ORA_API_URL` | audit, webmcp-audit, journey with no goal, skill, generate-files | `https://ora.ai` | Public API base (no auth); also the ingest base for `webmcp-audit` | | `ORA_PLATFORM_URL` | journey, audit `--tunnel ora` | `https://api.agentfront.sh` | Authenticated platform API base | | `ORA_TUNNEL` | audit | — (optional) | `ora` opens ora's own tunnel for a local target (same as `--tunnel ora`) | | `ORA_TUNNEL_CMD` | audit | — (optional) | Tunnel command for a local target (same as `--tunnel-cmd`); wins over `ORA_TUNNEL` | -| `ORA_API_KEY` | journey, audit `--tunnel ora` | — (required for both) | ora platform key (`ora_sk_…`); exchanged for a bearer token (the tunnel needs the `tunnels:write` + `tunnels:connect` scopes) | -| `ORA_SCAN_API_KEY` | audit, deep-journey, webmcp-audit | — (optional) | ora-issued scan API key; lifts every scan rate limit (issued manually by ora). deep-journey and webmcp-audit accept it as a partner-key fallback | -| `ORA_PARTNER_API_KEY` | deep-journey, webmcp-audit | — (optional) | ora-issued partner API key; unlocks `--task` and the 1000/24h keyed allowance on deep-journey, and lifts webmcp-audit's daily limit | +| `ORA_API_KEY` | journey, audit `--tunnel ora` | none (required for both) | Front API key (`ora_sk_…`, made in your Front account at app.ora.ai); exchanged for a bearer token (the tunnel needs the `tunnels:write` + `tunnels:connect` scopes, a set of journeys the five scopes listed under [Get a key](#get-a-key)) | +| `ORA_SCAN_API_KEY` | audit, journey with no goal, webmcp-audit | — (optional) | ora-issued scan API key; lifts every scan rate limit (issued manually by ora). The free journey and webmcp-audit accept it as a partner-key fallback | +| `ORA_PARTNER_API_KEY` | journey with no goal, webmcp-audit | — (optional) | ora-issued partner API key; unlocks a goal of your own and the 1000/24h keyed allowance on the free journey, and lifts webmcp-audit's daily limit | | `CHROME_PATH` | webmcp-audit | — (auto-detected) | The Chrome to launch, when yours is not in a standard location | | `AX_WEBMCP_NO_SANDBOX` | webmcp-audit | — (unset) | `1` launches Chrome without its sandbox. Containers only: the audited page is untrusted and the sandbox contains it | +`ORA_API_URL` and `ORA_PLATFORM_URL` decide where your keys go, so ax checks them before it sends one. Their defaults always work. Any other host gets a key only over https (plain http only to this machine, such as `localhost`), and only when you set the variable in your shell: a `.env` belongs to whatever directory you run in, so a host named there is refused before anything is sent (`ax journey` exits 2 with `KEY_DESTINATION_REFUSED`). The exceptions are `localhost`, `127.0.0.1` and `[::1]`. Requests that carry no key are not checked. + ## Contract versioning The audit types are generated from ora's served OpenAPI spec (`pnpm contract:gen`); the build is pinned to a contract version and CI fails when production drifts from the checked-in types. At runtime, the CLI prints one stderr warning when ora reports a newer contract than the build — upgrade with `npm i -g ax@latest`. diff --git a/biome.json b/biome.json index 782b07f..4875692 100644 --- a/biome.json +++ b/biome.json @@ -37,7 +37,8 @@ "!**/.context", "!src/contract/types.gen.ts", "!src/webmcp/vendor", - "!src/tunnel/vendor" + "!src/tunnel/vendor", + "!src/journey-set/vendor" ] } } diff --git a/scripts/smoke.mjs b/scripts/smoke.mjs index 2f8dac4..0d37d7c 100755 --- a/scripts/smoke.mjs +++ b/scripts/smoke.mjs @@ -7,7 +7,7 @@ // Also checks the things that break `npx ax` but never break `pnpm test`: // a missing shebang or a non-executable bin. -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { readFileSync, statSync } from "node:fs"; const BIN = "dist/main.cjs"; @@ -80,6 +80,41 @@ check("webmcp-audit --help exits 0", () => ax(["webmcp-audit", "--help"])); check("deep-journey --help exits 0", () => ax(["deep-journey", "--help"])); check("journey --help exits 0", () => ax(["journey", "--help"])); +// Each call below fails on its flags before any network call, so these run offline. +function axFails(args) { + const run = spawnSync(process.execPath, [BIN, ...args], { encoding: "utf8" }); + return { status: run.status, stderr: run.stderr }; +} + +check("deep-journey still runs, notes its new name, and keeps its exit codes", () => { + const args = [ + "deep-journey", + "example.com", + "--task", + "Find the pricing page", + "--intent", + "pricing", + ]; + const { status, stderr } = axFails(args); + if (status !== 2) throw new Error(`exit ${status}, want 2`); + if ( + !stderr.includes( + "ax deep-journey is now ax journey 'Find the pricing page' --domain example.com", + ) + ) { + throw new Error("no deprecation note naming the new call"); + } + if (!stderr.includes("ax deep-journey: --task and --intent")) + throw new Error("the error changed"); +}); + +check("journey with no goal takes the free path, which needs --domain", () => { + const { status, stderr } = axFails(["journey", "--intent", "pricing"]); + if (status !== 2) throw new Error(`exit ${status}, want 2`); + if (!stderr.includes("--domain names the site")) throw new Error("not the free path's error"); + if (stderr.includes("ORA_API_KEY")) throw new Error("the free path asked for a Front key"); +}); + if (failures.length > 0) { console.error(`\nsmoke failed (${failures.length}):`); for (const failure of failures) console.error(` - ${failure}`); diff --git a/src/api/__fixtures__/deep-journey-detail-running.json b/src/api/__fixtures__/deep-journey-detail-running.json new file mode 100644 index 0000000..f28b69e --- /dev/null +++ b/src/api/__fixtures__/deep-journey-detail-running.json @@ -0,0 +1,18 @@ +{ + "id": "795ec07c-342a-4603-9649-66be4bb7936e", + "status": "running", + "intent_id": "pricing", + "domain": "linear.app", + "agent": { + "harness": "claude-agent-sdk", + "model": "claude-haiku-5-5" + }, + "started_at": "2026-11-13T12:13:32.627Z", + "stream_url": "/api/journey/runs/795ec07c-342a-4603-9649-66be4bb7936e/stream", + "contractVersion": "1.42.0", + "next_action": { + "method": "GET", + "url": "/api/journey/runs/795ec07c-342a-4603-9649-66be4bb7936e/stream", + "reason": "Hold the stream open to the end; only a followed stream saves the result." + } +} diff --git a/src/api/__fixtures__/deep-journey-detail.json b/src/api/__fixtures__/deep-journey-detail.json index 1d110c0..abc36f8 100644 --- a/src/api/__fixtures__/deep-journey-detail.json +++ b/src/api/__fixtures__/deep-journey-detail.json @@ -1,46 +1,58 @@ { - "id": "98cacf5c-4c08-4c21-97df-4f8e3e1f6f19", + "id": "795ec07c-342a-4603-9649-66be4bb7936e", "status": "succeeded", - "intent_id": "signup", - "domain": "captest-orank.com", + "intent_id": "pricing", + "domain": "linear.app", "agent": { "harness": "claude-agent-sdk", - "model": "claude-haiku-4-5" + "model": "claude-haiku-5-5" + }, + "started_at": "2026-11-13T12:13:32.627Z", + "finished_at": "2026-11-13T12:13:57.406Z", + "stream_url": "/api/journey/runs/795ec07c-342a-4603-9649-66be4bb7936e/stream", + "contractVersion": "1.42.0", + "verdict": "satisfied", + "step_count": 12, + "flags": { + "blocked": { + "flagged": false + }, + "answered_from_memory": { + "flagged": false + } }, - "started_at": "2026-08-14T01:51:54.882Z", - "finished_at": "2026-08-14T01:53:40.360Z", - "stream_url": "/api/journey/runs/98cacf5c-4c08-4c21-97df-4f8e3e1f6f19/stream", - "contractVersion": "1.9.0", - "verdict": "unsatisfied", - "step_count": 19, "result": { - "run_id": "98cacf5c-4c08-4c21-97df-4f8e3e1f6f19", - "intent_id": "signup", - "domain": "captest-orank.com", + "run_id": "795ec07c-342a-4603-9649-66be4bb7936e", + "intent_id": "pricing", + "domain": "linear.app", "harness": "claude-agent-sdk", - "model": "claude-haiku-4-5", - "started_at": "2026-08-14T01:51:54.882Z", - "finished_at": "2026-08-14T01:53:40.360Z", - "outcome": "failure", - "verdict": "unsatisfied", - "agent_response": "I'll help you access captest-orank.com and look for diagnostic tools or health-check endpoints. Let me start by fetching the main domain.", - "num_turns": 10, - "duration_ms": 38200, + "model": "claude-haiku-5-5", + "started_at": "2026-11-13T12:13:32.627Z", + "finished_at": "2026-11-13T12:13:57.406Z", + "outcome": "success", + "verdict": "satisfied", + "agent_response": "Excellent! Now I have comprehensive information. Let me compile a detailed summary of Linear's programmatic capabilities and integration options.\n\n## Summary: Linear's Programmatic Capabilities and Integration Options\n\nBased on my research of linear.app, here's a comprehensive overview of Linear's programmatic capabilities and integration options:\n\n### **1. Core Integration Ecosystem**\n- **9,000+ pre-built app integrations** covering major business applications (Salesforce, HubSpot, Slack, Google Workspace, Microsoft 365, Zendesk, Jira, NetSuite, and thousands more)\n- **30,000+ pre-built actions and triggers** across integrated apps\n- **66,000+ total triggers and actions** available for use\n\n### **2. Programmatic Access Options**\n\n#### **A. Linear SDK (Open Beta - Free)**\n- **For:** Coding agents and developers integrating Linear into products\n- **Language:** TypeScript/JavaScript via npm (`@linear/linear-sdk`)\n- **Key Features:**\n - Full programmatic access to 9,000+ apps without OAuth setup\n - Raw API access to ~3,000 apps for custom API calls\n - Type-safe TypeScript interfaces with generated types\n - Handles authentication, token refresh, and retries automatically\n - Works with coding agents (Cursor, Claude Code, VS Code Copilot)\n - Includes `runAction()` for invoking pre-built actions and `.fetch()` for custom API calls\n - App and action governance policies apply automatically to pre-built actions\n\n#### **B. Linear MCP (Model Context Protocol) - Available on All Plans**\n- **For:** AI chat tools and agents (Claude, ChatGPT, Cursor, etc.)\n- **Key Features:**\n - 9,000+ apps with 30,000+ pre-built actions accessible via MCP standard\n - Requires no terminal or code setup (guided UI-based installation)\n - Two tasks per tool call included in standard quota\n - Compatible with any MCP client\n - Full governance controls (app restrictions, action restrictions, audit logs)\n - 195,000+ MCP servers already created\n\n#### **C. Linear CLI (Command-line Interface)**\n", + "num_turns": 7, + "duration_ms": 40689, + "cost_usd": 0.34632524000000003, + "input_tokens": 323468, + "output_tokens": 2345, + "cache_read_tokens": 197998, "trajectory": { "steps": [ { "id": 0, "type": "text", - "text": "I'll help you access stripe.com and look for diagnostic tools or health-check endpoints. Let me start by fetching the main domain." + "text": "I'll research Zapier's programmatic capabilities and integration options by exploring their website and documentation. Let me start by gathering information." }, { "id": 1, "turn": 1, "type": "tool_call", "action": "fetch", - "tool": "WebFetch", - "url": "captest-orank.com/", - "url_host": "captest-orank.com", + "tool": "web_fetch", + "url": "linear.app/", + "url_host": "linear.app", "url_path": "/", "source": "direct", "anchor_relation": "exact", @@ -49,409 +61,297 @@ "artifact_kind": "homepage" }, "status": 200, - "duration_ms": 5234, + "duration_ms": 10, "completed": true }, { "id": 2, - "turn": 2, + "turn": 1, "type": "tool_call", - "parent_id": 1, "action": "fetch", - "tool": "WebFetch", - "url": "status.captest-orank.com/", - "url_host": "status.captest-orank.com", - "url_path": "/", - "source": "follow", - "anchor_relation": "subdomain", + "tool": "web_fetch", + "url": "linear.app/help", + "url_host": "linear.app", + "url_path": "/help", + "source": "direct", + "anchor_relation": "exact", "attribution": { - "kind": "previous_artifact", - "artifact_kind": "homepage", - "referrer": { - "turn": 1, - "artifact_kind": "homepage", - "url_path": "/" - } + "kind": "prior_knowledge" }, - "status": 200, - "duration_ms": 2978, + "duration_ms": 33, "completed": true }, { "id": 3, - "turn": 2, + "turn": 1, "type": "tool_call", - "action": "fetch", - "tool": "WebFetch", - "url": "captest-orank.com/docs/health-check", - "url_host": "captest-orank.com", - "url_path": "/docs/health-check", + "action": "search", + "tool": "web_search", + "search_query": "Linear API programmatic integration capabilities", "source": "direct", - "anchor_relation": "exact", "attribution": { - "kind": "prior_knowledge", - "artifact_kind": "docs_page", - "signal_id": "developer-portal" + "kind": "web_search" }, - "status": 301, - "duration_ms": 72, + "duration_ms": 20, "completed": true }, { "id": 4, - "turn": 3, - "type": "tool_call", - "action": "fetch", - "tool": "WebFetch", - "url": "captest-orank.com/.well-known/health", - "url_host": "captest-orank.com", - "url_path": "/.well-known/health", - "source": "direct", - "anchor_relation": "exact", - "attribution": { - "kind": "prior_knowledge", - "artifact_kind": "well_known" - }, - "status": 404, - "duration_ms": 344, - "completed": true + "type": "text", + "text": "Let me get more detailed information about Zapier's specific programmatic capabilities and developer tools." }, { "id": 5, - "turn": 4, + "turn": 2, "type": "tool_call", - "parent_id": 3, + "parent_id": 1, "action": "fetch", - "tool": "WebFetch", - "url": "docs.captest-orank.com/health-check", - "url_host": "docs.captest-orank.com", - "url_path": "/health-check", - "source": "follow", - "anchor_relation": "subdomain", + "tool": "web_fetch", + "url": "linear.app/developer-platform", + "url_host": "linear.app", + "url_path": "/developer-platform", + "source": "direct", + "anchor_relation": "exact", "attribution": { "kind": "previous_artifact", - "artifact_kind": "docs_page", - "signal_id": "developer-portal", "referrer": { - "turn": 2, - "artifact_kind": "docs_page", - "url_path": "/docs/health-check" + "turn": 1, + "artifact_kind": "homepage", + "url_path": "/" } }, - "status": 404, - "duration_ms": 495, + "status": 200, + "duration_ms": 23, "completed": true }, { "id": 6, - "turn": 5, + "turn": 2, "type": "tool_call", - "action": "search", - "tool": "WebSearch", - "search_query": "captest-orank.com health check endpoint MCP server diagnostic tools", + "parent_id": 1, + "action": "fetch", + "tool": "web_fetch", + "url": "linear.app/mcp", + "url_host": "linear.app", + "url_path": "/mcp", "source": "direct", + "anchor_relation": "exact", "attribution": { - "kind": "web_search" + "kind": "previous_artifact", + "artifact_kind": "mcp_endpoint_path", + "signal_id": "mcp-server", + "referrer": { + "turn": 1, + "artifact_kind": "homepage", + "url_path": "/" + } }, - "duration_ms": 531, + "status": 200, + "duration_ms": 12, "completed": true }, { "id": 7, - "turn": 5, + "turn": 2, "type": "tool_call", - "parent_id": 3, + "parent_id": 1, "action": "fetch", - "tool": "WebFetch", - "url": "captest-orank.com/docs", - "url_host": "captest-orank.com", - "url_path": "/docs", - "source": "follow", + "tool": "web_fetch", + "url": "linear.app/sdk", + "url_host": "linear.app", + "url_path": "/sdk", + "source": "direct", "anchor_relation": "exact", "attribution": { "kind": "previous_artifact", - "artifact_kind": "docs_page", - "signal_id": "developer-portal", "referrer": { - "turn": 2, - "artifact_kind": "docs_page", - "url_path": "/docs/health-check" + "turn": 1, + "artifact_kind": "homepage", + "url_path": "/" } }, - "status": 301, - "duration_ms": 74, + "status": 200, + "duration_ms": 34, "completed": true }, { "id": 8, - "turn": 6, - "type": "tool_call", - "parent_id": 6, - "action": "fetch", - "tool": "WebFetch", - "url": "docs.captest-orank.com/mcp", - "url_host": "docs.captest-orank.com", - "url_path": "/mcp", - "source": "follow", - "anchor_relation": "subdomain", - "attribution": { - "kind": "web_search", - "artifact_kind": "docs_page", - "signal_id": "developer-portal" - }, - "status": 200, - "duration_ms": 4636, - "completed": true + "type": "text", + "text": "Now let me fetch the documentation pages to get more detailed technical information." }, { "id": 9, - "turn": 6, + "turn": 3, "type": "tool_call", - "parent_id": 6, + "parent_id": 5, "action": "fetch", - "tool": "WebFetch", - "url": "docs.captest-orank.com/health-alerts", - "url_host": "docs.captest-orank.com", - "url_path": "/health-alerts", + "tool": "web_fetch", + "url": "docs.linear.app/platform/home", + "url_host": "docs.linear.app", + "url_path": "/platform/home", "source": "follow", "anchor_relation": "subdomain", "attribution": { - "kind": "web_search", + "kind": "previous_artifact", "artifact_kind": "docs_page", - "signal_id": "developer-portal" + "signal_id": "developer-portal", + "referrer": { + "turn": 2, + "url_path": "/developer-platform" + } }, "status": 200, - "duration_ms": 5716, + "duration_ms": 23, "completed": true }, { "id": 10, - "turn": 7, + "turn": 3, "type": "tool_call", - "parent_id": 8, + "parent_id": 7, "action": "fetch", - "tool": "WebFetch", - "url": "mcp.captest-orank.com/", - "url_host": "mcp.captest-orank.com", - "url_path": "/", + "tool": "web_fetch", + "url": "docs.linear.app/sdk", + "url_host": "docs.linear.app", + "url_path": "/sdk", "source": "follow", "anchor_relation": "subdomain", "attribution": { "kind": "previous_artifact", - "artifact_kind": "mcp_server", - "signal_id": "mcp-server", + "artifact_kind": "docs_page", + "signal_id": "developer-portal", "referrer": { - "turn": 6, - "artifact_kind": "mcp_endpoint_path", - "url_path": "/mcp" + "turn": 2, + "url_path": "/sdk" } }, - "status": 401, - "duration_ms": 697, + "status": 200, + "duration_ms": 12, "completed": true }, { "id": 11, - "turn": 7, - "type": "tool_call", - "action": "search", - "tool": "WebSearch", - "search_query": "Captest-orank MCP server health check endpoint localhost:3000", - "source": "direct", - "attribution": { - "kind": "web_search" - }, - "duration_ms": 516, - "completed": true + "type": "text", + "text": "Perfect! Now let me gather one more piece of information about the Workflow API and embeddings capabilities." }, { "id": 12, - "turn": 8, + "turn": 4, "type": "tool_call", - "parent_id": 11, "action": "fetch", - "tool": "WebFetch", - "url": "mcpcat.io/guides/building-health-check-endpoint-mcp-server", - "url_host": "mcpcat.io", - "url_path": "/guides/building-health-check-endpoint-mcp-server", - "source": "follow", - "anchor_relation": "external", + "tool": "web_fetch", + "url": "docs.linear.app/powered-by-zapier/workflow", + "url_host": "docs.linear.app", + "url_path": "/powered-by-zapier/workflow", + "source": "direct", + "anchor_relation": "subdomain", "attribution": { - "kind": "web_search" + "kind": "prior_knowledge", + "artifact_kind": "docs_page", + "signal_id": "developer-portal" }, - "status": 200, - "duration_ms": 6122, + "duration_ms": 11, "completed": true }, { "id": 13, - "turn": 8, - "type": "tool_call", - "parent_id": 9, - "action": "fetch", - "tool": "WebFetch", - "url": "docs.captest-orank.com/workbench/health", - "url_host": "docs.captest-orank.com", - "url_path": "/workbench/health", - "source": "follow", - "anchor_relation": "subdomain", - "attribution": { - "kind": "previous_artifact", - "artifact_kind": "docs_page", - "signal_id": "developer-portal", - "referrer": { - "turn": 6, - "artifact_kind": "docs_page", - "url_path": "/health-alerts" - } - }, - "status": 200, - "duration_ms": 4148, - "completed": true + "type": "text", + "text": "Let me search for more information about their API and integration building capabilities." }, { "id": 14, - "turn": 9, + "turn": 5, "type": "tool_call", "action": "search", - "tool": "WebSearch", - "search_query": "\"captest-orank mcp server\" \"health check\" OR \"health endpoint\" OR \"diagnostic\"", + "tool": "web_search", + "search_query": "Linear Workflow API embed tools documentation", "source": "direct", "attribution": { "kind": "web_search" }, - "duration_ms": 927, + "duration_ms": 26, "completed": true }, { "id": 15, - "turn": 9, + "turn": 5, "type": "tool_call", - "parent_id": 7, - "action": "fetch", - "tool": "WebFetch", - "url": "docs.captest-orank.com/", - "url_host": "docs.captest-orank.com", - "url_path": "/", - "source": "follow", - "anchor_relation": "subdomain", + "action": "search", + "tool": "web_search", + "search_query": "Linear REST API programmatic access", + "source": "direct", "attribution": { - "kind": "previous_artifact", - "artifact_kind": "homepage", - "referrer": { - "turn": 5, - "artifact_kind": "docs_page", - "url_path": "/docs" - } + "kind": "web_search" }, - "status": 200, - "duration_ms": 3297, + "duration_ms": 14, "completed": true }, { "id": 16, - "turn": 10, - "type": "tool_call", - "parent_id": 6, - "action": "fetch", - "tool": "WebFetch", - "url": "github.com/hideokamoto/stripe-testing-mcp-tools", - "url_host": "github.com", - "url_path": "/hideokamoto/stripe-testing-mcp-tools", - "source": "follow", - "anchor_relation": "external", - "attribution": { - "kind": "web_search" - }, - "status": 200, - "duration_ms": 4261, - "completed": true + "type": "text", + "text": "Let me get more details on the integration building capabilities and authentication options." }, { "id": 17, - "turn": 10, - "type": "tool_call", - "action": "search", - "tool": "WebSearch", - "search_query": "captest-orank api health check endpoint status", - "source": "direct", - "attribution": { - "kind": "web_search" - }, - "duration_ms": 638, - "completed": true - }, - { - "id": 18, - "turn": 11, + "turn": 6, "type": "tool_call", - "parent_id": 17, + "parent_id": 9, "action": "fetch", - "tool": "WebFetch", - "url": "api.captest-orank.com/healthcheck", - "url_host": "api.captest-orank.com", - "url_path": "/healthcheck", + "tool": "web_fetch", + "url": "docs.linear.app/integrations/quickstart/build-integration", + "url_host": "docs.linear.app", + "url_path": "/integrations/quickstart/build-integration", "source": "follow", "anchor_relation": "subdomain", "attribution": { - "kind": "web_search", + "kind": "previous_artifact", "artifact_kind": "docs_page", - "signal_id": "developer-portal" + "signal_id": "developer-portal", + "referrer": { + "turn": 3, + "artifact_kind": "docs_page", + "url_path": "/platform/home" + } }, "status": 200, - "duration_ms": 2911, + "duration_ms": 11, "completed": true }, { - "id": 19, - "turn": 11, - "type": "tool_call", - "parent_id": 6, - "action": "fetch", - "tool": "WebFetch", - "url": "status.captest-orank.com/reachability", - "url_host": "status.captest-orank.com", - "url_path": "/reachability", - "source": "follow", - "anchor_relation": "subdomain", - "attribution": { - "kind": "web_search" - }, - "status": 200, - "duration_ms": 2486, - "completed": true + "id": 18, + "type": "text", + "text": "Excellent! Now I have comprehensive information. Let me compile a detailed summary of Zapier's programmatic capabilities and integration options.\n\n## Summary: Zapier's Programmatic Capabilities and Integration Options\n\nBased on my research of zapier.com, here's a comprehensive overview of Zapier's programmatic capabilities and integration options:\n\n### **1. Core Integration Ecosystem**\n- **9,000+ pre-built app integrations** covering major business applications (Salesforce, HubSpot, Slack, Google Workspace, Microsoft 365, Zendesk, Jira, NetSuite, and thousands more)\n- **30,000+ pre-built actions and triggers** across integrated apps\n- **66,000+ total triggers and actions** available for use\n\n### **2. Programmatic Access Options**\n\n#### **A. Zapier SDK (Open Beta - Free)**\n- **For:** Coding agents and developers integrating Zapier into products\n- **Language:** TypeScript/JavaScript via npm (`@zapier/zapier-sdk`)\n- **Key Features:**\n - Full programmatic access to 9,000+ apps without OAuth setup\n - Raw API access to ~3,000 apps for custom API calls\n - Type-safe TypeScript interfaces with generated types\n - Handles authentication, token refresh, and retries automatically\n - Works with coding agents (Cursor, Claude Code, VS Code Copilot)\n - Includes `runAction()` for invoking pre-built actions and `.fetch()` for custom API calls\n - App and action governance policies apply automatically to pre-built actions\n\n#### **B. Zapier MCP (Model Context Protocol) - Available on All Plans**\n- **For:** AI chat tools and agents (Claude, ChatGPT, Cursor, etc.)\n- **Key Features:**\n - 9,000+ apps with 30,000+ pre-built actions accessible via MCP standard\n - Requires no terminal or code setup (guided UI-based installation)\n - Two tasks per tool call included in standard quota\n - Compatible with any MCP client\n - Full governance controls (app restrictions, action restrictions, audit logs)\n - 195,000+ MCP servers already created\n\n#### **C. Zapier CLI (Command-line Interface)**\n- **For:** Developers building custom integrations and managing Zapier programmatically\n- **Package:** `@zapier/zapier-sdk-cli` (npm install)\n- **Capabilities:**\n - Explore available actions across connected apps\n - Run one-off actions from terminal\n - List and manage connections\n - Version control integration\n\n### **3. Integration Building Platform**\n\n#### **Two Development Approaches:**\n1. **Visual Builder (No-Code)**\n - Browser-based UI for creating integrations\n - Configurable API endpoints without writing code\n - Custom URL parameters, HTTP headers, request body options\n - Fast deployment suitable for teams without developer expertise\n\n2. **Platform CLI (Code-First)**\n - Local development environment integration\n - Full control with custom JavaScript/Node.js code\n - Version control and CI/CD tool compatibility\n - Support for complex authentication schemes, dynamic dropdowns, webhooks\n\n#### **Integration Components:**\n- **Authentication schemes** (API key, OAuth 2.0, Basic Auth, custom)\n- **Triggers** (webhook subscriptions or polling endpoints for workflow initiation)\n- **Actions** (create, search, update record operations)\n- **Searches** (query capabilities for records)\n- **Tests and validation** (automated checks before publishing)\n\n### **4. Workflow API & Embedding Options**\n\n#### **Workflow API**\n- Programmatic access to create and manage automated workflows\n- Enables embedding Zapier automation directly into third-party products\n- Full-featured automation within custom UIs\n\n#### **Embed Tools (Powered by Zapier)**\n- **Workflow Element:** Pre-built UI component for embedding in products with minimal engineering\n- **App Directory:** Expose Zapier's 9,000+ apps within your product\n- **Partner API:** Complete style control for custom branding and user experience\n- Fully managed authentication (no OAuth flow development needed)\n- Supports 3 integration approaches: quick embed (~2 weeks), full API customization, or template embedding\n\n### **5. Developer Platform & Partner Ecosystem**\n\n- **Free integration building** (no fees to create integrations)\n- **Integration Partner Program** with GTM collaboration and support\n- **Developer dashboard** for managing apps and API keys\n- **Community support** with developer forums\n- **Deployment options:**\n - Private integrations (internal use)\n - Public marketplace (1M+ Zapier users)\n - Versioning and team collaboration\n\n### **6. Advanced Capabilities**\n\n- **Webhooks support** for real-time event triggering\n- **Dynamic dropdowns** for contextual data selection\n- **Custom code** support in CLI for complex logic\n- **Rate limit handling** automated by Zapier\n- **Credential management:** 13+ years of production infrastructure for secure auth\n- **Governance & Compliance:**\n - SOC 2 Type II certified\n - GDPR + CCPA compliant\n - Workspace-level controls\n - Audit logs for all actions\n - Bring-your-own-model (BYOM) support for enterprise\n\n### **7. Authentication Handling**\n- No hardcoded API keys required\n- OAuth flows managed by Zapier\n- Unified credential infrastructure across all integrations\n- Automatic token refresh and retry logic\n- Workspace-level managed connections\n\n### **8. Documentation & Resources**\n- **Comprehensive docs** at `docs.zapier.com` and `platform.zapier.com`\n- **SDK/MCP guides and technical documentation**\n- **CLI reference** and API schemas\n- **GitHub integration** with public repositories\n- **Multiple programming languages supported** (primarily TypeScript/JavaScript for SDK)\n\n---\n\n**OUTCOME: success** Zapier offers comprehensive programmatic capabilities through SDK (for coding agents), MCP (for AI chat tools), and CLI (for developers), plus robust integration building platforms with 9,000+ pre-built connectors, managed authentication, embedding APIs, and enterprise governance features suitable for teams of any size." } ], - "action_sequence": "fetch,fetch+fetch,fetch,fetch,search+fetch,fetch+fetch,fetch+search,fetch+fetch,search+fetch,fetch+search,fetch+fetch", + "action_sequence": "fetch+fetch+search,fetch+fetch+fetch,fetch+fetch,fetch,search+search,fetch", "signals_observed": [ + "mcp-server", "developer-portal" ], - "friction_outcome": "timeout", + "friction_outcome": "succeeded_natively", "status_profile": { - "count_2xx": 10, - "count_3xx": 2, - "count_4xx": 3, - "count_5xx": 0, - "first_error_step": 7, - "first_error_kind": "well_known" + "count_2xx": 7, + "count_3xx": 0, + "count_4xx": 0, + "count_5xx": 0 }, "well_known_probes": [], "path_origin_distribution": { - "prior_knowledge": 0.2, - "web_search": 0.4, - "previous_artifact": 0.4, + "prior_knowledge": 0.33, + "web_search": 0, + "previous_artifact": 0.67, "other": 0 }, - "link_following_rate": 0.4 + "link_following_rate": 0.67 }, "insight": { - "summary": "The agent probed widely but couldn't complete the task: alongside successful reads it hit several 4xx responses and redirects, and the resource it needed was gated or absent. It exhausted reasonable paths and ended without a confident answer.", + "summary": "The agent answered cleanly: every fetch returned 200 and each step followed a link surfaced by the previous page, so it built one connected path through the site with no dead ends. It reached a confident answer without searching in circles or guessing URLs.", "key_observations": [ - "A cluster of 4xx responses shows the agent tried endpoints that don't exist or require authentication - the target wasn't discoverable from a published hint.", - "Redirects (3xx) bounced the agent across canonical URLs, adding navigation cost without resolving the task.", - "The failure reflects a real agent-readiness gap for this task: the needed capability isn't exposed in a way a cold-start agent can reach." + "Navigation was link-led - most steps were attributed to a previous artifact (a URL found in an earlier response), the strongest signal of an agent-navigable site.", + "Every request resolved 2xx; the agent never had to recover from an error or fall back to web search.", + "The connected parent→child chain shows the site exposes its structure so agents can follow it directly, rather than forcing prior-knowledge guesses." ], - "generated_at": "2026-05-30T07:26:26.282Z" + "generated_at": "2026-06-02T08:01:28.276Z" } } } diff --git a/src/api/__fixtures__/deep-journey-refusals.json b/src/api/__fixtures__/deep-journey-refusals.json new file mode 100644 index 0000000..31a07d0 --- /dev/null +++ b/src/api/__fixtures__/deep-journey-refusals.json @@ -0,0 +1,29 @@ +{ + "INVALID_DOMAIN": { + "error": "Invalid input", + "code": "INVALID_DOMAIN", + "next_action": { + "method": "POST", + "url": "/api/journey/runs", + "reason": "Send a public host such as stripe.com as intent.domain, then send the request again." + } + }, + "UNSUPPORTED_AGENT": { + "error": "Unsupported agent", + "code": "UNSUPPORTED_AGENT", + "next_action": { + "method": "GET", + "url": "/api/journey/agents", + "reason": "This agent cannot run here. Pick one from this list and send its harness and model." + } + }, + "CUSTOM_INTENT_REQUIRES_KEY": { + "error": "Custom intents require a partner API key", + "code": "CUSTOM_INTENT_REQUIRES_KEY", + "next_action": { + "method": "GET", + "url": "https://ora.ai/docs/journeys", + "reason": "Free-text tasks need an ora partner API key, and this page says how to get one. Without a key, send a curated intent_id instead." + } + } +} diff --git a/src/api/__fixtures__/ora-skill-with-workflows.md b/src/api/__fixtures__/ora-skill-with-workflows.md new file mode 100644 index 0000000..f523ee9 --- /dev/null +++ b/src/api/__fixtures__/ora-skill-with-workflows.md @@ -0,0 +1,77 @@ +--- +name: ora +description: Discover agent-ready products, check scores, and share feedback using ora. Use when scanning a domain for agent readiness, finding the best product for a task, or reading/submitting agent feedback about a product. Designed for agents evaluating products before integration. +--- + +# Ora - Agent-Readiness Platform + +Discover products, check agent-readiness scores, read and submit agent feedback. + +## When to Use + +Use this skill when the user or task involves any of the following: + +- **Evaluating a product for agent use** - "Is Stripe agent-ready?", "Can an AI agent use this API?", "How well does X support MCP?" +- **Choosing between products** - "What's the best payment API for agents?", "Which CRM has the best agent integration?" +- **Checking agent-readiness score** - "What's the ora score for notion.com?", "Scan linear.app" +- **Submitting feedback after using a product** - After an agent interaction with an external product, submit a feedback report +- **Reviewing check accuracy** - Report a false pass, false fail, or outdated check result for a domain + +Do NOT use this skill for general web searches, internal product questions, or tasks unrelated to evaluating or discovering external software products. + +## Capabilities + +1. **Scan a domain** - Check how agent-ready a product is (score 0-100, grade A+ to F) +2. **Discover products** - Find the best product for a task by describing what you need +3. **Read feedback** - See what other agents experienced with a product +4. **Submit feedback** - Share your experience after using a product (helps future agents) + +## Scanning a Domain + +1. **Quick lookup first**: Try `GET https://ora.ai/api/score/{domain}` to check for a cached score. +2. **If 404 (no cached score)**: Run a fresh scan with `POST https://ora.ai/api/scan` with body `{"url": "{domain}"}`. +3. **Display results** in this format: + +``` +## ora Score: {score}/100 (Grade: {grade}) + +Domain: {domain} +Scanned: {scannedAt} + +### Layers +| Layer | Score | Checks Passed | +|---|---|---| +| {layer.name} | {layer.score}/{layer.maxScore} | {passed}/{total} | + +### Failed Checks +| Check | Layer | Details | +|---|---|---| +| {check.name} | {layer.name} | {check.details} | + +Full report: https://ora.ai/score/{domain} +``` + +4. Only show failed checks (status = "fail") - skip passed/warning/error checks in the table. +5. If there are no failed checks, say "All checks passed!" instead of the Failed Checks table. + +## Workflows + +Every ax command ends with a `Next:` line that names the call to make next. + +### Score a site, then send an agent at it + +1. `ax audit {domain}` scores the site and lists its Top fixes. +2. Read the report: start with the Top fixes, ranked by ora. +3. `ax deep-journey {domain}` sends a real agent at the site on a task. +4. Read its result: whether the agent finished, and where it got stuck. + +``` +## not a heading: a fenced example inside the section +``` + +## Discovering Products + +When the user needs a product for a task: + +1. `GET https://ora.ai/api/discover?intent={description}&limit=5` +2. Display the top results with score, grade, and category. diff --git a/src/api/__fixtures__/studio/agents.json b/src/api/__fixtures__/studio/agents.json new file mode 100644 index 0000000..7d907d1 --- /dev/null +++ b/src/api/__fixtures__/studio/agents.json @@ -0,0 +1,34 @@ +{ + "success": true, + "data": { + "agents": [ + { + "harness": "codex", + "harnessName": "Codex", + "model": "gpt-5.6-luna", + "modelName": "GPT-5.6 Luna", + "provider": "openai", + "available": false, + "reason": "needs the Standard plan" + }, + { + "harness": "claude-code", + "harnessName": "Claude Code", + "model": "claude-haiku-4-5-20251001", + "modelName": "Claude Haiku 4.5", + "provider": "anthropic", + "available": true, + "reason": null + }, + { + "harness": "claude-code", + "harnessName": "Claude Code", + "model": "claude-sonnet-5-5", + "modelName": "Claude Sonnet 5.5", + "provider": "anthropic", + "available": true, + "reason": null + } + ] + } +} diff --git a/src/api/__fixtures__/studio/cancel.json b/src/api/__fixtures__/studio/cancel.json new file mode 100644 index 0000000..a2803cc --- /dev/null +++ b/src/api/__fixtures__/studio/cancel.json @@ -0,0 +1,12 @@ +{ + "requested": 3, + "canceled": [ + "2b7e9c41-6d0a-4f38-8a1e-5c9d3f7b2e64" + ], + "alreadyTerminal": [ + "c40a8f16-93be-4d27-b5f1-0e6a2d8c9b37" + ], + "skipped": [ + "7a2d4e81-5c3f-4b90-9e6a-1f8b0c2d7e45" + ] +} diff --git a/src/api/__fixtures__/studio/error-402-quota.json b/src/api/__fixtures__/studio/error-402-quota.json new file mode 100644 index 0000000..20423ae --- /dev/null +++ b/src/api/__fixtures__/studio/error-402-quota.json @@ -0,0 +1,10 @@ +{ + "error": "Payment Required", + "message": "Usage limit exceeded for Agent Runs", + "code": "LIMIT_EXCEEDED", + "details": { + "dimension": "agent_runs", + "limit": 50, + "used": 50 + } +} diff --git a/src/api/__fixtures__/studio/error-403-missing-scope.json b/src/api/__fixtures__/studio/error-403-missing-scope.json new file mode 100644 index 0000000..55b5e75 --- /dev/null +++ b/src/api/__fixtures__/studio/error-403-missing-scope.json @@ -0,0 +1,9 @@ +{ + "error": "Forbidden", + "code": "INSUFFICIENT_SCOPE", + "details": { + "missing": [ + "journeys:write" + ] + } +} diff --git a/src/api/__fixtures__/studio/journey-create-partial.json b/src/api/__fixtures__/studio/journey-create-partial.json new file mode 100644 index 0000000..b3c48df --- /dev/null +++ b/src/api/__fixtures__/studio/journey-create-partial.json @@ -0,0 +1,19 @@ +{ + "success": true, + "data": { + "journeyId": "e17b5c90-2a4d-4f6e-8b13-9d0c7a5e3f21", + "runIds": [ + "7a2d4e81-5c3f-4b90-9e6a-1f8b0c2d7e45" + ], + "streamUrls": [ + "/studio/v1/runs/7a2d4e81-5c3f-4b90-9e6a-1f8b0c2d7e45/stream" + ], + "failed": [ + { + "harness": "claude-code", + "model": "claude-sonnet-5-5", + "reason": "Could not resolve your plan tier (billing service unavailable). Retry shortly." + } + ] + } +} diff --git a/src/api/__fixtures__/studio/journey-create.json b/src/api/__fixtures__/studio/journey-create.json new file mode 100644 index 0000000..4998ecc --- /dev/null +++ b/src/api/__fixtures__/studio/journey-create.json @@ -0,0 +1,14 @@ +{ + "success": true, + "data": { + "journeyId": "8d3f6a2e-41c7-4b9a-9e25-7f0c1d4b6a83", + "runIds": [ + "2b7e9c41-6d0a-4f38-8a1e-5c9d3f7b2e64", + "c40a8f16-93be-4d27-b5f1-0e6a2d8c9b37" + ], + "streamUrls": [ + "/studio/v1/runs/2b7e9c41-6d0a-4f38-8a1e-5c9d3f7b2e64/stream", + "/studio/v1/runs/c40a8f16-93be-4d27-b5f1-0e6a2d8c9b37/stream" + ] + } +} diff --git a/src/api/__fixtures__/studio/journey-detail-final.json b/src/api/__fixtures__/studio/journey-detail-final.json new file mode 100644 index 0000000..a002fd5 --- /dev/null +++ b/src/api/__fixtures__/studio/journey-detail-final.json @@ -0,0 +1,70 @@ +{ + "success": true, + "data": { + "id": "8d3f6a2e-41c7-4b9a-9e25-7f0c1d4b6a83", + "goal": "Find how to create an issue from the API. Use Linear.", + "targetDomain": "linear.app", + "status": "completed", + "outcome": "partial", + "competitors": [], + "layer": null, + "intentId": null, + "intentTitle": null, + "createdAt": "2026-11-13T14:02:11.482Z", + "updatedAt": "2026-11-13T14:09:47.215Z", + "agents": [ + { + "runId": "2b7e9c41-6d0a-4f38-8a1e-5c9d3f7b2e64", + "harness": "claude-code", + "model": "claude-haiku-4-5-20251001", + "status": "succeeded", + "targetDomain": "linear.app", + "provider": "anthropic", + "executionKind": "harness", + "intentText": "Find how to create an issue from the API. Use Linear.", + "startedAt": "2026-11-13T14:02:14.120Z", + "finishedAt": "2026-11-13T14:05:02.877Z", + "createdAt": "2026-11-13T14:02:11.530Z", + "usage": { + "durationMs": 168757, + "totalTokens": 119100, + "inputTokens": 101000, + "cacheReadTokens": 14000, + "cacheCreationTokens": 0, + "outputTokens": 4100, + "numTurns": 22, + "costUsd": 0.031 + }, + "targetCheck": null, + "skillUse": null, + "verdict": "succeeded" + }, + { + "runId": "c40a8f16-93be-4d27-b5f1-0e6a2d8c9b37", + "harness": "claude-code", + "model": "claude-sonnet-5-5", + "status": "succeeded", + "targetDomain": "linear.app", + "provider": "anthropic", + "executionKind": "harness", + "intentText": "Find how to create an issue from the API. Use Linear.", + "startedAt": "2026-11-13T14:05:03.402Z", + "finishedAt": "2026-11-13T14:08:55.019Z", + "createdAt": "2026-11-13T14:02:11.561Z", + "usage": { + "durationMs": 231617, + "totalTokens": 204310, + "inputTokens": 188020, + "cacheReadTokens": 52000, + "cacheCreationTokens": 0, + "outputTokens": 6290, + "numTurns": 31, + "costUsd": 0.412 + }, + "targetCheck": null, + "skillUse": null, + "verdict": "failed" + } + ] + } +} diff --git a/src/api/__fixtures__/studio/journey-detail-running.json b/src/api/__fixtures__/studio/journey-detail-running.json new file mode 100644 index 0000000..282d401 --- /dev/null +++ b/src/api/__fixtures__/studio/journey-detail-running.json @@ -0,0 +1,70 @@ +{ + "success": true, + "data": { + "id": "8d3f6a2e-41c7-4b9a-9e25-7f0c1d4b6a83", + "goal": "Find how to create an issue from the API. Use Linear.", + "targetDomain": "linear.app", + "status": "running", + "outcome": null, + "competitors": [], + "layer": null, + "intentId": null, + "intentTitle": null, + "createdAt": "2026-11-13T14:02:11.482Z", + "updatedAt": "2026-11-13T14:02:19.907Z", + "agents": [ + { + "runId": "2b7e9c41-6d0a-4f38-8a1e-5c9d3f7b2e64", + "harness": "claude-code", + "model": "claude-haiku-4-5-20251001", + "status": "running", + "targetDomain": "linear.app", + "provider": "anthropic", + "executionKind": "harness", + "intentText": "Find how to create an issue from the API. Use Linear.", + "startedAt": "2026-11-13T14:02:14.120Z", + "finishedAt": null, + "createdAt": "2026-11-13T14:02:11.530Z", + "usage": { + "durationMs": null, + "totalTokens": null, + "inputTokens": null, + "cacheReadTokens": null, + "cacheCreationTokens": null, + "outputTokens": null, + "numTurns": null, + "costUsd": null + }, + "targetCheck": null, + "skillUse": null, + "verdict": null + }, + { + "runId": "c40a8f16-93be-4d27-b5f1-0e6a2d8c9b37", + "harness": "claude-code", + "model": "claude-sonnet-5-5", + "status": "queued", + "targetDomain": "linear.app", + "provider": "anthropic", + "executionKind": "harness", + "intentText": "Find how to create an issue from the API. Use Linear.", + "startedAt": null, + "finishedAt": null, + "createdAt": "2026-11-13T14:02:11.561Z", + "usage": { + "durationMs": null, + "totalTokens": null, + "inputTokens": null, + "cacheReadTokens": null, + "cacheCreationTokens": null, + "outputTokens": null, + "numTurns": null, + "costUsd": null + }, + "targetCheck": null, + "skillUse": null, + "verdict": null + } + ] + } +} diff --git a/src/api/__fixtures__/studio/trajectory.json b/src/api/__fixtures__/studio/trajectory.json new file mode 100644 index 0000000..8704bc4 --- /dev/null +++ b/src/api/__fixtures__/studio/trajectory.json @@ -0,0 +1,131 @@ +{ + "steps": [ + { + "id": 0, + "turn": 1, + "type": "text" + }, + { + "id": 1, + "turn": 2, + "type": "tool_call", + "tool": "WebSearch" + }, + { + "id": 2, + "turn": 3, + "type": "text" + }, + { + "id": 3, + "turn": 4, + "type": "tool_call", + "tool": "WebFetch", + "url_host": "ora.ai" + }, + { + "id": 4, + "turn": 5, + "type": "text" + }, + { + "id": 5, + "turn": 6, + "type": "tool_call", + "tool": "WebFetch", + "url_host": "ora.ai" + }, + { + "id": 6, + "turn": 7, + "type": "text" + }, + { + "id": 7, + "turn": 8, + "type": "tool_call", + "tool": "WebSearch" + }, + { + "id": 8, + "turn": 9, + "type": "text" + }, + { + "id": 9, + "turn": 10, + "type": "tool_call", + "tool": "WebFetch", + "url_host": "ora.ai" + }, + { + "id": 10, + "turn": 11, + "type": "text" + }, + { + "id": 11, + "turn": 12, + "type": "tool_call", + "tool": "WebFetch", + "url_host": "ora.ai" + }, + { + "id": 12, + "turn": 13, + "type": "text" + }, + { + "id": 13, + "turn": 14, + "type": "tool_call", + "tool": "WebFetch", + "url_host": "ora.run", + "status": 308 + }, + { + "id": 14, + "turn": 15, + "type": "text" + }, + { + "id": 15, + "turn": 16, + "type": "tool_call", + "tool": "WebFetch", + "url_host": "ora.ai" + }, + { + "id": 16, + "turn": 17, + "type": "text" + }, + { + "id": 17, + "turn": 18, + "type": "tool_call", + "tool": "WebSearch" + }, + { + "id": 18, + "turn": 19, + "type": "text" + }, + { + "id": 19, + "turn": 20, + "type": "tool_call", + "tool": "WebSearch" + }, + { + "id": 20, + "turn": 21, + "type": "text" + }, + { + "id": 21, + "turn": 22, + "type": "text" + } + ] +} diff --git a/src/api/__fixtures__/studio/usage-unlimited.json b/src/api/__fixtures__/studio/usage-unlimited.json new file mode 100644 index 0000000..0a4753d --- /dev/null +++ b/src/api/__fixtures__/studio/usage-unlimited.json @@ -0,0 +1,16 @@ +{ + "success": true, + "data": { + "runs": { + "used": 212, + "limit": null, + "remaining": null + }, + "scans": { + "used": null, + "limit": null, + "remaining": null + }, + "resetsAt": null + } +} diff --git a/src/api/__fixtures__/studio/usage.json b/src/api/__fixtures__/studio/usage.json new file mode 100644 index 0000000..f80eb43 --- /dev/null +++ b/src/api/__fixtures__/studio/usage.json @@ -0,0 +1,16 @@ +{ + "success": true, + "data": { + "runs": { + "used": 12, + "limit": 50, + "remaining": 38 + }, + "scans": { + "used": 4, + "limit": 30, + "remaining": 26 + }, + "resetsAt": "2026-12-01T00:00:00.000Z" + } +} diff --git a/src/api/agent-files.ts b/src/api/agent-files.ts index 9986076..7e695dc 100644 --- a/src/api/agent-files.ts +++ b/src/api/agent-files.ts @@ -44,7 +44,7 @@ export interface AgentFilesBundle { /** * Any failure to get the files from ora. `status` is the HTTP status when ora * answered (400 = not a public domain), and `payload` the error body exactly as - * served, which is what `--json` prints. + * served, whose `errorCode` is the code `--json` reports. */ export class AgentFilesApiError extends Error { readonly status: number | null; diff --git a/src/api/audit.test.ts b/src/api/audit.test.ts index 5197091..c4693dd 100644 --- a/src/api/audit.test.ts +++ b/src/api/audit.test.ts @@ -6,6 +6,7 @@ import { BUILT_AGAINST, resetContractWarning } from "../contract"; // hand-rolled - tests derive variations from it with explicit deltas. import realAuditScan from "./__fixtures__/audit-scan.json"; import { AuditApiError, isMcpAuthRequired, MCP_AUTH_REQUIRED, performAudit } from "./audit"; +import { KeyDestinationError } from "./key-destination"; const FIXTURE = realAuditScan as unknown as AuditScanResult; @@ -394,3 +395,29 @@ describe("performAudit", () => { await expect(performAudit("example.com", { idleMs: 40 })).rejects.toThrow(/timed out/i); }); }); + +describe("performAudit key destination", () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + it("never sends a scan key over plain http to a host other than this machine", async () => { + const fetchMock = vi.fn(async () => new Response("{}")); + vi.stubGlobal("fetch", fetchMock); + const keyed = { apiKey: "sk_test", baseUrl: "http://collector.example" }; + await expect(performAudit("example.com", keyed)).rejects.toThrow(KeyDestinationError); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("still audits without a key against any host the caller named", async () => { + const fetchMock = vi.fn(async () => { + throw new TypeError("fetch failed"); + }); + vi.stubGlobal("fetch", fetchMock); + vi.stubEnv("ORA_SCAN_API_KEY", ""); + const keyless = { apiKey: "", baseUrl: "http://collector.example" }; + await expect(performAudit("example.com", keyless)).rejects.not.toThrow(KeyDestinationError); + expect(fetchMock).toHaveBeenCalled(); + }); +}); diff --git a/src/api/audit.ts b/src/api/audit.ts index bbe76f2..e5ce459 100644 --- a/src/api/audit.ts +++ b/src/api/audit.ts @@ -1,5 +1,6 @@ import type { AuditScanResult, AuditScoreResult } from "../contract"; import { warnOnNewerContract } from "../contract"; +import { assertKeyDestination } from "./key-destination"; import { errorBody, pause, usageLimitMessage, watchdog } from "./shared"; // Client for ora's public agent-readiness audit. Uses the SSE endpoint @@ -108,6 +109,7 @@ export async function performAudit( // agree — the poll target is rate limited too, and a keyed scan whose polls // went keyless would throttle exactly the high-volume callers keys exist for. options = { ...options, apiKey: options.apiKey || process.env.ORA_SCAN_API_KEY || undefined }; + if (options.apiKey) assertKeyDestination(base, "ORA_API_URL", PUBLIC_BASE); options.progress?.(`Auditing ${target} with ora`); const streamed = await consumeScanStream(base, target, options); @@ -158,10 +160,12 @@ async function consumeScanStream( if (res.status === 429) { dog.disarm(); const usage = await usageLimitMessage(res); - if (usage) throw new AuditApiError(`ora usage limit reached: ${usage}`); + const { payload } = await errorBody(res); + if (usage) throw new AuditApiError(`ora usage limit reached: ${usage}`, { payload }); const wait = res.headers.get("retry-after"); throw new AuditApiError( `ora rate limit exceeded${wait ? ` — retry after ${wait}s` : ""} (burst: 10 scans/min/IP; daily: 30 scans + 6 force per 24h). An ora-issued scan API key lifts these limits: set ORA_SCAN_API_KEY or pass --api-key.`, + { payload }, ); } if (!res.ok || !res.body) { diff --git a/src/api/deep-journey.test.ts b/src/api/deep-journey.test.ts index dcf68ea..2b133ee 100644 --- a/src/api/deep-journey.test.ts +++ b/src/api/deep-journey.test.ts @@ -1,17 +1,27 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import realDetail from "./__fixtures__/deep-journey-detail.json"; +import runningDetail from "./__fixtures__/deep-journey-detail-running.json"; +import refusals from "./__fixtures__/deep-journey-refusals.json"; import { DeepJourneyApiError, fetchJourneyAgents, formatWait, performDeepJourney, + servedNextAction, } from "./deep-journey"; +import { KeyDestinationError } from "./key-destination"; -// Fixture provenance: deep-journey-detail.json is a real projected -// GET /api/journey/runs/{id} response recorded from a dev run (contract -// 1.9.0). Variations below are explicit deltas on it, never hand-rolled. +// Fixture provenance, all real responses from a local dev server (contract +// 1.42.0, mock engine): deep-journey-detail.json is the projected +// GET /api/journey/runs/{id} of one run after its stream was followed to the +// result, deep-journey-detail-running.json the same GET before that, and +// deep-journey-refusals.json the POST /api/journey/runs 400 bodies for +// intent.domain "localhost" and an unlisted model, keyed by their code. +// Variations below are explicit deltas on them, never hand-rolled. const RUN_ID = (realDetail as { id: string }).id; +const STREAM_NEXT = runningDetail.next_action; +const INVALID_DOMAIN_BODY = refusals.INVALID_DOMAIN; // The journey stream is named SSE, the same framing the platform client // decodes (event: X + data: {json} + ": ping" heartbeats). @@ -121,8 +131,12 @@ describe("performDeepJourney", () => { expect(outcome.cached).toBe(false); expect(outcome.allowance).toEqual({ limit: 5, remaining: 4, resetAtMs: 1770000000000 }); expect(outcome.detail.status).toBe("succeeded"); - expect(outcome.detail.step_count).toBe(19); - expect(outcome.detail.verdict).toBe("unsatisfied"); + expect(outcome.detail.step_count).toBe(12); + expect(outcome.detail.verdict).toBe("satisfied"); + expect(outcome.detail.flags).toEqual({ + blocked: { flagged: false }, + answered_from_memory: { flagged: false }, + }); expect(lines.some((line) => line.includes("step 1"))).toBe(true); }); @@ -208,7 +222,22 @@ describe("performDeepJourney", () => { }), ); - await expect(performDeepJourney("vercel.com", OPTIONS)).rejects.toThrow(DeepJourneyApiError); + const failure = performDeepJourney("vercel.com", OPTIONS); + await expect(failure).rejects.toThrow(DeepJourneyApiError); + await expect(failure).rejects.toMatchObject({ + code: "RATE_LIMITED", + payload: { error: "Too many requests", retry_after_ms: 86_400_000 }, + }); + }); + + it("carries the next action ora served with a refusal", async () => { + vi.stubGlobal("fetch", journeyMock({ trigger: asJson(INVALID_DOMAIN_BODY, 400) })); + + const failure = performDeepJourney("localhost", OPTIONS); + await expect(failure).rejects.toMatchObject({ + payload: INVALID_DOMAIN_BODY, + nextAction: INVALID_DOMAIN_BODY.next_action, + }); }); it("names the key's monthly journey limit on a usage_limit_reached 429", async () => { @@ -233,6 +262,10 @@ describe("performDeepJourney", () => { const failure = performDeepJourney("vercel.com", { ...OPTIONS, apiKey: "k" }); await expect(failure).rejects.toThrow(/journeys limit for the month \(200 of 200\)/); + await expect(failure).rejects.toMatchObject({ + code: "USAGE_LIMIT_REACHED", + payload: { error: "usage_limit_reached", used: 200, limit: 200 }, + }); }); it("refuses to treat a never-settling run as terminal (truncated stream)", async () => { @@ -268,6 +301,42 @@ describe("performDeepJourney", () => { const error = await settled; expect(error).toBeInstanceOf(DeepJourneyApiError); expect(String(error)).toMatch(/still executing — the stream could not be followed/); + // Coded so the command points at the run instead of starting another. + expect(error).toMatchObject({ + code: "RUN_STILL_RUNNING", + runUrl: `https://journey.test/api/journey/runs/${RUN_ID}`, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("hands up the stream next action of a run it leaves running, --no-stream included", async () => { + vi.useFakeTimers(); + try { + vi.stubGlobal( + "fetch", + journeyMock({ + stream: () => journeyStream([["run_id", { run_id: RUN_ID }]]), + details: [runningDetail], + }), + ); + + const settled = performDeepJourney("vercel.com", { + ...OPTIONS, + noStream: true, + reopenPauseMs: 1, + pollEveryMs: 1, + pollLimit: 2, + }).then( + () => "resolved", + (cause) => cause, + ); + await vi.advanceTimersByTimeAsync(10_000); + expect(await settled).toMatchObject({ + code: "RUN_STILL_RUNNING", + nextAction: STREAM_NEXT, + }); } finally { vi.useRealTimers(); } @@ -447,9 +516,19 @@ describe("performDeepJourney - keyed tier", () => { }), ); - await expect( - performDeepJourney("vercel.com", { ...OPTIONS, intentId: undefined, task: "Find the docs" }), - ).rejects.toThrow(/ORA_PARTNER_API_KEY|--api-key/); + const failure = performDeepJourney("vercel.com", { + ...OPTIONS, + intentId: undefined, + task: "Find the docs", + }); + await expect(failure).rejects.toThrow(/ORA_PARTNER_API_KEY|--api-key/); + await expect(failure).rejects.toMatchObject({ + code: "NEEDS_KEY", + payload: { + error: "Custom intents require a partner API key", + code: "CUSTOM_INTENT_REQUIRES_KEY", + }, + }); }); it("sends the partner key on the stream too, which lifts its per-IP limit", async () => { @@ -568,6 +647,19 @@ describe("fetchJourneyAgents", () => { }); }); +describe("servedNextAction", () => { + it("reads a next action in the REST shape and ignores anything else", () => { + expect(servedNextAction(INVALID_DOMAIN_BODY)).toEqual(INVALID_DOMAIN_BODY.next_action); + expect(servedNextAction(realDetail)).toBeUndefined(); + expect( + servedNextAction({ next_action: { call: "ax deep-journey zapier.com" } }), + ).toBeUndefined(); + expect(servedNextAction({ next_action: { ...STREAM_NEXT, method: "DELETE" } })).toBeUndefined(); + expect(servedNextAction(null)).toBeUndefined(); + expect(servedNextAction("Too many requests")).toBeUndefined(); + }); +}); + describe("formatWait", () => { it("rounds up to the readable unit", () => { expect(formatWait(86_400_000)).toBe("24h"); @@ -576,3 +668,15 @@ describe("formatWait", () => { expect(formatWait(0)).toBe("soon"); }); }); + +describe("performDeepJourney key destination", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("never sends a partner key over plain http to a host other than this machine", async () => { + const fetchMock = vi.fn(async () => new Response("{}")); + vi.stubGlobal("fetch", fetchMock); + const keyed = { ...OPTIONS, apiKey: "pk_test", baseUrl: "http://collector.example" }; + await expect(performDeepJourney("vercel.com", keyed)).rejects.toThrow(KeyDestinationError); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/api/deep-journey.ts b/src/api/deep-journey.ts index 7043d6b..ce100ae 100644 --- a/src/api/deep-journey.ts +++ b/src/api/deep-journey.ts @@ -1,12 +1,14 @@ import type { + JourneyNextAction, JourneyRun, JourneyRunDetail, JourneyRunResult, JourneyTrajectoryStep, } from "../contract"; import { warnOnNewerContract } from "../contract"; +import { assertKeyDestination } from "./key-destination"; import { decodeEventBlock } from "./platform"; -import { errorBodyText, pause, usageLimitMessage, watchdog } from "./shared"; +import { errorBody, pause, usageLimitMessage, watchdog } from "./shared"; // Client for ora's public deep-journey API (/api/journey/*): a real AI agent // attempts a curated task against a domain while ora records the trajectory. @@ -22,6 +24,8 @@ import { errorBodyText, pause, usageLimitMessage, watchdog } from "./shared"; // shared from there. const PUBLIC_BASE = "https://ora.ai"; +/** Where a run is started: `ax journey --domain ` is the ax call that sends this POST. */ +export const JOURNEY_RUNS_PATH = "/api/journey/runs"; // A live engine run can take several minutes end to end, but frames arrive at // least every few seconds while it works; this bounds SILENCE, not the run. const STREAM_IDLE_MS = 120_000; @@ -40,8 +44,60 @@ const MAX_RETRY_AFTER_MS = 60_000; const STREAM_REOPENS = 5; const STREAM_REOPEN_PAUSE_MS = 2_000; -/** Any failure to obtain a run from ora: network, HTTP, rate limit, stream error. */ -export class DeepJourneyApiError extends Error {} +export type DeepJourneyErrorCode = + | "NEEDS_KEY" + | "RATE_LIMITED" + | "USAGE_LIMIT_REACHED" + | "ENGINE_UNAVAILABLE" + | "RUN_STILL_RUNNING"; + +/** + * The `next_action` a journey body carries (a run detail or an error body), + * when it has one in the REST shape; anything else carries none. + */ +export function servedNextAction(body: unknown): JourneyNextAction | undefined { + if (!body || typeof body !== "object") return undefined; + const action = (body as { next_action?: unknown }).next_action; + if (!action || typeof action !== "object") return undefined; + const { method, url, reason } = action as Record; + return (method === "GET" || method === "POST") && + typeof url === "string" && + typeof reason === "string" + ? (action as JourneyNextAction) + : undefined; +} + +/** + * Any failure to obtain a run from ora: network, HTTP, rate limit, stream error. + * `code` names the failures a caller answers differently (a key to fix, a wait, + * a run to read rather than start again) and is null for the rest; `runUrl` is + * the run's detail URL when the run exists and is still executing; `payload` + * is the error body exactly as ora served it when ora answered with JSON + * (what `--json` prints), and null or undefined otherwise; `nextAction` is the + * request ora says to send next, from that body or from the run's detail. + */ +export class DeepJourneyApiError extends Error { + readonly code: DeepJourneyErrorCode | null; + readonly runUrl: string | undefined; + readonly payload: unknown; + readonly nextAction: JourneyNextAction | undefined; + + constructor( + message: string, + options: { + code?: DeepJourneyErrorCode | null; + runUrl?: string; + payload?: unknown; + nextAction?: JourneyNextAction; + } = {}, + ) { + super(message); + this.code = options.code ?? null; + this.runUrl = options.runUrl; + this.payload = options.payload; + this.nextAction = options.nextAction ?? servedNextAction(options.payload); + } +} /** The stream answered with an HTTP error rather than an event stream. */ class JourneyStreamHttpError extends DeepJourneyApiError { @@ -49,8 +105,9 @@ class JourneyStreamHttpError extends DeepJourneyApiError { message: string, readonly status: number, readonly retryAfterMs: number | undefined, + payload: unknown, ) { - super(message); + super(message, { payload }); } } @@ -138,7 +195,8 @@ export interface DeepJourneyOutcome { engineError?: string; } -function apiBase(baseUrl?: string): string { +/** The origin journey requests go to: the override, else $ORA_API_URL, else https://ora.ai. */ +export function apiBase(baseUrl?: string): string { return (baseUrl ?? process.env.ORA_API_URL ?? PUBLIC_BASE).replace(/\/+$/, ""); } @@ -153,7 +211,10 @@ async function getJson(url: string): Promise { const detail = cause instanceof Error ? cause.message : String(cause); throw new DeepJourneyApiError(`ora request failed: ${detail}`); } - if (!res.ok) throw new DeepJourneyApiError(`ora request failed: ${await errorBodyText(res)}`); + if (!res.ok) { + const { message, payload } = await errorBody(res); + throw new DeepJourneyApiError(`ora request failed: ${message}`, { payload }); + } return (await res.json()) as T; } @@ -213,7 +274,7 @@ async function triggerRun( let res: Response; try { - res = await fetch(`${base}/api/journey/runs`, { + res = await fetch(`${base}${JOURNEY_RUNS_PATH}`, { method: "POST", headers: { "content-type": "application/json", @@ -231,26 +292,40 @@ async function triggerRun( if (res.status === 401) { // The capability 401: free text needs a recognized partner key. (A // curated body never 401s - a bad key silently degrades to keyless.) + const { payload } = await errorBody(res); throw new DeepJourneyApiError( "free-text tasks need an ora partner API key — set ORA_PARTNER_API_KEY or pass --api-key (keys are issued by ora on request)", + { code: "NEEDS_KEY", payload }, ); } if (res.status === 429) { const usage = await usageLimitMessage(res); - if (usage) throw new DeepJourneyApiError(`ora usage limit reached: ${usage}`); + const { payload } = await errorBody(res); + if (usage) { + throw new DeepJourneyApiError(`ora usage limit reached: ${usage}`, { + code: "USAGE_LIMIT_REACHED", + payload, + }); + } const wait = res.headers.get("retry-after"); const caps = options.apiKey ? "keyed allowance: 1000 runs per 24h per key" : "burst: 20/min/IP; daily: 20 runs per 24h per IP — an ora partner API key raises this to 1000/24h per key: set ORA_PARTNER_API_KEY or pass --api-key"; throw new DeepJourneyApiError( `ora journey caller limit exceeded${wait ? ` — retry after ${formatWait(Number(wait) * 1000)}` : ""} (${caps})`, + { code: "RATE_LIMITED", payload }, ); } if (res.status === 503) { - throw new DeepJourneyApiError("ora's run engine is unavailable right now — try again shortly"); + const { payload } = await errorBody(res); + throw new DeepJourneyApiError("ora's run engine is unavailable right now — try again shortly", { + code: "ENGINE_UNAVAILABLE", + payload, + }); } if (!res.ok) { - throw new DeepJourneyApiError(`ora journey request failed: ${await errorBodyText(res)}`); + const { message, payload } = await errorBody(res); + throw new DeepJourneyApiError(`ora journey request failed: ${message}`, { payload }); } const record = (await res.json()) as TriggeredRun["record"]; @@ -323,10 +398,12 @@ async function followJourneyStream( if (!res.ok || !res.body) { dog.disarm(); const retryAfterS = Number(res.headers.get("retry-after")); + const { message, payload } = await errorBody(res); throw new JourneyStreamHttpError( - `ora journey stream failed (${res.status}): ${await errorBodyText(res)}`, + `ora journey stream failed (${res.status}): ${message}`, res.status, Number.isFinite(retryAfterS) && retryAfterS > 0 ? retryAfterS * 1000 : undefined, + payload, ); } @@ -473,6 +550,7 @@ export async function performDeepJourney( process.env.ORA_SCAN_API_KEY || undefined, }; + if (options.apiKey) assertKeyDestination(base, "ORA_API_URL", PUBLIC_BASE); const triggered = await triggerRun(base, target, options); const { record } = triggered; @@ -519,12 +597,14 @@ export async function performDeepJourney( if (engineError) { throw new DeepJourneyApiError(`ora journey run failed: ${engineError}`); } + const runUrl = `${base}/api/journey/runs/${record.id}`; throw new DeepJourneyApiError( `run ${record.id} is still executing — ${ streamFailure ? `the stream could not be followed (${streamFailure}) and polling ran out of time` : "the stream ended" - } before a terminal result. Check ${base}/api/journey/runs/${record.id} shortly`, + } before a terminal result. Check ${runUrl} shortly`, + { code: "RUN_STILL_RUNNING", runUrl, nextAction: servedNextAction(detail) }, ); } warnOnNewerContract(detail.contractVersion); diff --git a/src/api/key-destination.test.ts b/src/api/key-destination.test.ts new file mode 100644 index 0000000..30b4e9a --- /dev/null +++ b/src/api/key-destination.test.ts @@ -0,0 +1,132 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { loadLocalEnv } from "../env"; +import { + assertKeyDestination, + KeyDestinationError, + keyDestinationProblem, +} from "./key-destination"; + +const DEFAULT = "https://api.agentfront.sh"; +const problem = (base: string) => keyDestinationProblem(base, "ORA_PLATFORM_URL", DEFAULT); + +// A `.env` is resolved from cwd, so the cases that load one run in a throwaway +// directory: the repo's own .env must never feed a test. +describe("keyDestinationProblem", () => { + const originalCwd = process.cwd(); + const original = process.env.ORA_PLATFORM_URL; + let dir: string | undefined; + + afterEach(() => { + if (dir) { + // A load that finds no file forgets the last file's variables. It runs here, never + // back in the repo, whose .env must not be read. + rmSync(join(dir, ".env"), { force: true }); + loadLocalEnv(); + } + process.chdir(originalCwd); + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + // delete, not assignment: process.env would store the string "undefined". + if (original === undefined) delete process.env.ORA_PLATFORM_URL; + else process.env.ORA_PLATFORM_URL = original; + }); + + function loadEnvFile(contents: string): void { + // An exported value wins over the file, so one inherited from the shell would hide it. + delete process.env.ORA_PLATFORM_URL; + dir = mkdtempSync(join(tmpdir(), "ax-key-destination-")); + writeFileSync(join(dir, ".env"), contents); + process.chdir(dir); + loadLocalEnv(); + } + + it("passes the default host", () => { + expect(problem(DEFAULT)).toBeUndefined(); + expect(problem(`${DEFAULT}/`)).toBeUndefined(); + }); + + it("passes an https host the shell exported", () => { + process.env.ORA_PLATFORM_URL = "https://staging.example"; + expect(problem("https://staging.example")).toBeUndefined(); + }); + + it("passes plain http to this machine", () => { + for (const base of [ + "http://localhost:3000", + "http://127.0.0.1:9999", + "http://ora.localhost:1355", + "http://[::1]:8080", + ]) { + expect(problem(base), base).toBeUndefined(); + } + }); + + it("refuses plain http to any other host, including ones that only start like this machine", () => { + for (const base of [ + "http://api.example", + "http://127.0.0.1.collector.example", + "http://localhost.collector.example", + ]) { + const refused = problem(base); + expect(refused, base).toContain("ORA_PLATFORM_URL"); + expect(refused, base).toContain("https"); + } + }); + + it("refuses a value that is not a URL without echoing it", () => { + const refused = problem("not a url\u001b[2J"); + expect(refused).toContain("ORA_PLATFORM_URL"); + expect(refused).not.toContain("\u001b"); + }); + + it("refuses a host that only ./.env named, and says how to choose it", () => { + loadEnvFile("ORA_PLATFORM_URL=https://collector.example\n"); + const refused = problem("https://collector.example"); + expect(refused).toContain("./.env"); + expect(refused).toContain("collector.example"); + expect(refused).toContain("export ORA_PLATFORM_URL"); + }); + + it("passes the default host when ./.env names it", () => { + loadEnvFile("ORA_PLATFORM_URL=https://api.agentfront.sh\n"); + expect(problem(DEFAULT)).toBeUndefined(); + }); + + it("passes this machine when ./.env names it", () => { + loadEnvFile("ORA_PLATFORM_URL=http://localhost:4000\n"); + expect(problem("http://localhost:4000")).toBeUndefined(); + }); + + it("refuses a .localhost name from ./.env, since a resolver may send it off this machine", () => { + loadEnvFile("ORA_PLATFORM_URL=http://evil.localhost:4000\n"); + expect(problem("http://evil.localhost:4000")).toContain("export ORA_PLATFORM_URL"); + }); + + it("passes a .localhost name the shell exported", () => { + process.env.ORA_PLATFORM_URL = "http://ora.localhost:1355"; + expect(problem("http://ora.localhost:1355")).toBeUndefined(); + }); + + it("judges the host after any user info, not the user info", () => { + expect(problem("http://localhost@collector.example")).toContain("https"); + }); + + it("does not hold ./.env against a host the caller passed in code", () => { + loadEnvFile("ORA_PLATFORM_URL=https://collector.example\n"); + expect(problem("https://staging.example")).toBeUndefined(); + }); +}); + +describe("assertKeyDestination", () => { + it("throws the problem as a KeyDestinationError", () => { + expect(() => + assertKeyDestination("http://api.example", "ORA_API_URL", "https://ora.ai"), + ).toThrow(KeyDestinationError); + expect(() => + assertKeyDestination("https://ora.ai", "ORA_API_URL", "https://ora.ai"), + ).not.toThrow(); + }); +}); diff --git a/src/api/key-destination.ts b/src/api/key-destination.ts new file mode 100644 index 0000000..9c9ae26 --- /dev/null +++ b/src/api/key-destination.ts @@ -0,0 +1,74 @@ +import { fromLocalEnvFile } from "../env"; + +/** The variables that move where ax sends a key. */ +export type KeyBaseVariable = "ORA_PLATFORM_URL" | "ORA_API_URL"; + +/** ax refused to send a key to the host a base URL names. */ +export class KeyDestinationError extends Error { + constructor(message: string) { + super(message); + this.name = "KeyDestinationError"; + } +} + +function isLoopback(hostname: string): boolean { + return hostname === "localhost" || hostname === "[::1]" || /^127(\.\d{1,3}){3}$/.test(hostname); +} + +// `*.localhost` resolves to this machine (RFC 6761), which is how ora runs in development. +// Some Linux resolvers still ask the network for it, so a `.env` may name only `isLoopback` hosts. +function isThisMachine(hostname: string): boolean { + return isLoopback(hostname) || hostname.endsWith(".localhost"); +} + +/** + * Why ax will not send a key to `base`, or undefined when it may. + * + * The default host is always fine. Any other host gets the key only over https + * (plain http only to this machine), and only when the user chose it: a host + * that came from a `.env` in the current directory is refused, because that + * file may belong to whatever repository ax happens to run in. + */ +export function keyDestinationProblem( + base: string, + variable: KeyBaseVariable, + defaultBase: string, +): string | undefined { + let url: URL; + try { + url = new URL(base); + } catch { + return `${variable} is not a URL, so ax did not send your key anywhere.`; + } + if (url.origin === new URL(defaultBase).origin) return undefined; + const local = isThisMachine(url.hostname); + if (url.protocol !== "https:" && !local) { + return `${variable} is ${url.origin}. ax sends your key only over https, or over plain http to this machine.`; + } + if ( + !isLoopback(url.hostname) && + fromLocalEnvFile(variable) && + sameOrigin(process.env[variable], url) + ) { + return `${variable} in ./.env points at ${url.host}, so ax did not send your key there. To use that host, export ${variable} in your shell.`; + } + return undefined; +} + +function sameOrigin(value: string | undefined, url: URL): boolean { + try { + return value !== undefined && new URL(value).origin === url.origin; + } catch { + return false; + } +} + +/** Throws `KeyDestinationError` when ax must not send a key to `base`. */ +export function assertKeyDestination( + base: string, + variable: KeyBaseVariable, + defaultBase: string, +): void { + const problem = keyDestinationProblem(base, variable, defaultBase); + if (problem) throw new KeyDestinationError(problem); +} diff --git a/src/api/platform.test.ts b/src/api/platform.test.ts index bcfd513..86afe19 100644 --- a/src/api/platform.test.ts +++ b/src/api/platform.test.ts @@ -1,5 +1,12 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { decodeEventBlock, launchJourney, type RawRunStep } from "./platform"; +import { KeyDestinationError } from "./key-destination"; +import { + decodeEventBlock, + exchangeKeyGrant, + launchJourney, + PlatformApiError, + type RawRunStep, +} from "./platform"; // The run stream is *named* SSE — `event: X` + `data: {json}` blocks with // `: ping` heartbeat comments in between. A different framing from the scan @@ -225,6 +232,26 @@ describe("launchJourney", () => { await expect(launchJourney(REQUEST)).rejects.toThrow(/rate limit/i); }); + it("keeps the platform's JSON error body on the error, and none for a non-JSON one", async () => { + const served = { error: "invalid_harness", message: "unknown harness: nope" }; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string | URL) => { + if (String(url).endsWith("/auth/token")) return asJson({ token: "t" }); + return asJson(served, 422); + }), + ); + const failure = launchJourney(REQUEST); + await expect(failure).rejects.toThrow(PlatformApiError); + await expect(failure).rejects.toMatchObject({ status: 422, payload: served }); + + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("bad gateway", { status: 502 })), + ); + await expect(launchJourney(REQUEST)).rejects.toMatchObject({ status: 502, payload: null }); + }); + it("times out when the run stream goes silent", async () => { vi.stubGlobal( "fetch", @@ -247,3 +274,34 @@ describe("launchJourney", () => { await expect(launchJourney({ ...REQUEST, idleMs: 40 })).rejects.toThrow(/timed out/i); }); }); + +describe("exchangeKeyGrant", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("never sends the key over plain http to a host other than this machine", async () => { + const fetchMock = vi.fn(async () => asJson({ token: "t" })); + vi.stubGlobal("fetch", fetchMock); + await expect(exchangeKeyGrant("http://collector.example", "ora_sk_local")).rejects.toThrow( + KeyDestinationError, + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("bounds the exchange, so a platform that never answers does not hang the command", async () => { + const fetchMock = vi.fn(async (_url: string | URL, _init?: RequestInit) => + asJson({ token: "t" }), + ); + vi.stubGlobal("fetch", fetchMock); + await exchangeKeyGrant("https://platform.test", "ora_sk_local"); + expect(fetchMock.mock.calls[0][1]?.signal).toBeInstanceOf(AbortSignal); + }); + + it("checks the destination of a single journey's key too", async () => { + const fetchMock = vi.fn(async () => asJson({ token: "t" })); + vi.stubGlobal("fetch", fetchMock); + await expect( + launchJourney({ ...REQUEST, baseUrl: "http://collector.example" }), + ).rejects.toThrow(KeyDestinationError); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/api/platform.ts b/src/api/platform.ts index db41326..284fc9f 100644 --- a/src/api/platform.ts +++ b/src/api/platform.ts @@ -1,10 +1,11 @@ +import { assertKeyDestination, keyDestinationProblem } from "./key-destination"; import { pause, watchdog } from "./shared"; // Client for ora's authenticated platform API — the machinery behind `journey`. // One journey == one run: an agent harness pursuing an intent against a domain. -// (The separate POST /studio/v1/journeys endpoint is a server-side multi-agent -// fan-out needing the journeys:write scope; a runs:read + runs:write key is all -// this client requires, so it stays away from that endpoint on purpose.) +// A runs:read + runs:write key is all `journey` requires. The multi-agent +// fan-out at POST /studio/v1/journeys needs journeys:write and belongs to +// a set of journeys, through ./studio.ts, which reuses this file's key exchange. const PLATFORM_BASE = "https://api.agentfront.sh"; @@ -12,6 +13,39 @@ const PLATFORM_BASE = "https://api.agentfront.sh"; export function platformBase(override?: string): string { return (override ?? process.env.ORA_PLATFORM_URL ?? PLATFORM_BASE).replace(/\/+$/, ""); } + +/** Why the key must not go to the platform `platformBase()` names, checked before any call. */ +export function platformKeyProblem(): string | undefined { + return keyDestinationProblem(platformBase(), "ORA_PLATFORM_URL", PLATFORM_BASE); +} + +const KEY_EXCHANGE_TIMEOUT_MS = 30_000; + +/** + * The platform answered with an HTTP error. `payload` is the error body + * exactly as served when it was JSON (what `--json` prints), null otherwise. + */ +export class PlatformApiError extends Error { + readonly status: number; + readonly payload: unknown; + + constructor(message: string, options: { status: number; payload?: unknown }) { + super(message); + this.name = "PlatformApiError"; + this.status = options.status; + this.payload = options.payload; + } +} + +/** A response body parsed as JSON, or null when it is not JSON. */ +export function jsonOrNull(text: string): unknown { + try { + return JSON.parse(text); + } catch (err) { + if (err instanceof SyntaxError) return null; + throw err; + } +} const RUN_IDLE_MS = 120_000; // agents legitimately go quiet between turns // --- Wire shapes --- @@ -71,6 +105,8 @@ export interface JourneyStep { summary?: string; label?: string; status?: number; + /** The host a URL-bearing step requested; absent on steps that name no URL. */ + url_host?: string; duration_ms?: number; is_error?: boolean; output_structured?: { body?: string; links?: Array<{ title?: string; url?: string }> }; @@ -149,18 +185,39 @@ export interface JourneyRun { // The ora_sk_ secret goes in the authorization HEADER (never the body) and is // exchanged for a ~15-minute bearer token used on every later call. +/** The key exchange's answer: the bearer and, for an ora_sk_ key, its lifetime in seconds. */ +export interface KeyGrant { + token: string; + expiresIn?: number; +} + export async function exchangeKey(base: string, secret: string): Promise { + return (await exchangeKeyGrant(base, secret)).token; +} + +export async function exchangeKeyGrant(base: string, secret: string): Promise { + assertKeyDestination(base, "ORA_PLATFORM_URL", PLATFORM_BASE); const res = await fetch(`${base}/auth/token`, { method: "POST", headers: { authorization: `Bearer ${secret}`, "content-type": "application/json" }, body: "{}", + signal: AbortSignal.timeout(KEY_EXCHANGE_TIMEOUT_MS), }); - if (res.status === 401) throw new Error("ora rejected the API key (401). Check ORA_API_KEY."); - const payload = (await res.json().catch(() => ({}))) as { token?: string; error?: string }; - if (!res.ok || !payload.token) { - throw new Error(`ora auth failed (${res.status})${payload.error ? `: ${payload.error}` : ""}`); + const body = jsonOrNull(await res.text()); + if (res.status === 401) { + throw new PlatformApiError("ora rejected the API key (401). Check ORA_API_KEY.", { + status: 401, + payload: body, + }); } - return payload.token; + const payload = (body ?? {}) as { token?: string; expiresIn?: unknown; error?: string }; + const failed = `ora auth failed (${res.status})${payload.error ? `: ${payload.error}` : ""}`; + if (!res.ok) throw new PlatformApiError(failed, { status: res.status, payload: body }); + if (!payload.token) throw new Error(failed); + const { expiresIn } = payload; + return typeof expiresIn === "number" && expiresIn > 0 + ? { token: payload.token, expiresIn } + : { token: payload.token }; } // --- Run creation --- @@ -184,18 +241,23 @@ async function createRun(base: string, bearer: string, request: RunRequest): Pro headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json" }, body: JSON.stringify(body), }); - if (res.status === 429) throw new Error("ora rate limit hit (429). Wait a moment and retry."); - const text = await res.text(); + if (res.status === 429) { + throw new PlatformApiError("ora rate limit hit (429). Wait a moment and retry.", { + status: 429, + payload: jsonOrNull(text), + }); + } + let created: CreatedRun & { error?: string; message?: string }; try { created = JSON.parse(text); } catch { throw new Error(`ora returned a non-JSON response (${res.status})`); } - if (!res.ok || !created.runId) { - throw new Error(`ora run failed (${res.status}): ${created.message ?? created.error ?? text}`); - } + const failed = `ora run failed (${res.status}): ${created.message ?? created.error ?? text}`; + if (!res.ok) throw new PlatformApiError(failed, { status: res.status, payload: created }); + if (!created.runId) throw new Error(failed); return created; } @@ -237,7 +299,14 @@ async function followRunStream( headers: { authorization: `Bearer ${bearer}`, accept: "text/event-stream" }, signal: dog.signal, }); - if (!res.ok || !res.body) { + if (!res.ok) { + dog.disarm(); + throw new PlatformApiError(`ora run stream failed (${res.status})`, { + status: res.status, + payload: jsonOrNull(await res.text()), + }); + } + if (!res.body) { dog.disarm(); throw new Error(`ora run stream failed (${res.status})`); } diff --git a/src/api/skills.test.ts b/src/api/skills.test.ts index 60bcd82..d541019 100644 --- a/src/api/skills.test.ts +++ b/src/api/skills.test.ts @@ -1,6 +1,15 @@ import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { digestOf, fetchSkill, fetchSkillIndex, SkillApiError } from "./skills"; +import { digestOf, fetchSkill, fetchSkillIndex, SkillApiError, skillSection } from "./skills"; + +// The first 62 lines of ora's published skill +// (/.well-known/agent-skills/ora/SKILL.md) verbatim, with a Workflows section +// added before "Discovering Products": ora's skill does not carry one yet. +const ORA_SKILL = readFileSync( + new URL("./__fixtures__/ora-skill-with-workflows.md", import.meta.url), + "utf8", +); // Index shape mirrors ora's real /.well-known/agent-skills/index.json // (agentskills.io discovery 0.2.0): name, type, description, url, digest. @@ -51,11 +60,15 @@ describe("skill registry client", () => { it("refuses bytes whose digest does not match the index", async () => { vi.stubGlobal("fetch", route(INDEX, new TextEncoder().encode("tampered content"))); await expect(fetchSkill("agent-ready-website")).rejects.toThrow(/digest mismatch/); + await expect(fetchSkill("agent-ready-website")).rejects.toMatchObject({ + code: "DIGEST_MISMATCH", + }); }); it("names the known skills when asked for an unknown one", async () => { vi.stubGlobal("fetch", route(INDEX, SKILL_BYTES)); await expect(fetchSkill("nope")).rejects.toThrow(/unknown skill.*agent-ready-website/); + await expect(fetchSkill("nope")).rejects.toMatchObject({ code: "UNKNOWN_SKILL" }); }); it("wraps network failures as SkillApiError", async () => { @@ -66,5 +79,71 @@ describe("skill registry client", () => { }), ); await expect(fetchSkillIndex()).rejects.toThrow(SkillApiError); + await expect(fetchSkillIndex()).rejects.toMatchObject({ code: "REGISTRY_UNREACHABLE" }); + }); + + it("codes an HTTP failure as REGISTRY_ERROR", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("nope", { status: 503 })), + ); + await expect(fetchSkillIndex()).rejects.toMatchObject({ + code: "REGISTRY_ERROR", + payload: null, + }); + }); + + it("keeps a JSON error body from the registry as the payload", async () => { + const served = { error: "rate_limited", retry_after_ms: 1000 }; + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json(served, { status: 429 })), + ); + await expect(fetchSkillIndex()).rejects.toMatchObject({ + code: "REGISTRY_ERROR", + payload: served, + }); + }); +}); + +describe("skillSection", () => { + it("returns the section with its subsections, up to the next heading at its level", () => { + const section = skillSection(ORA_SKILL, "Workflows"); + expect(section?.split("\n")[0]).toBe("## Workflows"); + expect(section).toContain("### Score a site, then send an agent at it"); + expect(section).toContain("4. Read its result: whether the agent finished"); + expect(section).not.toContain("Discovering Products"); + }); + + it("reads a # line inside a fenced block as content, not a heading", () => { + // The fenced example report in "Scanning a Domain" opens with "## ora Score". + expect(skillSection(ORA_SKILL, "ora Score: {score}/100 (Grade: {grade})")).toBeNull(); + const scanning = skillSection(ORA_SKILL, "Scanning a Domain"); + expect(scanning).toContain("### Failed Checks"); + expect(scanning).toContain("If there are no failed checks"); + expect(scanning).not.toContain("## Workflows"); + expect(skillSection(ORA_SKILL, "Workflows")).toContain( + "## not a heading: a fenced example inside the section", + ); + }); + + it("matches the title case-insensitively and ignores closing hashes", () => { + expect(skillSection("# Skill\n\n## workflows ##\n\nsteps\n", "Workflows")).toBe( + "## workflows ##\n\nsteps", + ); + }); + + it("does not close a fence on a line that opens another with an info string", () => { + const markdown = ["## Workflows", "", "```", "```bash", "## inside", "```", "", "## Next"].join( + "\n", + ); + expect(skillSection(markdown, "Workflows")).toBe( + ["## Workflows", "", "```", "```bash", "## inside", "```"].join("\n"), + ); + }); + + it("is null when the skill has no such section, as ora's does not yet", () => { + const published = ORA_SKILL.replace(/## Workflows[\s\S]*?(?=## Discovering Products)/, ""); + expect(skillSection(published, "Workflows")).toBeNull(); }); }); diff --git a/src/api/skills.ts b/src/api/skills.ts index 105b923..dfa89cf 100644 --- a/src/api/skills.ts +++ b/src/api/skills.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { errorBody } from "./shared"; // Client for ora's digest-bound skill registry at // /.well-known/agent-skills/index.json. Skill content is NEVER bundled into @@ -8,7 +9,28 @@ import { createHash } from "node:crypto"; const PUBLIC_BASE = "https://ora.ai"; -export class SkillApiError extends Error {} +export type SkillErrorCode = + | "REGISTRY_UNREACHABLE" + | "REGISTRY_ERROR" + | "UNKNOWN_SKILL" + | "DIGEST_MISMATCH"; + +/** + * Any failure to get a skill from the registry; `code` says which. `payload` + * is the registry's error body exactly as served when it was JSON (what + * `--json` prints), null or undefined otherwise. + */ +export class SkillApiError extends Error { + readonly code: SkillErrorCode; + readonly payload: unknown; + + constructor(message: string, code: SkillErrorCode, options: { payload?: unknown } = {}) { + super(message); + this.name = "SkillApiError"; + this.code = code; + this.payload = options.payload; + } +} export interface SkillEntry { name: string; @@ -35,11 +57,19 @@ export async function fetchSkillIndex(baseUrl?: string): Promise { } catch (cause) { throw new SkillApiError( `could not reach the skill registry at ${base}: ${cause instanceof Error ? cause.message : String(cause)}`, + "REGISTRY_UNREACHABLE", ); } - if (!res.ok) throw new SkillApiError(`skill registry answered HTTP ${res.status}`); + if (!res.ok) { + const { payload } = await errorBody(res); + throw new SkillApiError(`skill registry answered HTTP ${res.status}`, "REGISTRY_ERROR", { + payload, + }); + } const body = (await res.json()) as { skills?: SkillEntry[] }; - if (!Array.isArray(body.skills)) throw new SkillApiError("skill registry index has no skills[]"); + if (!Array.isArray(body.skills)) { + throw new SkillApiError("skill registry index has no skills[]", "REGISTRY_ERROR"); + } return body.skills; } @@ -64,7 +94,10 @@ export async function fetchSkill(name: string, baseUrl?: string): Promise s.name === name); if (!entry) { const known = index.map((s) => s.name).join(", "); - throw new SkillApiError(`unknown skill ${JSON.stringify(name)} - the registry has: ${known}`); + throw new SkillApiError( + `unknown skill ${JSON.stringify(name)} - the registry has: ${known}`, + "UNKNOWN_SKILL", + ); } const url = entry.url.startsWith("http") ? entry.url : `${base}${entry.url}`; @@ -74,15 +107,67 @@ export async function fetchSkill(name: string, baseUrl?: string): Promise= fence.length && !fenceOpen[2].trim()) { + fence = undefined; + } + continue; + } + if (fence !== undefined) continue; + const heading = HEADING.exec(lines[i]); + if (!heading) continue; + const depth = heading[1].length; + if (start === -1) { + if (heading[2].toLowerCase() === wanted) { + start = i; + level = depth; + } + } else if (depth <= level) { + return lines.slice(start, i).join("\n").trimEnd(); + } + } + return start === -1 ? null : lines.slice(start).join("\n").trimEnd(); +} diff --git a/src/api/studio-errors.ts b/src/api/studio-errors.ts new file mode 100644 index 0000000..179a81a --- /dev/null +++ b/src/api/studio-errors.ts @@ -0,0 +1,139 @@ +import { plain } from "../ui/ansi"; +import { PlatformApiError } from "./platform"; +import type { CreateFailure, NotStartedJourney } from "./studio-types"; + +// The ways a Studio call fails, and the body readers they need. `isStudioFailure` is the one rule +// for which errors are the platform's (an HTTP error, no connection, an answer of the wrong shape), +// so a caller retries or reports those and lets a bug in ax surface. + +/** An HTTP answer as the client read it: the status and the parsed body, or null. */ +export interface StudioAnswer { + status: number; + payload: unknown; +} + +/** + * Studio answered with an HTTP error. `code` is the Error body's `code`, or the snake_case + * `error` a runs route answers with instead (`run_not_found`); `missingScopes` is a 403's + * `details.missing`, empty otherwise. + */ +export class StudioApiError extends PlatformApiError { + readonly code: string | null; + readonly missingScopes: string[]; + + constructor( + message: string, + options: { status: number; payload: unknown; code: string | null; missingScopes: string[] }, + ) { + super(message, options); + this.name = "StudioApiError"; + this.code = options.code; + this.missingScopes = options.missingScopes; + } + + /** A 502 without Studio's own body: a gateway failed, so runs may have started. */ + get gateway(): boolean { + return this.status === 502 && notStartedJourney(this) === null; + } +} + +/** No answer came back: the request failed to connect, broke off, or timed out. */ +export class StudioConnectionError extends Error { + constructor(base: string, cause: unknown) { + super( + `could not reach ora at ${base}: ${cause instanceof Error ? cause.message : String(cause)}`, + { + cause, + }, + ); + this.name = "StudioConnectionError"; + } +} + +/** An answer came back without the shape the public spec gives it. */ +export class StudioShapeError extends Error { + constructor(what: string, expected: string) { + super(`ora answered ${what} without ${expected}.`); + this.name = "StudioShapeError"; + } +} + +/** True for the platform's failures; false for anything else, which is a bug to let through. */ +export function isStudioFailure(error: unknown): boolean { + return ( + error instanceof PlatformApiError || + error instanceof StudioConnectionError || + error instanceof StudioShapeError + ); +} + +/** + * Studio's own 502 on create: no agent started, and the body still names the journey and each + * agent that failed (apps/studio-service/src/journeys/journeys-router.ts:340). Null for any other + * error, a gateway's 502 included, since that does not prove nothing started. + */ +export function notStartedJourney(error: unknown): NotStartedJourney | null { + if (!(error instanceof StudioApiError) || error.status !== 502) return null; + const data = isRecord(error.payload) ? error.payload.data : undefined; + if ( + !isRecord(data) || + typeof data.journeyId !== "string" || + !Array.isArray(data.runIds) || + data.runIds.length > 0 + ) { + return null; + } + return { journeyId: data.journeyId, failed: failuresOf(data.failed) }; +} + +export function studioError({ status, payload }: StudioAnswer): StudioApiError { + const code = errorCode(payload); + const details = isRecord(payload) ? payload.details : undefined; + const missingScopes = status === 403 && isRecord(details) ? stringList(details.missing) : []; + const message = plain(failureMessage(status, code, payload, missingScopes)); + return new StudioApiError(message, { status, payload, code, missingScopes }); +} + +export function expectShape(ok: boolean, what: string, expected: string): void { + if (!ok) throw new StudioShapeError(what, expected); +} + +// 402 and 502 are told apart by status alone: the quota guard's body is not in the public spec, +// and a gateway's 502 has no Studio body at all. +function failureMessage( + status: number, + code: string | null, + payload: unknown, + missingScopes: string[], +): string { + if (status === 401) return "ora rejected the request (401). Check ORA_API_KEY."; + if (status === 402) return "Your ora plan has no runs left for this (402)."; + if (missingScopes.length > 0) return `The ora API key lacks ${missingScopes.join(", ")} (403).`; + const said = isRecord(payload) ? stringField(payload.message) || stringField(payload.error) : ""; + return `ora request failed (${status}${code ? ` ${code}` : ""})${said ? `: ${said}` : ""}`; +} + +function errorCode(payload: unknown): string | null { + if (!isRecord(payload)) return null; + if (typeof payload.code === "string") return payload.code; + const error = stringField(payload.error); + return /^[a-z][a-z0-9_]*$/.test(error) ? error : null; +} + +export function failuresOf(value: unknown): CreateFailure[] { + return Array.isArray(value) ? (value as CreateFailure[]) : []; +} + +export function stringList(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function stringField(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/api/studio-types.ts b/src/api/studio-types.ts new file mode 100644 index 0000000..f9ecf72 --- /dev/null +++ b/src/api/studio-types.ts @@ -0,0 +1,176 @@ +import type { JourneyStep } from "./platform"; + +// Wire shapes of the Studio endpoints a set of journeys calls on the ora platform, read from the +// public OpenAPI spec (`/studio/v1/*`). Where the spec is loose (the trajectory and insight bodies +// are `{}`, a create's 502 and the quota guard's 402 are not listed) the shape comes from the +// service code and is named where it is used, in ./studio.ts. + +/** + * Every scope a set of journeys needs, one per call: create (`journeys:write`), journey reads + * (`journeys:read`), trajectory and insight reads (`runs:read`), cancel (`runs:write`), and the + * agents and usage reads (`intents:read`). + */ +export const STUDIO_SCOPES = [ + "journeys:write", + "journeys:read", + "runs:read", + "runs:write", + "intents:read", +] as const; + +export type StudioScope = (typeof STUDIO_SCOPES)[number]; + +/** One agent (harness and model) the organization can run, from `GET /studio/v1/agents`. */ +export interface StudioAgent { + harness: string; + harnessName: string; + model: string; + modelName: string; + provider: string; + available: boolean; + /** Why the agent is unavailable (a plan that does not include it); null when available. */ + reason: string | null; +} + +export interface UsageMeter { + used: number; + /** Null when the plan sets no limit. */ + limit: number | null; + /** Null when the plan sets no limit. */ + remaining: number | null; +} + +/** This month's allowance, from `GET /studio/v1/usage`. */ +export interface StudioUsage { + runs: UsageMeter; + /** `used` is null when the plan sets no scan limit. */ + scans: { used: number | null; limit: number | null; remaining: number | null }; + /** ISO time the counts reset; null when unknown. */ + resetsAt: string | null; +} + +/** An explicit agent pair; ax always sends the catalog's model. */ +export interface StudioAgentPair { + harness: string; + model: string; +} + +export interface CreateJourneyInput { + goal: string; + agents: StudioAgentPair[]; + /** The site, never empty: an empty one falls back to the organization's tracked domain. */ + targetDomain: string; +} + +/** One agent a create could not start. `model` is sent by the service though the spec omits it. */ +export interface CreateFailure { + harness: string; + model?: string; + site?: string; + reason: string; +} + +/** A create that started at least one agent. `failed` is empty on a clean create. */ +export interface CreatedJourney { + journeyId: string; + runIds: string[]; + streamUrls: string[]; + failed: CreateFailure[]; +} + +/** Studio's 502 on create: no agent started, and the body still names the journey. */ +export interface NotStartedJourney { + journeyId: string; + failed: CreateFailure[]; +} + +export interface JourneyRunUsage { + durationMs: number | null; + totalTokens: number | null; + inputTokens: number | null; + cacheReadTokens: number | null; + cacheCreationTokens: number | null; + outputTokens: number | null; + numTurns: number | null; + costUsd: number | null; +} + +/** One agent's run inside a journey read. */ +export interface JourneyAgentRun { + runId: string; + harness: string; + model: string | null; + /** Raw run status; the set's outcome rules classify it. */ + status: string; + targetDomain: string | null; + provider: string | null; + executionKind: string | null; + intentText: string | null; + startedAt: string | null; + finishedAt: string | null; + createdAt: string; + usage: JourneyRunUsage; + /** Target pages check; ax reads none of it. */ + targetCheck: unknown; + /** Skill use; ax reads none of it. */ + skillUse: unknown; + /** + * `succeeded`, `partial`, `failed` or `not_measured` in the spec, null while the run is in + * flight. Typed as a string so a value the spec does not list is kept raw, never guessed. + */ + verdict: string | null; +} + +/** One journey and its per-agent runs, from `GET /studio/v1/journeys/{id}`. */ +export interface JourneyDetail { + id: string; + goal: string; + targetDomain: string | null; + status: string; + outcome: string | null; + competitors: string[]; + layer: string | null; + intentId: string | null; + intentTitle: string | null; + createdAt: string; + updatedAt: string; + agents: JourneyAgentRun[]; +} + +/** The answer of `POST /studio/v1/runs/cancel`; every requested id lands in one list. */ +export interface CancelResult { + requested: number; + canceled: string[]; + alreadyTerminal: string[]; + skipped: string[]; +} + +export type TrajectoryProfile = "raw" | "heuristic" | "full"; + +/** A run's normalized trajectory; the body carries more aggregates than the steps ax reads. */ +export interface StudioTrajectory { + steps: JourneyStep[]; +} + +export interface StudioClientOptions { + /** The ora_sk_ account key; falls back to $ORA_API_KEY. */ + apiKey?: string; + /** Platform API base; falls back to $ORA_PLATFORM_URL, then production. */ + baseUrl?: string; + /** Millisecond clock the bearer refresh reads; tests inject one. */ + now?: () => number; +} + +export interface StudioClient { + /** The scopes the key's bearer carries. */ + scopes(): Promise; + agents(): Promise; + usage(): Promise; + /** Any 2xx is a create that started at least one agent; every non-2xx throws StudioApiError. */ + createJourney(input: CreateJourneyInput): Promise; + journey(id: string): Promise; + trajectory(runId: string): Promise; + /** True once the insight reads 200, false while it is not ready; throws on anything else. */ + insightLanded(runId: string): Promise; + cancelRuns(runIds: readonly string[]): Promise; +} diff --git a/src/api/studio.test.ts b/src/api/studio.test.ts new file mode 100644 index 0000000..45684bb --- /dev/null +++ b/src/api/studio.test.ts @@ -0,0 +1,624 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import pkg from "../../package.json"; +import agents from "./__fixtures__/studio/agents.json"; +import cancel from "./__fixtures__/studio/cancel.json"; +import quota402 from "./__fixtures__/studio/error-402-quota.json"; +import missingScope403 from "./__fixtures__/studio/error-403-missing-scope.json"; +import created from "./__fixtures__/studio/journey-create.json"; +import createdPartial from "./__fixtures__/studio/journey-create-partial.json"; +import detailFinal from "./__fixtures__/studio/journey-detail-final.json"; +import detailRunning from "./__fixtures__/studio/journey-detail-running.json"; +import trajectory from "./__fixtures__/studio/trajectory.json"; +import usage from "./__fixtures__/studio/usage.json"; +import usageUnlimited from "./__fixtures__/studio/usage-unlimited.json"; +import { exchangeKeyGrant, PlatformApiError } from "./platform"; +import { + createStudioClient, + isStudioFailure, + notStartedJourney, + STUDIO_USER_AGENT, + StudioApiError, + StudioConnectionError, + StudioShapeError, + TRAJECTORY_PROFILE, +} from "./studio"; + +// Fixture provenance: shapes from oramono develop (deb82e2, 2026-11-13). Values are placeholders +// in those shapes; M9.1 replaces them with responses recorded from a live set. +// - agents.json: openapi/ora-public.openapi.json line 38025 (GET /studio/v1/agents). Ids and names +// from packages/node/harness-catalog/src/catalog.ts; the reason wording from +// apps/studio-service/src/plan/agent-availability.ts:20. +// - usage.json, usage-unlimited.json: spec line 52683 (GET /studio/v1/usage); the nulls of an +// uncapped plan from apps/studio-service/src/plan/studio-usage.ts (meterOf). +// - journey-create.json: spec line 49966 (the create's 200 body). The service answers 201 and +// leaves `failed` out of a clean create (apps/studio-service/src/journeys/journeys-router.ts:340). +// - journey-create-partial.json: the same body with `failed[]`, whose entries also carry `model` +// (FailedJourneyRun, apps/studio-service/src/journeys/journeys-service.ts:115). The reason is an +// example refusal message (packages/node/run-spine/src/harness-guard.ts, TIER_UNRESOLVED). +// - journey-detail-running.json, journey-detail-final.json: spec line 50081 (GET +// /studio/v1/journeys/{id}); journey status and outcome per +// apps/studio-service/src/journeys/journey-result.ts, a null verdict in flight per +// packages/node/journey-stages/src/run-verdict.ts:41. +// - cancel.json: spec line 51997. No envelope: runs-service answers the report bare +// (apps/runs-service/src/runs/routes/cancel-runs-route.ts) and Studio relays it +// (packages/node/runs-api/src/routes/cancel-runs-proxy-route.ts). +// - error-403-missing-scope.json: the scope guard's body +// (packages/node/auth/src/middleware/require-scope.ts:62 and :86), the spec's Error at line 575. +// - error-402-quota.json: the quota guard the create route mounts +// (packages/node/entitlements/src/middleware.ts:378, wired at +// apps/studio-service/src/wiring/journeys.ts:134); the public spec does not list a 402. +// - trajectory.json: oramono apps/studio-ui/src/lib/trajectory/__fixtures__/run-49ab0c88.trajectory.json +// trimmed to steps[].{id, turn, type, tool, url_host, status}. Studio serves the trajectory bare +// (packages/node/run-spine/src/trajectory-serve.ts). +// Bodies with no fixture file are named deltas below, each citing its source. + +const BASE = "https://platform.test"; +const KEY = "ora_sk_local"; + +/** A token shaped like the key exchange mints for an ora_sk_ key (auth-service api-key-grant.ts:105). */ +function jwt(claims: Record): string { + const part = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url"); + return [part({ alg: "RS256", typ: "JWT" }), part(claims), "signature"].join("."); +} + +const ALL_SCOPES = ["journeys:write", "journeys:read", "runs:read", "runs:write", "intents:read"]; +const TOKEN = jwt({ type: "api_key", sub: "key:k1", scopes: ALL_SCOPES, products: ["studio"] }); +const GRANT = { token: TOKEN, expiresIn: 900 }; + +const asJson = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + +interface Call { + url: string; + method: string; + headers: Record; + body?: string; +} + +/** + * Answers the key exchange with each grant in turn (the last one repeats) and every Studio call + * from `routes`, keyed by method and path. An unrouted call fails the test loudly. + */ +function studioMock( + routes: Record Response>, + grants: Array<{ token: string; expiresIn?: number }> = [GRANT], +) { + const calls: Call[] = []; + let exchanges = 0; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string | URL, init: RequestInit = {}) => { + const at = String(url); + const call: Call = { + url: at, + method: init.method ?? "GET", + headers: (init.headers ?? {}) as Record, + body: init.body as string | undefined, + }; + calls.push(call); + if (at.endsWith("/auth/token")) { + const grant = grants[Math.min(exchanges, grants.length - 1)]; + exchanges++; + return asJson({ ...grant, tokenType: "Bearer" }); + } + const route = routes[`${call.method} ${at.slice(BASE.length)}`]; + if (!route) throw new Error(`unrouted ${call.method} ${at}`); + return route(); + }), + ); + return { + calls, + studioCalls: () => calls.filter((c) => !c.url.endsWith("/auth/token")), + exchanges: () => exchanges, + }; +} + +const client = (now?: () => number) => createStudioClient({ apiKey: KEY, baseUrl: BASE, now }); + +const CREATE_INPUT = { + goal: "Find how to create an issue from the API. Use Linear.", + agents: [ + { harness: "claude-code", model: "claude-haiku-4-5-20251001" }, + { harness: "claude-code", model: "claude-sonnet-5-5" }, + ], + targetDomain: "linear.app", +}; + +// Studio's own 502 on create (journeys-router.ts:340): nothing started, success false, the journey +// id kept and every agent in failed[]. Delta on journey-create-partial.json. +const NONE_STARTED_502 = { + ...createdPartial, + success: false, + data: { + ...createdPartial.data, + runIds: [], + streamUrls: [], + failed: [ + { ...createdPartial.data.failed[0], model: "claude-haiku-4-5-20251001" }, + createdPartial.data.failed[0], + ], + }, +}; + +afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); +}); + +describe("exchangeKeyGrant", () => { + it("returns the token and its lifetime from the key exchange", async () => { + studioMock({}); + expect(await exchangeKeyGrant(BASE, KEY)).toEqual({ token: TOKEN, expiresIn: 900 }); + }); + + it("leaves expiresIn out when the answer carries none", async () => { + studioMock({}, [{ token: TOKEN }]); + expect(await exchangeKeyGrant(BASE, KEY)).toEqual({ token: TOKEN }); + }); +}); + +describe("createStudioClient", () => { + it("demands ORA_API_KEY up front", () => { + vi.stubEnv("ORA_API_KEY", ""); + expect(() => createStudioClient({ baseUrl: BASE })).toThrow(/ORA_API_KEY/); + }); + + it("exchanges the key once, then sends the bearer and the set's user agent to Studio", async () => { + const mock = studioMock({ + "GET /studio/v1/agents": () => asJson(agents), + "GET /studio/v1/usage": () => asJson(usage), + }); + const studio = client(); + await studio.agents(); + await studio.usage(); + + expect(mock.calls[0]).toMatchObject({ + url: `${BASE}/auth/token`, + method: "POST", + headers: { authorization: `Bearer ${KEY}` }, + }); + expect(mock.exchanges()).toBe(1); + expect(STUDIO_USER_AGENT).toBe(`ax/${pkg.version} (journey set)`); + for (const call of mock.studioCalls()) { + expect(call.headers.authorization).toBe(`Bearer ${TOKEN}`); + expect(call.headers["user-agent"]).toBe(STUDIO_USER_AGENT); + } + }); + + it("takes the base from ORA_PLATFORM_URL when none is passed", async () => { + vi.stubEnv("ORA_PLATFORM_URL", `${BASE}/`); + const mock = studioMock({ "GET /studio/v1/agents": () => asJson(agents) }); + await createStudioClient({ apiKey: KEY }).agents(); + expect(mock.studioCalls()[0].url).toBe(`${BASE}/studio/v1/agents`); + }); +}); + +describe("reads", () => { + it("agents() unwraps the envelope", async () => { + studioMock({ "GET /studio/v1/agents": () => asJson(agents) }); + expect(await client().agents()).toEqual(agents.data.agents); + }); + + it("usage() unwraps the envelope, limited and uncapped", async () => { + studioMock({ "GET /studio/v1/usage": () => asJson(usage) }); + expect(await client().usage()).toEqual(usage.data); + + studioMock({ "GET /studio/v1/usage": () => asJson(usageUnlimited) }); + const uncapped = await client().usage(); + expect(uncapped.runs).toEqual({ used: 212, limit: null, remaining: null }); + expect(uncapped.resetsAt).toBeNull(); + }); + + it("journey() reads one journey with its per-agent runs", async () => { + const id = detailRunning.data.id; + studioMock({ + [`GET /studio/v1/journeys/${id}`]: () => asJson(detailRunning), + }); + const journey = await client().journey(id); + expect(journey).toEqual(detailRunning.data); + expect(journey.agents.map((a) => [a.status, a.startedAt === null, a.verdict])).toEqual([ + ["running", false, null], + ["queued", true, null], + ]); + }); + + it("journey() keeps the final verdicts raw", async () => { + const id = detailFinal.data.id; + studioMock({ [`GET /studio/v1/journeys/${id}`]: () => asJson(detailFinal) }); + const journey = await client().journey(id); + expect(journey.agents.map((a) => a.verdict)).toEqual(["succeeded", "failed"]); + }); + + it("journey() encodes the id into the path", async () => { + const mock = studioMock({ "GET /studio/v1/journeys/a%2Fb": () => asJson(detailRunning) }); + await client().journey("a/b"); + expect(mock.studioCalls()[0].url).toBe(`${BASE}/studio/v1/journeys/a%2Fb`); + }); + + it("refuses an enveloped endpoint that answers without its envelope", async () => { + studioMock({ "GET /studio/v1/agents": () => asJson(agents.data) }); + await expect(client().agents()).rejects.toThrow(/envelope/); + }); + + it("trajectory() reads the bare body and keeps url_host per step", async () => { + const runId = detailFinal.data.agents[0].runId; + const mock = studioMock({ + [`GET /studio/v1/runs/${runId}/trajectory`]: () => asJson(trajectory), + }); + const read = await client().trajectory(runId); + expect(read.steps).toHaveLength(trajectory.steps.length); + expect(read.steps[3]).toEqual({ + id: 3, + turn: 4, + type: "tool_call", + tool: "WebFetch", + url_host: "ora.ai", + }); + expect(read.steps[13]).toMatchObject({ url_host: "ora.run", status: 308 }); + // No ?profile: the stored artifact already carries url_host on every profile. + expect(TRAJECTORY_PROFILE).toBeUndefined(); + expect(mock.studioCalls()[0].url).toBe(`${BASE}/studio/v1/runs/${runId}/trajectory`); + }); + + it("trajectory() refuses a body with no steps", async () => { + studioMock({ "GET /studio/v1/runs/r1/trajectory": () => asJson({ seq: 1 }) }); + await expect(client().trajectory("r1")).rejects.toThrow(/steps/); + }); +}); + +describe("createJourney", () => { + it("sends exactly goal, agents and targetDomain, and never domains", async () => { + const mock = studioMock({ "POST /studio/v1/journeys": () => asJson(created, 201) }); + await client().createJourney({ + ...CREATE_INPUT, + agents: CREATE_INPUT.agents.map((a) => ({ ...a, extra: "dropped" })), + }); + const call = mock.studioCalls()[0]; + expect(call.headers["content-type"]).toBe("application/json"); + const body = JSON.parse(call.body ?? "{}"); + expect(Object.keys(body).sort()).toEqual(["agents", "goal", "targetDomain"]); + expect(body).toEqual(CREATE_INPUT); + expect(body).not.toHaveProperty("domains"); + }); + + it("returns the created journey with an empty failed list on a clean 201", async () => { + studioMock({ "POST /studio/v1/journeys": () => asJson(created, 201) }); + expect(await client().createJourney(CREATE_INPUT)).toEqual({ ...created.data, failed: [] }); + }); + + it("accepts the 200 the spec lists", async () => { + studioMock({ "POST /studio/v1/journeys": () => asJson(created, 200) }); + expect((await client().createJourney(CREATE_INPUT)).journeyId).toBe(created.data.journeyId); + }); + + it("keeps a partial create's failed agents", async () => { + studioMock({ "POST /studio/v1/journeys": () => asJson(createdPartial, 201) }); + const journey = await client().createJourney(CREATE_INPUT); + expect(journey.runIds).toEqual(createdPartial.data.runIds); + expect(journey.failed).toEqual(createdPartial.data.failed); + }); + + it("refuses an empty targetDomain before any request", async () => { + const mock = studioMock({}); + await expect(client().createJourney({ ...CREATE_INPUT, targetDomain: " " })).rejects.toThrow( + /targetDomain/, + ); + expect(mock.calls).toHaveLength(0); + }); + + it("maps Studio's 502 to an error that still names the journey and its failed agents", async () => { + studioMock({ "POST /studio/v1/journeys": () => asJson(NONE_STARTED_502, 502) }); + const failure = await client() + .createJourney(CREATE_INPUT) + .catch((err: unknown) => err); + expect(failure).toBeInstanceOf(StudioApiError); + expect(failure).toMatchObject({ status: 502 }); + expect(notStartedJourney(failure)).toEqual({ + journeyId: NONE_STARTED_502.data.journeyId, + failed: NONE_STARTED_502.data.failed, + }); + }); + + it("recognises a gateway 502 by its status, with nothing to say what started", async () => { + studioMock({ + "POST /studio/v1/journeys": () => new Response("bad gateway", { status: 502 }), + }); + const failure = await client() + .createJourney(CREATE_INPUT) + .catch((err: unknown) => err); + expect(failure).toBeInstanceOf(StudioApiError); + expect(failure).toMatchObject({ status: 502, payload: null, code: null }); + expect(notStartedJourney(failure)).toBeNull(); + expect((failure as StudioApiError).gateway).toBe(true); + }); + + it("marks only a 502 without Studio's body as a gateway failure", async () => { + studioMock({ "POST /studio/v1/journeys": () => asJson(NONE_STARTED_502, 502) }); + const studio502 = await client() + .createJourney(CREATE_INPUT) + .catch((err: unknown) => err); + expect((studio502 as StudioApiError).gateway).toBe(false); + studioMock({ "POST /studio/v1/journeys": () => asJson({ error: "quota" }, 402) }); + const quota = await client() + .createJourney(CREATE_INPUT) + .catch((err: unknown) => err); + expect((quota as StudioApiError).gateway).toBe(false); + }); + + it("maps the quota guard's 402, and recognises a 402 by status alone", async () => { + studioMock({ "POST /studio/v1/journeys": () => asJson(quota402, 402) }); + const quota = await client() + .createJourney(CREATE_INPUT) + .catch((err: unknown) => err); + expect(quota).toBeInstanceOf(StudioApiError); + expect(quota).toMatchObject({ status: 402, code: "LIMIT_EXCEEDED", payload: quota402 }); + + studioMock({ "POST /studio/v1/journeys": () => new Response("", { status: 402 }) }); + await expect(client().createJourney(CREATE_INPUT)).rejects.toMatchObject({ + status: 402, + code: null, + }); + }); +}); + +describe("errors", () => { + it("a 403 names the scopes the key lacks", async () => { + studioMock({ "POST /studio/v1/journeys": () => asJson(missingScope403, 403) }); + const failure = await client() + .createJourney(CREATE_INPUT) + .catch((err: unknown) => err); + expect(failure).toBeInstanceOf(StudioApiError); + expect(failure).toBeInstanceOf(PlatformApiError); + expect(failure).toMatchObject({ + status: 403, + code: "INSUFFICIENT_SCOPE", + missingScopes: ["journeys:write"], + payload: missingScope403, + }); + expect((failure as Error).message).toContain("journeys:write"); + }); + + it("a 403 with no details lists no missing scopes", async () => { + const { details: _dropped, ...noDetails } = missingScope403; + studioMock({ "GET /studio/v1/agents": () => asJson(noDetails, 403) }); + await expect(client().agents()).rejects.toMatchObject({ status: 403, missingScopes: [] }); + }); + + it("a 401 on a Studio call points at the key", async () => { + studioMock({ + "GET /studio/v1/usage": () => asJson({ error: "Unauthorized", code: "UNAUTHORIZED" }, 401), + }); + const failure = await client() + .usage() + .catch((err: unknown) => err); + expect(failure).toBeInstanceOf(StudioApiError); + expect(failure).toMatchObject({ status: 401, code: "UNAUTHORIZED" }); + expect((failure as Error).message).toMatch(/ORA_API_KEY/); + }); + + it("strips terminal control characters from server text in the message", async () => { + studioMock({ + "GET /studio/v1/agents": () => + asJson({ error: "agents read failed\u001b[2J", code: "agents_read_failed" }, 502), + }); + const failure = await client() + .agents() + .catch((err: unknown) => err); + expect((failure as Error).message).not.toContain("\u001b"); + expect(failure).toMatchObject({ status: 502, code: "agents_read_failed" }); + }); + + it("a key the exchange rejects surfaces as the platform's 401", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("{}", { status: 401 })), + ); + await expect(client().agents()).rejects.toMatchObject({ status: 401 }); + await expect(client().agents()).rejects.toThrow(/API key/i); + }); +}); + +describe("transport failures", () => { + it("gives a Studio request a timeout", async () => { + studioMock({ "GET /studio/v1/agents": () => asJson(agents) }); + await client().agents(); + const [, init] = vi.mocked(fetch).mock.calls[1]; + expect(init?.signal).toBeInstanceOf(AbortSignal); + }); + + it("a request that cannot reach the platform is a connection error naming it", async () => { + studioMock({ + "GET /studio/v1/journeys/j-1": () => { + throw new TypeError("fetch failed"); + }, + }); + const failure = await client() + .journey("j-1") + .catch((err: unknown) => err); + expect(failure).toBeInstanceOf(StudioConnectionError); + expect((failure as Error).message).toContain(BASE); + expect((failure as Error).message).toContain("fetch failed"); + expect(isStudioFailure(failure)).toBe(true); + }); + + it("a timed out request is a connection error", async () => { + studioMock({ + "GET /studio/v1/usage": () => { + throw new DOMException("The operation was aborted due to timeout", "TimeoutError"); + }, + }); + await expect(client().usage()).rejects.toBeInstanceOf(StudioConnectionError); + }); + + it("a key exchange that cannot reach the platform is a connection error", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => { + throw new TypeError("fetch failed"); + }), + ); + const failure = await client() + .agents() + .catch((err: unknown) => err); + expect(failure).toBeInstanceOf(StudioConnectionError); + expect(isStudioFailure(failure)).toBe(true); + }); + + it("a key exchange that answers 200 without a token is a shape error", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => asJson({ tokenType: "Bearer" })), + ); + await expect(client().agents()).rejects.toBeInstanceOf(StudioShapeError); + }); + + it("an answer without the expected shape is a shape error", async () => { + studioMock({ "GET /studio/v1/journeys/j-1": () => asJson({ success: true, data: {} }) }); + const failure = await client() + .journey("j-1") + .catch((err: unknown) => err); + expect(failure).toBeInstanceOf(StudioShapeError); + expect((failure as Error).message).toMatch(/agents list/); + expect(isStudioFailure(failure)).toBe(true); + }); + + it("a gateway page answered with 200 is a shape error", async () => { + studioMock({ + "GET /studio/v1/journeys/j-1": () => + new Response("maintenance", { status: 200 }), + }); + await expect(client().journey("j-1")).rejects.toBeInstanceOf(StudioShapeError); + }); + + it("isStudioFailure names the platform's failures and nothing else", () => { + const http = new StudioApiError("x", { + status: 500, + payload: null, + code: null, + missingScopes: [], + }); + expect(isStudioFailure(http)).toBe(true); + expect(isStudioFailure(new PlatformApiError("x", { status: 401, payload: null }))).toBe(true); + expect(isStudioFailure(new TypeError("cannot read properties of undefined"))).toBe(false); + expect(isStudioFailure(new RangeError("x"))).toBe(false); + expect(isStudioFailure(new Error("x"))).toBe(false); + expect(isStudioFailure("x")).toBe(false); + }); +}); + +describe("insightLanded", () => { + const runId = detailFinal.data.agents[0].runId; + const path = `GET /studio/v1/runs/${runId}/insight`; + + it("is true once the insight reads 200", async () => { + // The insight body's schema is `{}` in the spec; only the status matters here. + studioMock({ [path]: () => asJson({}) }); + expect(await client().insightLanded(runId)).toBe(true); + }); + + it("is false on a 404 whose body says insight_not_ready", async () => { + // packages/node/run-spine/src/artifact-serve.ts:50, enriched with terminal and hint by + // packages/node/runs-api/src/routes/run-detail-proxy-routes.ts (the hint's dash written as a + // hyphen, per this repo's rule on dashes). + const notReady = { + error: "insight_not_ready", + terminal: true, + hint: "the run is terminal - this artifact was never produced and will not appear; see the run's outcome_reason", + }; + studioMock({ [path]: () => asJson(notReady, 404) }); + expect(await client().insightLanded(runId)).toBe(false); + }); + + it("throws on a 404 for a run Studio does not know", async () => { + // packages/node/run-spine/src/run-access.ts:75. + studioMock({ [path]: () => asJson({ error: "run_not_found" }, 404) }); + await expect(client().insightLanded(runId)).rejects.toMatchObject({ + status: 404, + code: "run_not_found", + }); + }); + + it("throws on any other status", async () => { + studioMock({ + [path]: () => asJson({ error: "Internal server error", code: "INTERNAL_ERROR" }, 500), + }); + await expect(client().insightLanded(runId)).rejects.toBeInstanceOf(StudioApiError); + }); +}); + +describe("cancelRuns", () => { + it("posts the run ids in one call and returns the bare report", async () => { + const mock = studioMock({ "POST /studio/v1/runs/cancel": () => asJson(cancel) }); + const runIds = [...cancel.canceled, ...cancel.alreadyTerminal, ...cancel.skipped]; + expect(await client().cancelRuns(runIds)).toEqual(cancel); + expect(JSON.parse(mock.studioCalls()[0].body ?? "{}")).toEqual({ runIds }); + }); + + it("refuses an empty list and more than 500 ids before any request", async () => { + const mock = studioMock({}); + await expect(client().cancelRuns([])).rejects.toThrow(/1 to 500/); + const tooMany = Array.from({ length: 501 }, (_, i) => `run-${i}`); + await expect(client().cancelRuns(tooMany)).rejects.toThrow(/1 to 500/); + expect(mock.calls).toHaveLength(0); + }); +}); + +describe("scopes and token refresh", () => { + it("scopes() reads the token's scopes claim", async () => { + studioMock({}, [{ token: jwt({ type: "api_key", scopes: ["runs:read"] }), expiresIn: 900 }]); + expect(await client().scopes()).toEqual(["runs:read"]); + }); + + it("scopes() reads an absent claim as no scopes, as the server does", async () => { + studioMock({}, [{ token: jwt({ type: "api_key" }), expiresIn: 900 }]); + expect(await client().scopes()).toEqual([]); + }); + + it("exchanges again once under 60 seconds of expiresIn remain", async () => { + const second = jwt({ type: "api_key", scopes: ALL_SCOPES, n: 2 }); + const mock = studioMock({ "GET /studio/v1/agents": () => asJson(agents) }, [ + GRANT, + { token: second, expiresIn: 900 }, + ]); + let clock = 1_000_000; + const studio = client(() => clock); + + await studio.agents(); + clock += (900 - 61) * 1000; + await studio.agents(); + expect(mock.exchanges()).toBe(1); + + clock += 2000; + await studio.agents(); + expect(mock.exchanges()).toBe(2); + expect(mock.studioCalls()[2].headers.authorization).toBe(`Bearer ${second}`); + }); + + it("exchanges again every 10 minutes when expiresIn is absent", async () => { + const mock = studioMock({ "GET /studio/v1/agents": () => asJson(agents) }, [{ token: TOKEN }]); + let clock = 0; + const studio = client(() => clock); + + await studio.agents(); + clock += 10 * 60_000 - 1; + await studio.agents(); + expect(mock.exchanges()).toBe(1); + + clock += 1; + await studio.agents(); + expect(mock.exchanges()).toBe(2); + }); + + it("shares one exchange between calls that start together", async () => { + const mock = studioMock({ + "GET /studio/v1/agents": () => asJson(agents), + "GET /studio/v1/usage": () => asJson(usage), + }); + const studio = client(); + await Promise.all([studio.agents(), studio.usage(), studio.scopes()]); + expect(mock.exchanges()).toBe(1); + }); +}); diff --git a/src/api/studio.ts b/src/api/studio.ts new file mode 100644 index 0000000..fa9b3c8 --- /dev/null +++ b/src/api/studio.ts @@ -0,0 +1,247 @@ +import pkg from "../../package.json"; +import { KeyDestinationError } from "./key-destination"; +import type { JourneyStep } from "./platform"; +import { exchangeKeyGrant, jsonOrNull, PlatformApiError, platformBase } from "./platform"; +import { + expectShape, + failuresOf, + isRecord, + type StudioAnswer, + StudioConnectionError, + StudioShapeError, + stringList, + studioError, +} from "./studio-errors"; +import type { + CancelResult, + JourneyDetail, + StudioAgent, + StudioClient, + StudioClientOptions, + StudioUsage, + TrajectoryProfile, +} from "./studio-types"; + +export { + isStudioFailure, + notStartedJourney, + StudioApiError, + StudioConnectionError, + StudioShapeError, +} from "./studio-errors"; + +// Client for the Studio endpoints a set of journeys calls on the ora platform: agents, usage, +// journey create and read, a run's trajectory and insight, and the batch cancel. It holds one +// bearer from platform.ts's key exchange and exchanges the key again shortly before the bearer +// expires, so a long follow never runs on a stale token. Every Studio request names itself as a +// set of journeys. + +export const STUDIO_USER_AGENT = `ax/${pkg.version} (journey set)`; + +/** Exchange the key again once less than this remains of the bearer's lifetime. */ +const REFRESH_MARGIN_MS = 60_000; +/** How long a bearer is used when the exchange gives no lifetime. */ +const UNDATED_TOKEN_MS = 10 * 60_000; +/** How long one Studio request may take; a create fans out every agent before it answers. */ +const REQUEST_TIMEOUT_MS = 60_000; +/** `POST /studio/v1/runs/cancel` takes 1 to 500 run ids. */ +const MAX_CANCEL_IDS = 500; +/** The insight read's 404 while the judge has not written it (run-spine artifact-serve.ts:21). */ +const INSIGHT_NOT_READY = "insight_not_ready"; + +/** + * The trajectory read sends no `?profile`, so Studio serves its stored artifact (studio-service + * config TRACE_PROFILE_DEFAULT, default `full`). Every profile carries `url_host` per step: + * trace-normalize stamps the host while ingesting each step, before attribution runs, and the + * step builder copies it out (packages/node/trace-normalize/src/classify.ts:250, + * trajectory-state.ts:230, builder/merge-step.ts:164). + */ +export const TRAJECTORY_PROFILE: TrajectoryProfile | undefined = undefined; + +export function createStudioClient(options: StudioClientOptions = {}): StudioClient { + const secret = options.apiKey || process.env.ORA_API_KEY; + if (!secret) { + throw new Error( + "Missing ORA_API_KEY. Set it in your environment or a local .env file (see .env.example).", + ); + } + const base = platformBase(options.baseUrl); + const bearer = tokenKeeper(base, secret, options.now ?? Date.now); + + // Only the network sits in each try, so any rejection there is the connection's, never a bug. + async function send(method: "GET" | "POST", path: string, body?: unknown): Promise { + const headers: Record = { + authorization: `Bearer ${await bearer()}`, + accept: "application/json", + "user-agent": STUDIO_USER_AGENT, + }; + if (body !== undefined) headers["content-type"] = "application/json"; + try { + const res = await fetch(`${base}${path}`, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + return { status: res.status, payload: jsonOrNull(await res.text()) }; + } catch (cause) { + throw new StudioConnectionError(base, cause); + } + } + + async function expectOk(method: "GET" | "POST", path: string, body?: unknown): Promise { + const answer = await send(method, path, body); + if (answer.status < 200 || answer.status > 299) throw studioError(answer); + return answer.payload; + } + + const runPath = (runId: string, leaf: string) => + `/studio/v1/runs/${encodeURIComponent(runId)}/${leaf}`; + + return { + scopes: async () => scopesClaim(await bearer()), + + async agents() { + const what = "GET /studio/v1/agents"; + const data = unwrap(await expectOk("GET", "/studio/v1/agents"), what); + expectShape(Array.isArray(data.agents), what, "an agents list"); + return data.agents as StudioAgent[]; + }, + + async usage() { + const what = "GET /studio/v1/usage"; + const data = unwrap(await expectOk("GET", "/studio/v1/usage"), what); + expectShape(isRecord(data.runs), what, "a runs meter"); + return data as unknown as StudioUsage; + }, + + async createJourney(input) { + if (!input.targetDomain.trim()) { + throw new Error( + "createJourney needs a targetDomain: an empty one runs against the account's tracked domain instead.", + ); + } + const what = "POST /studio/v1/journeys"; + const data = unwrap( + await expectOk("POST", "/studio/v1/journeys", { + goal: input.goal, + agents: input.agents.map(({ harness, model }) => ({ harness, model })), + targetDomain: input.targetDomain, + }), + what, + ); + expectShape( + typeof data.journeyId === "string" && Array.isArray(data.runIds), + what, + "a journey id and run ids", + ); + return { + journeyId: data.journeyId as string, + runIds: data.runIds as string[], + streamUrls: Array.isArray(data.streamUrls) ? (data.streamUrls as string[]) : [], + failed: failuresOf(data.failed), + }; + }, + + async journey(id) { + const path = `/studio/v1/journeys/${encodeURIComponent(id)}`; + const data = unwrap(await expectOk("GET", path), `GET ${path}`); + expectShape(Array.isArray(data.agents), `GET ${path}`, "an agents list"); + return data as unknown as JourneyDetail; + }, + + // Served bare, without the envelope (run-spine trajectory-serve.ts). + async trajectory(runId) { + const query = TRAJECTORY_PROFILE ? `?profile=${TRAJECTORY_PROFILE}` : ""; + const path = `${runPath(runId, "trajectory")}${query}`; + const body = await expectOk("GET", path); + expectShape(isRecord(body) && Array.isArray(body.steps), `GET ${path}`, "a steps list"); + return { steps: (body as { steps: JourneyStep[] }).steps }; + }, + + // A 404 is pending only with Studio's not-ready body; a run_not_found 404 is an error + // (runs-api run-detail-proxy-routes.ts, run-spine run-access.ts:75). + async insightLanded(runId) { + const answer = await send("GET", runPath(runId, "insight")); + if (answer.status === 200) return true; + if ( + answer.status === 404 && + isRecord(answer.payload) && + answer.payload.error === INSIGHT_NOT_READY + ) { + return false; + } + throw studioError(answer); + }, + + // The report comes back bare, relayed from runs-service (cancel-runs-route.ts). + async cancelRuns(runIds) { + if (runIds.length < 1 || runIds.length > MAX_CANCEL_IDS) { + throw new RangeError( + `cancelRuns takes 1 to ${MAX_CANCEL_IDS} run ids, got ${runIds.length}.`, + ); + } + const what = "POST /studio/v1/runs/cancel"; + const body = await expectOk("POST", "/studio/v1/runs/cancel", { runIds: [...runIds] }); + expectShape( + isRecord(body) && + typeof body.requested === "number" && + Array.isArray(body.canceled) && + Array.isArray(body.alreadyTerminal) && + Array.isArray(body.skipped), + what, + "a cancel report", + ); + return body as unknown as CancelResult; + }, + }; +} + +/** One bearer for the client, shared by calls that start together, exchanged again before expiry. */ +function tokenKeeper(base: string, secret: string, now: () => number): () => Promise { + type Held = { token: string; refreshAt: number }; + let held: Held | undefined; + let exchanging: Promise | undefined; + + const exchange = async (): Promise => { + const startedAt = now(); + const grant = await exchangeKeyGrant(base, secret).catch((cause: unknown) => { + // The shared exchange throws PlatformApiError on an HTTP error, lets fetch's rejection + // through, and throws a plain Error only for a 200 that carries no token. + if (cause instanceof PlatformApiError || cause instanceof KeyDestinationError) throw cause; + if (cause instanceof TypeError || cause instanceof DOMException) { + throw new StudioConnectionError(base, cause); + } + throw new StudioShapeError("POST /auth/token", "a token"); + }); + const usable = + grant.expiresIn === undefined ? UNDATED_TOKEN_MS : grant.expiresIn * 1000 - REFRESH_MARGIN_MS; + return { token: grant.token, refreshAt: startedAt + usable }; + }; + + return async () => { + if (held && now() < held.refreshAt) return held.token; + exchanging ??= exchange().finally(() => { + exchanging = undefined; + }); + held = await exchanging; + return held.token; + }; +} + +/** + * The bearer's `scopes` claim, decoded without verification: it only tells the preview what the + * key lacks, and Studio enforces scopes itself. Anything but a list reads as no scopes, as the + * server reads it (packages/node/auth/src/middleware/require-scope.ts:39). + */ +function scopesClaim(token: string): string[] { + const payload = token.split(".")[1] ?? ""; + const claims = jsonOrNull(Buffer.from(payload, "base64url").toString("utf8")); + return isRecord(claims) ? stringList(claims.scopes) : []; +} + +/** Studio's `{ success, data }` envelope, unwrapped in this one place. */ +function unwrap(payload: unknown, what: string): Record { + expectShape(isRecord(payload) && isRecord(payload.data), what, "its { success, data } envelope"); + return (payload as { data: Record }).data; +} diff --git a/src/commands/audit.test.ts b/src/commands/audit.test.ts index 899680b..533adfb 100644 --- a/src/commands/audit.test.ts +++ b/src/commands/audit.test.ts @@ -214,6 +214,111 @@ describe("auditCommand exit codes", () => { }); }); +describe("auditCommand Next lines and the --json error envelope", () => { + beforeEach(() => { + vi.spyOn(process.stdout, "write").mockImplementation(() => true); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation(() => {}); + }); + afterEach(() => vi.restoreAllMocks()); + + const stdout = () => + vi + .mocked(process.stdout.write) + .mock.calls.map((c) => String(c[0])) + .join(""); + const stderrLines = () => vi.mocked(console.error).mock.calls.map((c) => String(c[0])); + + it("--json prints the envelope on a usage error and exits 2", async () => { + expect(await run({ url: "not a domain" })).toBe(EXIT.USAGE); + expect(JSON.parse(stdout())).toEqual({ + error: { + code: "INVALID_TARGET", + message: 'Not a scannable URL or domain: "not a domain"', + next: "ax audit --json", + }, + }); + expect(stderrLines()).toContain( + "Next: ax audit --json # a public URL or domain, e.g. docs.example.com", + ); + }); + + it("--json carries ora's code on an API error, with a retry that never echoes the key", async () => { + vi.mocked(api.performAudit).mockRejectedValue( + new api.AuditApiError("ora rate limit exceeded", { code: "RATE_LIMITED" }), + ); + expect(await run({ apiKey: "sk_live_not_for_echo" })).toBe(EXIT.API); + const envelope = JSON.parse(stdout()); + expect(envelope.error.code).toBe("RATE_LIMITED"); + expect(envelope.error.next).toBe("ax audit example.com --json --api-key "); + expect(`${stdout()}${stderrLines().join("\n")}`).not.toContain("sk_live_not_for_echo"); + }); + + it("--json prints ora's error body unchanged on an API error, the notes on stderr", async () => { + const served = { + error: "usage_limit_reached", + kind: "scans", + period: "month", + used: 500, + limit: 500, + }; + vi.mocked(api.performAudit).mockRejectedValue( + new api.AuditApiError("ora usage limit reached: this API key has used its scans limit", { + payload: served, + }), + ); + expect(await run()).toBe(EXIT.API); + expect(stdout()).toBe(`${JSON.stringify(served, null, 2)}\n`); + expect(stderrLines()).toEqual([ + "Audit failed: ora usage limit reached: this API key has used its scans limit", + "Next: ax audit example.com --json # retry; if ora rate limited you, wait as long as the message says first", + ]); + }); + + it("--json keeps stdout the raw payload and puts the Next line on stderr", async () => { + resolveWith(); + expect(await run()).toBe(EXIT.OK); + expect(JSON.parse(stdout())).toEqual(FIXTURE); + expect(stderrLines()).toContain( + "Next: ax journey --domain example.com --json # send a real agent at the site on a task, then read its result", + ); + }); + + it("below --min-score, the Next line is a forced rescan through the same gate", async () => { + resolveWith(); + expect(await run({ minScore: "70" })).toBe(EXIT.BELOW_MIN_SCORE); + expect(stderrLines()).toContain( + "Next: ax audit example.com --json --force --min-score 70 # after you apply the Top fixes above, rescan", + ); + }); + + it("ends the human report with the Next line on stdout", async () => { + resolveWith(); + expect(await run({ json: false })).toBe(EXIT.OK); + expect(console.log).toHaveBeenLastCalledWith( + "Next: ax journey --domain example.com # send a real agent at the site on a task, then read its result", + ); + }); + + it("MCP_AUTH_REQUIRED keeps the raw frame on stdout and points at the site that serves it", async () => { + rejectWithMcpAuth(); + expect(await run({ url: "https://example.com/mcp" })).toBe(EXIT.OK); + expect(JSON.parse(stdout())).toEqual(MCP_AUTH_FRAME); + expect(stderrLines()).toContain( + "Next: ax audit example.com --json # ora cannot inspect an MCP server that needs credentials; score the site that serves it", + ); + }); + + it("MCP_AUTH_REQUIRED at a host root asks for the site rather than repeating the call", async () => { + rejectWithMcpAuth(); + expect(await run({ url: "api.example.com" })).toBe(EXIT.OK); + expect(stderrLines().find((line) => line.startsWith("Next:"))).toMatch( + /^Next: ax audit --json {2}#/, + ); + }); +}); + describe("auditCommand tunnels", () => { const TUNNEL_VARS = ["ORA_TUNNEL_CMD", "ORA_TUNNEL", "ORA_API_KEY"] as const; const saved: Partial> = {}; @@ -317,6 +422,22 @@ describe("auditCommand tunnels", () => { expect(api.performAudit).not.toHaveBeenCalled(); }); + it("--json passes the platform's error body through when --tunnel ora is refused", async () => { + process.env.ORA_API_KEY = "ora_sk_test"; + const served = { error: "insufficient_scope", required: ["tunnels:write"] }; + vi.mocked(oraTunnel.openOraTunnel).mockRejectedValue( + new tunnel.TunnelError("ora refused the tunnel (403): insufficient_scope", { + payload: served, + }), + ); + expect(await run({ url: "localhost:3000", tunnel: "ora" })).toBe(EXIT.USAGE); + const printed = vi + .mocked(process.stdout.write) + .mock.calls.map((c) => String(c[0])) + .join(""); + expect(printed).toBe(`${JSON.stringify(served, null, 2)}\n`); + }); + it("ORA_TUNNEL=ora selects the ora tunnel, and --tunnel-cmd wins over it", async () => { process.env.ORA_API_KEY = "ora_sk_test"; process.env.ORA_TUNNEL = "ora"; diff --git a/src/commands/audit.ts b/src/commands/audit.ts index a6cb7ae..a229089 100644 --- a/src/commands/audit.ts +++ b/src/commands/audit.ts @@ -1,8 +1,16 @@ import pc from "picocolors"; -import { AuditApiError, type AuditOutcome, isMcpAuthRequired, performAudit } from "../api/audit"; +import { + AuditApiError, + type AuditOutcome, + type AuditResult, + isMcpAuthRequired, + performAudit, +} from "../api/audit"; +import { type CommandError, reportError } from "../report/json-error"; import { toReport } from "../report/model"; +import { axCall, type NextStep, placeholder, printNext, type Word } from "../report/next-line"; import { MCP_AUTH_REQUIRED_NOTICE, renderReport } from "../report/terminal"; -import { isLocalTarget, openTunnel, type Tunnel } from "../tunnel"; +import { isLocalTarget, openTunnel, type Tunnel, TunnelError } from "../tunnel"; import { MISSING_KEY_HINT, openOraTunnel } from "../tunnel/ora"; import { spinner } from "../ui/spinner"; @@ -110,18 +118,117 @@ function mcpUrlOf(payload: unknown): string | undefined { return typeof url === "string" && url ? url : undefined; } +/** + * The tunnel flags to repeat in a follow-up call. A mode read from the + * environment needs none: the same environment selects it again. + */ +function tunnelWords(input: AuditCommandInput): string[] { + const flagCmd = input.tunnelCmd?.trim(); + if (flagCmd) return ["--tunnel-cmd", flagCmd]; + if (input.tunnel?.trim() === "ora") return ["--tunnel", "ora"]; + return []; +} + +/** The key flag as a placeholder when the user passed one: never echo the key itself. */ +function keyWords(input: AuditCommandInput): Word[] { + return input.apiKey?.trim() ? ["--api-key", placeholder("key")] : []; +} + +/** A caller reading JSON keeps reading JSON on the next call. */ +function jsonWords(input: AuditCommandInput): string[] { + return input.json ? ["--json"] : []; +} + +/** + * For an MCP server that needs credentials: score the site that serves it + * instead, its host (port kept) minus a leading `mcp.` label. A server at the + * root of a host with no such label would make that the call just run, so the + * site is left for the caller to name. `target` has already parsed as a URL + * in normalizeTarget. + */ +function mcpAuthNext(input: AuditCommandInput, target: string, useTunnel: boolean): NextStep { + const url = new URL(target.includes("://") ? target : `https://${target}`); + const site = url.host.replace(/^mcp\./, ""); + const atRoot = url.pathname.replace(/\/+$/, "") === ""; + const why = + "ora cannot inspect an MCP server that needs credentials; score the site that serves it"; + if (atRoot && site === url.host) { + return { call: axCall(["audit", placeholder("site domain"), ...jsonWords(input)]), why }; + } + return { + call: axCall(["audit", site, ...jsonWords(input), ...(useTunnel ? tunnelWords(input) : [])]), + why, + }; +} + +/** What to run after a result: the journey on a public site, otherwise a rescan once fixed. */ +function resultNext( + input: AuditCommandInput, + target: string, + result: AuditResult, + gate: number | undefined, + useTunnel: boolean, +): NextStep { + if (result.mcpAuthRequired) return mcpAuthNext(input, target, useTunnel); + // A tunnel host is gone once the audit ends, so a rescan reopens it; a public + // target is rescanned with --force so the cache cannot answer for the fix. + const rescan = [ + "audit", + target, + ...jsonWords(input), + ...(useTunnel ? tunnelWords(input) : ["--force"]), + ...(gate !== undefined ? ["--min-score", String(gate)] : []), + ...keyWords(input), + ]; + const isSite = !result.urlKind || result.urlKind === "domain"; + if (useTunnel || !isSite || (gate !== undefined && result.score < gate)) { + return { call: axCall(rescan), why: "after you apply the Top fixes above, rescan" }; + } + return { + call: axCall(["journey", "--domain", result.domain, ...jsonWords(input)]), + why: "send a real agent at the site on a task, then read its result", + }; +} + export async function auditCommand(input: AuditCommandInput): Promise { + const fail = ( + error: CommandError, + exit: number, + display?: (message: string) => string, + served?: unknown, + ): number => { + reportError(error, { json: input.json, display, served }); + return exit; + }; const target = normalizeTarget(input.url); if (!target) { - console.error(`Not a scannable URL or domain: ${JSON.stringify(input.url)}`); - return EXIT.USAGE; + return fail( + { + code: "INVALID_TARGET", + message: `Not a scannable URL or domain: ${JSON.stringify(input.url)}`, + next: { + call: axCall(["audit", placeholder("domain"), ...jsonWords(input)]), + why: "a public URL or domain, e.g. docs.example.com", + }, + }, + EXIT.USAGE, + ); } const minScore = parseIntFlag(input.minScore, "--min-score", 0, 100); const maxAge = parseIntFlag(input.maxAge, "--max-age", 0, 86_400); - for (const flag of [minScore, maxAge]) { - if (flag.error) { - console.error(flag.error); - return EXIT.USAGE; + for (const [flag, parsed, example] of [ + ["--min-score", minScore, "70"], + ["--max-age", maxAge, "3600"], + ] as const) { + if (parsed.error) { + return fail( + { + code: "INVALID_FLAG", + message: parsed.error, + next: { call: axCall(["audit", target, ...jsonWords(input), flag, example]) }, + }, + EXIT.USAGE, + ); } } @@ -131,30 +238,60 @@ export async function auditCommand(input: AuditCommandInput): Promise { // that prints a public https URL. Either way the result is stored as // ephemeral so the throwaway hostname never pollutes rankings. const tunnelMode = resolveTunnelMode(input); + const viaOraTunnel = axCall(["audit", target, ...jsonWords(input), "--tunnel", "ora"]); if ("error" in tunnelMode) { - console.error(tunnelMode.error); - return EXIT.USAGE; + return fail( + { code: "INVALID_FLAG", message: tunnelMode.error, next: { call: viaOraTunnel } }, + EXIT.USAGE, + ); } const useTunnel = tunnelMode.kind !== "none"; if (!useTunnel && isLocalTarget(target)) { - console.error( - [ - `${target} only exists on this machine, and ora audits public URLs.`, - "Either audit a publicly reachable deployment of this site (e.g. a preview URL),", - "or run it through a tunnel:", - " ax audit localhost:3000 --tunnel ora # ora's own tunnel; needs ORA_API_KEY", - " ax audit localhost:3000 --tunnel-cmd 'ora tunnel 3000 --access public'", - " ax audit localhost:3000 --tunnel-cmd 'ngrok http 3000 --log stdout'", - "Any command that prints a public https URL works; the result is stored as", - "ephemeral (excluded from rankings, deleted after 7 days).", - ].join("\n"), + return fail( + { + code: "LOCAL_TARGET", + message: [ + `${target} only exists on this machine, and ora audits public URLs.`, + "Either audit a publicly reachable deployment of this site (e.g. a preview URL),", + "or run it through a tunnel:", + " ax audit localhost:3000 --tunnel ora # ora's own tunnel; needs ORA_API_KEY", + " ax audit localhost:3000 --tunnel-cmd 'ora tunnel 3000 --access public'", + " ax audit localhost:3000 --tunnel-cmd 'ngrok http 3000 --log stdout'", + "Any command that prints a public https URL works; the result is stored as", + "ephemeral (excluded from rankings, deleted after 7 days).", + ].join("\n"), + next: { call: viaOraTunnel, why: "ora's own tunnel; needs ORA_API_KEY" }, + }, + EXIT.USAGE, ); - return EXIT.USAGE; } if (tunnelMode.kind === "ora" && !process.env.ORA_API_KEY) { - console.error(MISSING_KEY_HINT); - return EXIT.USAGE; + return fail( + { + code: "MISSING_API_KEY", + message: MISSING_KEY_HINT, + next: { + call: `ORA_API_KEY= ${viaOraTunnel}`, + why: "a key with the tunnels:write + tunnels:connect scopes", + }, + }, + EXIT.USAGE, + ); } + // The same call again, for a retry after a failure that may clear. + const sameCall: NextStep = { + call: axCall([ + "audit", + target, + ...jsonWords(input), + ...tunnelWords(input), + ...(minScore.value !== undefined ? ["--min-score", String(minScore.value)] : []), + ...(maxAge.value !== undefined ? ["--max-age", String(maxAge.value)] : []), + ...(input.force ? ["--force"] : []), + ...keyWords(input), + ]), + why: "retry; if ora rate limited you, wait as long as the message says first", + }; const interactive = !input.json; if (interactive) spinner.start(`Auditing ${target} with ora`); @@ -195,8 +332,16 @@ export async function auditCommand(input: AuditCommandInput): Promise { spinner.stop(); process.removeListener("SIGINT", onSignal); process.removeListener("SIGTERM", onSignal); - console.error(cause instanceof Error ? cause.message : String(cause)); - return EXIT.USAGE; + return fail( + { + code: "TUNNEL_FAILED", + message: cause instanceof Error ? cause.message : String(cause), + next: { call: sameCall.call, why: "retry, or bring your own tunnel with --tunnel-cmd" }, + }, + EXIT.USAGE, + undefined, + cause instanceof TunnelError ? cause.payload : undefined, + ); } auditTarget = tunnel.url; } @@ -221,6 +366,7 @@ export async function auditCommand(input: AuditCommandInput): Promise { // credentials, which is the same unscored state the legacy marker // describes - report it and exit 0. if (isMcpAuthRequired(cause)) { + const next = mcpAuthNext(input, target, useTunnel); if (input.json) { // Raw passthrough: the error object exactly as ora served it. An // AuditApiError raised without one (the class is public) still has to @@ -238,10 +384,21 @@ export async function auditCommand(input: AuditCommandInput): Promise { console.log(""); } if (minScore.value !== undefined) process.stderr.write(MIN_SCORE_SKIPPED_NOTE); + printNext(next, { stderr: input.json }); return EXIT.OK; } - console.error(`Audit failed: ${cause instanceof Error ? cause.message : String(cause)}`); - return cause instanceof AuditApiError ? EXIT.API : EXIT.USAGE; + const message = cause instanceof Error ? cause.message : String(cause); + const apiError = cause instanceof AuditApiError; + return fail( + { + code: apiError ? cause.code || "API_ERROR" : "AUDIT_FAILED", + message, + next: sameCall, + }, + apiError ? EXIT.API : EXIT.USAGE, + (cleaned) => `Audit failed: ${cleaned}`, + apiError ? cause.payload : undefined, + ); } finally { await closeTunnel(); process.removeListener("SIGINT", onSignal); @@ -262,20 +419,24 @@ export async function auditCommand(input: AuditCommandInput): Promise { console.log(line); } } + const next = resultNext(input, target, result, minScore.value, useTunnel); if (minScore.value !== undefined) { // Legacy marker on a stored result (older servers, and score polls that // still return a marked row): unscored for the same reason. if (result.mcpAuthRequired) { process.stderr.write(MIN_SCORE_SKIPPED_NOTE); + printNext(next, { stderr: input.json }); return EXIT.OK; } if (result.score < minScore.value) { if (!input.json) { console.log(` Score ${result.score} is below the required minimum ${minScore.value}\n`); } + printNext(next, { stderr: input.json }); return EXIT.BELOW_MIN_SCORE; } } + printNext(next, { stderr: input.json }); return EXIT.OK; } diff --git a/src/commands/deep-journey.test.ts b/src/commands/deep-journey.test.ts index 444e5b1..0eb4e9e 100644 --- a/src/commands/deep-journey.test.ts +++ b/src/commands/deep-journey.test.ts @@ -42,10 +42,22 @@ vi.mock("../api/deep-journey", async (importOriginal) => { }; }); -import { fetchJourneyIntents, performDeepJourney } from "../api/deep-journey"; +import realDetail from "../api/__fixtures__/deep-journey-detail.json"; +import runningDetail from "../api/__fixtures__/deep-journey-detail-running.json"; +import refusals from "../api/__fixtures__/deep-journey-refusals.json"; +import { + DeepJourneyApiError, + type DeepJourneyOutcome, + fetchJourneyIntents, + performDeepJourney, +} from "../api/deep-journey"; +import type { JourneyNextAction } from "../contract"; const BASE = { url: "zapier.com", json: true, noStream: false }; +const KEYLESS_HINT = + "Repeat this goal across agents and levels of help with `ax journey --levels`; needs a Front API key."; + describe("deepJourneyCommand - keyed tier flags", () => { beforeEach(() => { vi.clearAllMocks(); @@ -53,6 +65,7 @@ describe("deepJourneyCommand - keyed tier flags", () => { vi.stubEnv("ORA_SCAN_API_KEY", ""); vi.spyOn(console, "error").mockImplementation(() => {}); vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); }); afterEach(() => { vi.unstubAllEnvs(); @@ -290,3 +303,407 @@ describe("deepJourneyCommand - keyed tier flags", () => { expect(out).not.toContain("intent signup"); }); }); + +describe("deepJourneyCommand - Next lines and the --json error envelope", () => { + const writes: string[] = []; + beforeEach(() => { + vi.clearAllMocks(); + writes.length = 0; + vi.stubEnv("ORA_PARTNER_API_KEY", ""); + vi.stubEnv("ORA_SCAN_API_KEY", ""); + vi.stubEnv("ORA_API_KEY", ""); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation((line?: unknown) => { + writes.push(String(line ?? "")); + }); + vi.spyOn(process.stdout, "write").mockImplementation((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + const stderrLines = () => vi.mocked(console.error).mock.calls.map((c) => String(c[0])); + const envelope = () => JSON.parse(writes.join("")); + + it("--json prints the envelope for --task without a key, naming the key in the next call", async () => { + expect(await deepJourneyCommand({ ...BASE, task: "Find the pricing page" })).toBe(EXIT.USAGE); + expect(envelope().error).toEqual({ + code: "NEEDS_KEY", + message: expect.stringContaining("--task needs an ora partner API key"), + next: "ORA_PARTNER_API_KEY= ax journey 'Find the pricing page' --domain zapier.com --json", + }); + }); + + it("an unknown intent suggests the default one", async () => { + expect(await deepJourneyCommand({ ...BASE, intent: "nope" })).toBe(EXIT.USAGE); + expect(envelope().error.code).toBe("UNKNOWN_INTENT"); + expect(envelope().error.next).toBe("ax journey --domain zapier.com --intent pricing --json"); + }); + + it("a run still executing is read, never started again", async () => { + vi.mocked(performDeepJourney).mockRejectedValueOnce( + new DeepJourneyApiError("run run_1 is still executing; the stream ended", { + code: "RUN_STILL_RUNNING", + runUrl: "https://ora.ai/api/journey/runs/run_1", + }), + ); + expect(await deepJourneyCommand(BASE)).toBe(EXIT.API); + expect(envelope().error).toEqual({ + code: "RUN_STILL_RUNNING", + message: expect.stringContaining("still executing"), + next: "curl -s https://ora.ai/api/journey/runs/run_1", + }); + }); + + it("a retry after an API error repeats the run with the key as a placeholder", async () => { + vi.mocked(performDeepJourney).mockRejectedValueOnce( + new DeepJourneyApiError("ora's run engine is unavailable right now", { + code: "ENGINE_UNAVAILABLE", + }), + ); + expect( + await deepJourneyCommand({ ...BASE, intent: "pricing", apiKey: "pk_live_not_for_echo" }), + ).toBe(EXIT.API); + expect(envelope().error.next).toBe( + "ax journey --domain zapier.com --intent pricing --agent cas-haiku --json --api-key ", + ); + expect(`${writes.join("")}${stderrLines().join("\n")}`).not.toContain("pk_live_not_for_echo"); + }); + + it("--json prints ora's error body unchanged and puts the message and Next line on stderr", async () => { + const served = { + error: "usage_limit_reached", + kind: "journey", + period: "month", + used: 1000, + limit: 1000, + resets_at: "2026-12-01T00:00:00Z", + next_action: { call: "ax deep-journey zapier.com" }, + }; + vi.mocked(performDeepJourney).mockRejectedValueOnce( + new DeepJourneyApiError("ora usage limit reached: this API key has used its journey limit", { + code: "USAGE_LIMIT_REACHED", + payload: served, + }), + ); + expect(await deepJourneyCommand(BASE)).toBe(EXIT.API); + expect(writes).toEqual([`${JSON.stringify(served, null, 2)}\n`]); + expect(stderrLines()).toEqual([ + expect.stringContaining("ora usage limit reached"), + "Next: ax journey --domain zapier.com --intent pricing --agent cas-haiku --json # after the wait the message names", + ]); + }); + + it("--json passes through ora's body when the roster fetch fails too", async () => { + const served = { error: "rate_limited", retry_after_ms: 30_000 }; + vi.mocked(fetchJourneyIntents).mockRejectedValueOnce( + new DeepJourneyApiError("ora request failed: rate_limited", { payload: served }), + ); + expect(await deepJourneyCommand(BASE)).toBe(EXIT.API); + expect(envelope()).toEqual(served); + }); + + it("--json keeps stdout the run detail and puts the Next line on stderr", async () => { + expect(await deepJourneyCommand(BASE)).toBe(EXIT.OK); + expect(envelope()).toMatchObject({ id: "run_1", status: "succeeded" }); + expect(stderrLines()).toContain( + "Next: ax audit zapier.com --json # the readiness audit: which checks explain what the agent hit", + ); + }); + + it("ends the human result with the Next line on stdout", async () => { + expect(await deepJourneyCommand({ ...BASE, json: false, noStream: true })).toBe(EXIT.OK); + // The keyless hint is the one dim line after Next (see the set describe). + expect(writes.at(-2)).toBe( + "Next: ax audit zapier.com # the readiness audit: which checks explain what the agent hit", + ); + expect(writes.at(-1)).toContain(KEYLESS_HINT); + }); +}); + +// Real 1.42.0 responses (provenance in src/api/deep-journey.test.ts). No local +// run is flagged, so the raised flag is a delta carrying the reason ora writes +// for a first request refused with a 403. +describe("deepJourneyCommand - flags and the next action ora served", () => { + const writes: string[] = []; + const BLOCKED_REASON = "The site refused the agent's first request with HTTP 403"; + const blockedDetail = { + ...realDetail, + flags: { ...realDetail.flags, blocked: { flagged: true, reason: BLOCKED_REASON } }, + }; + const STREAM_NEXT = runningDetail.next_action as JourneyNextAction; + const STREAM_LINE = `Next: curl -s https://ora.ai${runningDetail.stream_url} # ${STREAM_NEXT.reason}`; + const AUDIT_LINE = + "Next: ax audit linear.app --json # the readiness audit: which checks explain what the agent hit"; + const outcomeWith = (detail: object) => + ({ + record: { id: realDetail.id, status: "succeeded" }, + cached: false, + allowance: {}, + detail, + }) as unknown as DeepJourneyOutcome; + const RUN = { url: "linear.app", intent: "pricing", json: true, noStream: false }; + + beforeEach(() => { + vi.clearAllMocks(); + writes.length = 0; + vi.stubEnv("ORA_PARTNER_API_KEY", ""); + vi.stubEnv("ORA_SCAN_API_KEY", ""); + vi.stubEnv("ORA_API_KEY", ""); + vi.stubEnv("ORA_API_URL", "https://ora.ai"); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation((line?: unknown) => { + writes.push(String(line ?? "")); + }); + vi.spyOn(process.stdout, "write").mockImplementation((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + const stderrLines = () => vi.mocked(console.error).mock.calls.map((c) => String(c[0])); + const stdoutJson = () => JSON.parse(writes.join("")); + + it("prints a raised flag in ora's words, right under the verdict", async () => { + vi.mocked(performDeepJourney).mockResolvedValueOnce(outcomeWith(blockedDetail)); + expect(await deepJourneyCommand({ ...RUN, json: false, noStream: true })).toBe(EXIT.OK); + const flag = writes.findIndex((line) => line.includes(BLOCKED_REASON)); + expect(writes[flag]).toMatch(/!.* The site refused the agent's first request with HTTP 403$/); + const verdict = writes.findIndex((line) => line.includes("task satisfied")); + expect(flag).toBeGreaterThan(verdict); + expect(flag - verdict).toBeLessThanOrEqual(2); + }); + + it("--json keeps ora's detail unchanged on stdout and puts a raised flag on stderr", async () => { + vi.mocked(performDeepJourney).mockResolvedValueOnce(outcomeWith(blockedDetail)); + expect(await deepJourneyCommand(RUN)).toBe(EXIT.OK); + expect(stdoutJson()).toEqual(blockedDetail); + expect(stderrLines()).toEqual([ + expect.stringMatching(/!.* The site refused the agent's first request with HTTP 403$/), + AUDIT_LINE, + expect.stringContaining(KEYLESS_HINT), + ]); + }); + + it("prints no note when ora raised no flag", async () => { + vi.mocked(performDeepJourney).mockResolvedValueOnce(outcomeWith(realDetail)); + expect(await deepJourneyCommand(RUN)).toBe(EXIT.OK); + expect(stdoutJson()).toEqual(realDetail); + expect(stderrLines()).toEqual([AUDIT_LINE, expect.stringContaining(KEYLESS_HINT)]); + }); + + it("a run left running ends on the stream ora named, --no-stream included", async () => { + vi.mocked(performDeepJourney).mockRejectedValueOnce( + new DeepJourneyApiError(`run ${realDetail.id} is still executing`, { + code: "RUN_STILL_RUNNING", + runUrl: `https://ora.ai/api/journey/runs/${realDetail.id}`, + nextAction: STREAM_NEXT, + }), + ); + expect(await deepJourneyCommand({ ...RUN, json: false, noStream: true })).toBe(EXIT.API); + expect(stderrLines().at(-1)).toBe(STREAM_LINE); + }); + + it("a failed run ends on the stream ora named for it", async () => { + vi.mocked(performDeepJourney).mockResolvedValueOnce( + outcomeWith({ ...runningDetail, status: "failed" }), + ); + expect(await deepJourneyCommand(RUN)).toBe(EXIT.RUN_FAILED); + expect(stderrLines().at(-1)).toBe(STREAM_LINE); + }); + + it("a refusal that asks for a new run ends on the same ax call, with ora's reason", async () => { + vi.mocked(performDeepJourney).mockRejectedValueOnce( + new DeepJourneyApiError("ora journey request failed: Invalid input", { + payload: refusals.INVALID_DOMAIN, + }), + ); + expect(await deepJourneyCommand({ ...RUN, url: "localhost" })).toBe(EXIT.API); + expect(stdoutJson()).toEqual(refusals.INVALID_DOMAIN); + expect(stderrLines().at(-1)).toBe( + `Next: ax journey --domain localhost --intent pricing --agent cas-haiku --json # ${refusals.INVALID_DOMAIN.next_action.reason}`, + ); + }); + + it("a refusal that names another request writes it as curl", async () => { + vi.mocked(performDeepJourney).mockRejectedValueOnce( + new DeepJourneyApiError("ora journey request failed: Unsupported agent", { + payload: refusals.UNSUPPORTED_AGENT, + }), + ); + expect(await deepJourneyCommand(RUN)).toBe(EXIT.API); + expect(stderrLines().at(-1)).toBe( + `Next: curl -s https://ora.ai/api/journey/agents # ${refusals.UNSUPPORTED_AGENT.next_action.reason}`, + ); + }); +}); + +describe("deepJourneyCommand - the set Next line after a free run", () => { + const writes: string[] = []; + const NAMING_LINE = + "This goal names Zapier, so --levels runs it at the brand level. Add `--exact ` for the exact level, or write the goal without Zapier for the need level."; + const AUDIT_LINE = + "Next: ax audit zapier.com # the readiness audit: which checks explain what the agent hit"; + const setNext = (goal: string) => + `Next: ax journey '${goal}' --domain zapier.com --levels # the same goal at each level of help, with the verdicts counted`; + const PLAIN_GOAL = "Find the pricing page."; + const withTemplate = (template: string, hint = "find pricing") => + vi.mocked(fetchJourneyIntents).mockResolvedValueOnce({ + intents: [{ id: "pricing", label: "Pricing", hint, template }], + defaultId: "pricing", + }); + + beforeEach(() => { + vi.clearAllMocks(); + writes.length = 0; + vi.stubEnv("ORA_PARTNER_API_KEY", ""); + vi.stubEnv("ORA_SCAN_API_KEY", ""); + vi.stubEnv("ORA_API_KEY", ""); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation((line?: unknown) => { + writes.push(String(line ?? "")); + }); + vi.spyOn(process.stdout, "write").mockImplementation((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + const stderrLines = () => vi.mocked(console.error).mock.calls.map((c) => String(c[0])); + const human = { ...BASE, json: false, noStream: true }; + + it("with a Front key and a curated goal that names the site, points at a set and adds the levels line", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + expect(await deepJourneyCommand(human)).toBe(EXIT.OK); + expect(writes.slice(-2)).toEqual([setNext("Find pricing for zapier.com."), NAMING_LINE]); + expect(writes.join("\n")).not.toContain("ax audit"); + expect(writes.join("\n")).not.toContain(KEYLESS_HINT); + }); + + it("with a Front key and a --task that does not name the site, prints Next and no second line", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + const task = "Find how to build a Slack to Sheets flow"; + expect(await deepJourneyCommand({ ...human, task, apiKey: "pk_live_demo_0123456789" })).toBe( + EXIT.OK, + ); + expect(writes.at(-1)).toBe(setNext(task)); + expect(stderrLines().join("\n")).not.toContain("names Zapier"); + }); + + it("reduces a URL target to the set's site, so the brand and the call match what the set runs", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + expect(await deepJourneyCommand({ ...human, url: "https://www.zapier.com/pricing" })).toBe( + EXIT.OK, + ); + expect(writes.at(-2)).toMatch(/^Next: ax journey '.+' --domain zapier\.com --levels /); + expect(writes.at(-1)).toBe(NAMING_LINE); + }); + + it("offers no set for a target a set would refuse, and keeps the audit Next", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + expect(await deepJourneyCommand({ ...human, url: "localhost" })).toBe(EXIT.OK); + expect(writes.at(-1)).toMatch(/^Next: ax audit localhost /); + expect(writes.join("\n")).not.toContain("--levels"); + }); + + it("never adds --agents to the set call, and quotes the goal as one shell word", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + withTemplate(PLAIN_GOAL); + expect(await deepJourneyCommand(human)).toBe(EXIT.OK); + expect(writes.at(-1)).toBe(setNext(PLAIN_GOAL)); + expect(writes.at(-1)).not.toContain("--agents"); + }); + + it("without a Front key, keeps the audit Next and adds the dim hint after it", async () => { + withTemplate(PLAIN_GOAL); + expect(await deepJourneyCommand(human)).toBe(EXIT.OK); + expect(writes.slice(-2)).toEqual([AUDIT_LINE, expect.stringContaining(KEYLESS_HINT)]); + expect(writes.join("\n")).not.toContain("--domain zapier.com --levels"); + }); + + it("a --task run with a Front key never gets the hint", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + await deepJourneyCommand({ ...human, task: PLAIN_GOAL, apiKey: "pk_live_demo_0123456789" }); + expect(writes.join("\n")).not.toContain("needs a Front API key"); + }); + + it("a goal that fell back to the intent hint keeps the audit Next and prints no set lines", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + withTemplate("", "find pricing"); + expect(await deepJourneyCommand(human)).toBe(EXIT.OK); + expect(writes.at(-1)).toBe(AUDIT_LINE); + expect(writes.join("\n")).not.toContain("--levels"); + }); + + it("a fallback goal prints no keyless hint either", async () => { + withTemplate("", "find pricing"); + expect(await deepJourneyCommand(human)).toBe(EXIT.OK); + expect(writes.at(-1)).toBe(AUDIT_LINE); + expect(writes.join("\n")).not.toContain(KEYLESS_HINT); + }); + + it("--json puts the Next line and the levels line on stderr and keeps stdout the run detail", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + expect(await deepJourneyCommand(BASE)).toBe(EXIT.OK); + expect(JSON.parse(writes.join(""))).toMatchObject({ id: "run_1" }); + expect(stderrLines()).toEqual([setNext("Find pricing for zapier.com."), NAMING_LINE]); + }); + + it("--json puts the keyless hint on stderr", async () => { + withTemplate(PLAIN_GOAL); + expect(await deepJourneyCommand(BASE)).toBe(EXIT.OK); + expect(JSON.parse(writes.join(""))).toMatchObject({ id: "run_1" }); + expect(stderrLines()).toEqual([ + "Next: ax audit zapier.com --json # the readiness audit: which checks explain what the agent hit", + expect.stringContaining(KEYLESS_HINT), + ]); + }); + + it("a failed run keeps its own Next line and gets no set lines", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + vi.mocked(performDeepJourney).mockResolvedValueOnce({ + record: { id: "run_1", status: "failed" }, + cached: false, + allowance: {}, + detail: { id: "run_1", status: "failed", domain: "zapier.com", contractVersion: "1.15.0" }, + } as unknown as DeepJourneyOutcome); + expect(await deepJourneyCommand(BASE)).toBe(EXIT.RUN_FAILED); + expect(stderrLines().at(-1)).toContain("Next: ax journey --domain zapier.com"); + expect(stderrLines().join("\n")).not.toContain("--levels"); + }); + + it("a served next action keeps its Next line and gets no set lines", async () => { + vi.stubEnv("ORA_API_KEY", "ora_sk_front_key"); + vi.mocked(performDeepJourney).mockResolvedValueOnce({ + record: { id: "run_1", status: "succeeded" }, + cached: false, + allowance: {}, + detail: { + id: "run_1", + status: "succeeded", + domain: "zapier.com", + contractVersion: "1.15.0", + next_action: { + method: "GET", + url: "/api/journey/runs/run_1", + reason: "read the run again", + }, + }, + } as unknown as DeepJourneyOutcome); + expect(await deepJourneyCommand(BASE)).toBe(EXIT.OK); + expect(stderrLines().at(-1)).toContain("curl -s"); + expect(stderrLines().join("\n")).not.toContain("--levels"); + }); +}); diff --git a/src/commands/deep-journey.ts b/src/commands/deep-journey.ts index c32c94c..f6cb081 100644 --- a/src/commands/deep-journey.ts +++ b/src/commands/deep-journey.ts @@ -1,23 +1,44 @@ import pc from "picocolors"; import { + apiBase, DeepJourneyApiError, type DeepJourneyOutcome, fetchJourneyAgents, fetchJourneyIntents, formatWait, + JOURNEY_RUNS_PATH, type JourneyAgentOption, performDeepJourney, + servedNextAction, } from "../api/deep-journey"; import type { JourneyStep } from "../api/platform"; -import type { JourneyRunResult, JourneyTrajectoryStep } from "../contract"; +import type { + JourneyNextAction, + JourneyRunDetail, + JourneyRunResult, + JourneyTrajectoryStep, +} from "../contract"; +import { defaultBrand, namesSite } from "../journey-set/levels"; +import { parseSetSite } from "../journey-set/spec"; +import { type CommandError, reportError } from "../report/json-error"; +import { + axCall, + type NextStep, + placeholder, + printNext, + servedNextStep, + shellWord, + type Word, +} from "../report/next-line"; import { flow, plain, stdoutWidth } from "../ui/ansi"; import { buildJourneyTree, drawJourneyTree, journeySummary } from "../ui/graph"; import { LivePanel } from "../ui/panel"; import { spinner } from "../ui/spinner"; -// `ax deep-journey `: run a real agent at a site through ora's PUBLIC -// journey API (no key, no workspace - unlike `ax journey`, which drives the -// authenticated platform). Thin client: verdict, step_count, and insight are +// `ax journey --domain ` with no goal of its own (and the older +// `ax deep-journey `): run a real agent at a site through ora's PUBLIC +// journey API (no key, no workspace - unlike `ax journey ""`, which +// drives the authenticated platform). Thin client: verdict, step_count, and insight are // rendered exactly as the contract delivers them; nothing is re-judged here. export const EXIT = { OK: 0, RUN_FAILED: 1, USAGE: 2, API: 3 } as const; @@ -54,11 +75,77 @@ export interface DeepJourneyCommandInput { agent?: string; json: boolean; noStream: boolean; + /** The command typed: `ax journey` with no goal of its own, or the older `ax deep-journey`. */ + via?: "journey" | "deep-journey"; } -function fail(message: string): typeof EXIT.API { - console.error(pc.red(`ax deep-journey: ${message}`)); - return EXIT.API; +const SET_KEYLESS_HINT = + "Repeat this goal across agents and levels of help with `ax journey --levels`; needs a Front API key."; + +/** + * The lines that follow a finished free run whose goal came from a public + * template or `--task`: with a Front API key, the same goal as a set of + * journeys with `--levels` as the Next step (plus a note when the goal names + * the site, since the need level is then skipped); without one, a dim hint. + * `extra` prints after the Next line. Nothing when the target is not a site a + * set accepts. + */ +function setFollowUp( + goal: string, + target: string, + hasFrontKey: boolean, +): { next?: NextStep; extra: string[] } | undefined { + const parsed = parseSetSite(target); + if ("error" in parsed) return undefined; + const site = parsed.value; + if (!hasFrontKey) return { extra: [pc.dim(SET_KEYLESS_HINT)] }; + const next: NextStep = { + call: axCall(["journey", goal, "--domain", site, "--levels"]), + why: "the same goal at each level of help, with the verdicts counted", + }; + const brand = defaultBrand(site); + if (!namesSite(goal, site, brand)) return { next, extra: [] }; + return { + next, + extra: [ + plain( + `This goal names ${brand}, so --levels runs it at the brand level. Add \`--exact \` for the exact level, or write the goal without ${brand} for the need level.`, + ), + ], + }; +} + +/** + * The next call after a failed run request: the request ora named, when it + * named one. Otherwise a run that is still executing is read, never started + * again; a missing key is named; everything else is the same call, after the + * wait the message names. + */ +function failureNext( + error: unknown, + sameCall: string, + keyed: string, + served: (action: JourneyNextAction) => NextStep, +): NextStep { + if (error instanceof DeepJourneyApiError && error.nextAction) return served(error.nextAction); + const code = error instanceof DeepJourneyApiError ? error.code : null; + if (code === "RUN_STILL_RUNNING" && error instanceof DeepJourneyApiError && error.runUrl) { + return { + call: `curl -s ${shellWord(error.runUrl)}`, + why: "read the run; starting another one would not finish this one sooner", + }; + } + switch (code) { + case "NEEDS_KEY": + return { call: keyed, why: "ora did not recognise the key; partner keys come from ora" }; + case "RATE_LIMITED": + case "USAGE_LIMIT_REACHED": + return { call: sameCall, why: "after the wait the message names" }; + case "ENGINE_UNAVAILABLE": + return { call: sameCall, why: "in about a minute" }; + default: + return { call: sameCall, why: "retry; ora may be briefly unreachable" }; + } } function verdictLine(verdict: string | undefined, status: string): string { @@ -79,6 +166,16 @@ function agentLine(agent: JourneyAgentOption): string { return `${agent.label} · ${agent.variant}`; } +// The notes ora raised beside the verdict (the site refused the agent, the +// answer came from memory), in ora's words and order. They never change the +// verdict. A raised flag that came without a reason shows its id instead. +function flagLines(flags: JourneyRunDetail["flags"]): string[] { + if (!flags) return []; + return Object.entries(flags) + .filter(([, flag]) => flag?.flagged) + .map(([id, flag]) => `${pc.yellow("!")} ${plain(flag.reason || id.replaceAll("_", " "))}`); +} + // The live graph (ui/graph) is written against the platform `JourneyStep` // shape; the public journey stream speaks the contract `JourneyTrajectoryStep`. // This bridges the two so `deep-journey` draws the very same attribution tree @@ -371,6 +468,7 @@ function renderOutcome(outcome: DeepJourneyOutcome, url: string): string[] { if (tokens != null) chips.push(`${chipTokens(tokens)} tokens`); if (result?.cost_usd != null) chips.push(chipCost(result.cost_usd)); if (chips.length) lines.push(` ${pc.dim(chips.join(" · "))}`); + for (const line of flagLines(detail.flags)) lines.push(` ${line}`); // Behavioral signals: the answer-quality score cards and attribution // distributions, when the result carries them. @@ -402,15 +500,34 @@ function renderOutcome(outcome: DeepJourneyOutcome, url: string): string[] { ); } - lines.push("", pc.dim(` For the 4-layer readiness audit, run: ax audit ${url}`)); + lines.push(""); return lines; } export async function deepJourneyCommand(input: DeepJourneyCommandInput): Promise { + // Usage mistakes print plain, API failures in red, both prefixed. `cause` + // is the thrown error, whose served body (if any) is what --json prints. + const fail = (error: CommandError, exit: number, cause?: unknown): number => { + const prefixed = (message: string) => `ax ${input.via ?? "deep-journey"}: ${message}`; + reportError(error, { + json: input.json, + display: (message) => (exit === EXIT.API ? pc.red(prefixed(message)) : prefixed(message)), + served: cause instanceof DeepJourneyApiError ? cause.payload : undefined, + }); + return exit; + }; + // A caller reading JSON keeps reading JSON on the next call. + const json = input.json ? ["--json"] : []; const target = input.url.trim(); if (!target) { - console.error("ax deep-journey: a target URL or domain is required"); - return EXIT.USAGE; + return fail( + { + code: "INVALID_TARGET", + message: "a target URL or domain is required", + next: { call: axCall(["journey", "--domain", placeholder("domain"), ...json]) }, + }, + EXIT.USAGE, + ); } const interactive = !input.json && process.stderr.isTTY; @@ -421,17 +538,66 @@ export async function deepJourneyCommand(input: DeepJourneyCommandInput): Promis const task = input.task?.trim() || undefined; const apiKey = input.apiKey || process.env.ORA_PARTNER_API_KEY || process.env.ORA_SCAN_API_KEY || undefined; + // The run as a call, for the Next line. The key is never echoed: a key + // passed as a flag comes back as a placeholder, one from the environment + // needs no flag at all. + // Written as `ax journey`, the name this run has now, whichever name was typed. + const runWords = (intent: string | undefined, agentId: string | undefined): Word[] => [ + "journey", + ...(task ? [task] : []), + "--domain", + target, + ...(!task && intent ? ["--intent", intent] : []), + ...(agentId ? ["--agent", agentId] : []), + ...json, + ...(input.noStream ? ["--no-stream"] : []), + ]; + const keyWords: Word[] = input.apiKey ? ["--api-key", placeholder("key")] : []; + const sameCall = (intent: string | undefined, agentId: string | undefined) => + axCall([...runWords(intent, agentId), ...keyWords]); + const keyedCall = (intent: string | undefined, agentId: string | undefined) => + `ORA_PARTNER_API_KEY= ${axCall(runWords(intent, agentId))}`; + // A next action ora served is the Next line. Starting the run again is + // this same ax call, which follows the run it starts; any other request + // (the run's stream, the agent list) is written as curl. A docs page ora + // links to (the partner key page on a 401) puts the keyed retry first. + const base = apiBase(); + const served = + (intent: string | undefined, agentId: string | undefined) => + (action: JourneyNextAction): NextStep => + servedNextStep(action, { + base, + axEquivalent: { method: "POST", url: JOURNEY_RUNS_PATH, call: sameCall(intent, agentId) }, + pageCall: keyedCall(intent, agentId), + }); if (task && input.intent !== undefined) { - console.error( - "ax deep-journey: --task and --intent are mutually exclusive — a free-text task replaces the curated intent", + return fail( + { + code: "INVALID_FLAG", + message: + input.via === "journey" + ? "a goal of your own and --intent cannot be combined: --intent picks one of ora's curated tasks" + : "--task and --intent are mutually exclusive — a free-text task replaces the curated intent", + next: { call: sameCall(undefined, input.agent), why: "the task alone" }, + }, + EXIT.USAGE, ); - return EXIT.USAGE; } if (task && !apiKey) { - console.error( - "ax deep-journey: --task needs an ora partner API key — set ORA_PARTNER_API_KEY or pass --api-key (keys are issued by ora on request)", + return fail( + { + code: "NEEDS_KEY", + message: + input.via === "journey" + ? "a goal of your own on ora's public journeys needs an ora partner API key (set ORA_PARTNER_API_KEY or pass --api-key); without --agent, --intent or --no-stream it runs on your Front account with ORA_API_KEY" + : "--task needs an ora partner API key — set ORA_PARTNER_API_KEY or pass --api-key (keys are issued by ora on request)", + next: { + call: keyedCall(undefined, input.agent), + why: "or drop --task to run a curated task without a key", + }, + }, + EXIT.USAGE, ); - return EXIT.USAGE; } // Resolve the intent and agent against the live rosters (both static, @@ -443,6 +609,9 @@ export async function deepJourneyCommand(input: DeepJourneyCommandInput): Promis // The goal drawn at the graph's root: the free-text task verbatim, or the // curated intent's own phrasing (template › hint › label), never a bare id. let intentText = task ?? "journey"; + // The goal a set can repeat: the --task text or the curated template, never + // the hint, label, id or "journey" fallbacks intentText may settle on. + let setGoal = task; try { const [intents, agents] = await Promise.all([ task ? undefined : fetchJourneyIntents(), @@ -452,12 +621,19 @@ export async function deepJourneyCommand(input: DeepJourneyCommandInput): Promis if (intentId === undefined) { intentId = intents.defaultId; } else if (!intents.intents.some((option) => option.id === intentId)) { - console.error( - `ax deep-journey: unknown intent "${intentId}". Available intents:\n${intents.intents - .map((option) => ` ${option.id.padEnd(12)} ${option.hint}`) - .join("\n")}`, + return fail( + { + code: "UNKNOWN_INTENT", + message: `unknown intent "${intentId}". Available intents:\n${intents.intents + .map((option) => ` ${option.id.padEnd(12)} ${option.hint}`) + .join("\n")}`, + next: { + call: sameCall(intents.defaultId, input.agent), + why: "or any intent id listed above", + }, + }, + EXIT.USAGE, ); - return EXIT.USAGE; } const chosen = intents.intents.find((option) => option.id === intentId); // The curated templates carry a literal `{domain}` token the server fills @@ -471,6 +647,7 @@ export async function deepJourneyCommand(input: DeepJourneyCommandInput): Promis intentId || "journey" ).replace(/\{domain\}/g, target); + if (chosen?.template) setGoal = chosen.template.replace(/\{domain\}/g, target); } // The selectable set: the live roster plus our haiku default (prepended so // it is both the default and resolvable via --agent). A roster that already @@ -486,16 +663,36 @@ export async function deepJourneyCommand(input: DeepJourneyCommandInput): Promis const wantedAgent = input.agent ?? preferredDefault; const found = roster.find((option) => option.id === wantedAgent); if (!found) { - console.error( - `ax deep-journey: unknown agent "${wantedAgent}". Available agents:\n${roster - .map((option) => ` ${option.id.padEnd(12)} ${agentLine(option)}`) - .join("\n")}`, + return fail( + { + code: "UNKNOWN_AGENT", + message: `unknown agent "${wantedAgent}". Available agents:\n${roster + .map((option) => ` ${option.id.padEnd(12)} ${agentLine(option)}`) + .join("\n")}`, + next: { + call: sameCall(intentId, preferredDefault), + why: "or any agent id listed above", + }, + }, + EXIT.USAGE, ); - return EXIT.USAGE; } agent = found; } catch (error) { - return fail(error instanceof Error ? error.message : String(error)); + return fail( + { + code: "API_ERROR", + message: error instanceof Error ? error.message : String(error), + next: failureNext( + error, + sameCall(intentId, input.agent), + keyedCall(intentId, input.agent), + served(intentId, input.agent), + ), + }, + EXIT.API, + error, + ); } if (!input.json) { @@ -554,15 +751,50 @@ export async function deepJourneyCommand(input: DeepJourneyCommandInput): Promis } catch (error) { if (live) panel.close(); else if (interactive) spinner.stop(); - if (error instanceof DeepJourneyApiError) return fail(error.message); - return fail(error instanceof Error ? error.message : String(error)); + return fail( + { + code: (error instanceof DeepJourneyApiError && error.code) || "API_ERROR", + message: error instanceof Error ? error.message : String(error), + next: failureNext( + error, + sameCall(intentId, agent.id), + keyedCall(intentId, agent.id), + served(intentId, agent.id), + ), + }, + EXIT.API, + error, + ); } if (live) panel.close(); else if (interactive) spinner.stop(); + const failed = outcome.detail.status === "failed"; + const servedNext = servedNextAction(outcome.detail); + // Only the branch that ends on the audit offers a set; a served next action + // and a failed run keep their own Next lines. + const followUp = + !servedNext && !failed && setGoal + ? setFollowUp(setGoal, target, Boolean(process.env.ORA_API_KEY)) + : undefined; + const next: NextStep = servedNext + ? served(intentId, agent.id)(servedNext) + : failed + ? { + call: sameCall(intentId, agent.id), + why: "the run ended in an engine error; re-running is free to try", + } + : (followUp?.next ?? { + call: axCall(["audit", target, ...json]), + why: "the readiness audit: which checks explain what the agent hit", + }); + if (input.json) { console.log(JSON.stringify(outcome.detail, null, 2)); - return outcome.detail.status === "failed" ? EXIT.RUN_FAILED : EXIT.OK; + for (const line of flagLines(outcome.detail.flags)) console.error(line); + printNext(next, { stderr: true }); + for (const line of followUp?.extra ?? []) console.error(line); + return failed ? EXIT.RUN_FAILED : EXIT.OK; } if (outcome.cached) { @@ -592,5 +824,7 @@ export async function deepJourneyCommand(input: DeepJourneyCommandInput): Promis } for (const line of renderOutcome(outcome, target)) console.log(line); - return outcome.detail.status === "failed" ? EXIT.RUN_FAILED : EXIT.OK; + printNext(next, { stderr: false }); + for (const line of followUp?.extra ?? []) console.log(line); + return failed ? EXIT.RUN_FAILED : EXIT.OK; } diff --git a/src/commands/generate-files.test.ts b/src/commands/generate-files.test.ts index 873c2ea..33dac70 100644 --- a/src/commands/generate-files.test.ts +++ b/src/commands/generate-files.test.ts @@ -85,14 +85,62 @@ describe("generateFilesCommand", () => { expect(await run()).toBe(EXIT.API); }); - it("--json prints ora's error body on a refusal", async () => { + it("--json prints ora's error body on a refusal, with the Next line on stderr", async () => { vi.mocked(api.fetchAgentFiles).mockRejectedValueOnce( new api.AgentFilesApiError(invalidDomain.message, { status: 400, payload: invalidDomain }), ); expect(await run({ domain: "localhost", json: true })).toBe(EXIT.USAGE); + expect(process.stdout.write).toHaveBeenCalledTimes(1); expect(process.stdout.write).toHaveBeenCalledWith( `${JSON.stringify(invalidDomain, null, 2)}\n`, ); + expect(console.log).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenLastCalledWith( + "Next: ax generate-files --json # a public host such as example.com", + ); + }); + + it("--json reports a failure with no ora body as an API_ERROR envelope retrying the same call", async () => { + vi.mocked(api.fetchAgentFiles).mockRejectedValueOnce( + new api.AgentFilesApiError("could not reach ora at https://ora.ai: fetch failed"), + ); + expect(await run({ json: true })).toBe(EXIT.API); + expect(process.stdout.write).toHaveBeenCalledTimes(1); + const envelope = JSON.parse(String(vi.mocked(process.stdout.write).mock.calls[0][0])); + expect(envelope).toEqual({ + error: { + code: "API_ERROR", + message: "could not reach ora at https://ora.ai: fetch failed", + next: `ax generate-files stripe.com --json --out ${out}`, + }, + }); + }); + + it("--json gives a 2xx answer ax refused the envelope, never the refused body", async () => { + const refused = { domain: "stripe.com", generatedAt: "2026-11-13T00:00:00Z" }; + vi.mocked(api.fetchAgentFiles).mockRejectedValueOnce( + new api.AgentFilesApiError("ora answered without a files list", { + status: 200, + payload: refused, + }), + ); + expect(await run({ json: true })).toBe(EXIT.API); + expect(process.stdout.write).toHaveBeenCalledTimes(1); + const envelope = JSON.parse(String(vi.mocked(process.stdout.write).mock.calls[0][0])); + expect(envelope.error.code).toBe("API_ERROR"); + expect(envelope.error.message).toBe("ora answered without a files list"); + }); + + it("ends a write with a Next line to rescan, and a refusal to overwrite with --force", async () => { + expect(await run()).toBe(EXIT.OK); + expect(console.log).toHaveBeenLastCalledWith( + "Next: ax audit stripe.com --force # after you publish them, rescan to verify", + ); + + expect(await run()).toBe(EXIT.USAGE); + expect(console.error).toHaveBeenLastCalledWith( + `Next: ax generate-files stripe.com --out ${out} --force # overwrite them, or pass --out to write somewhere else`, + ); }); it("exits 2 with no domain", async () => { diff --git a/src/commands/generate-files.ts b/src/commands/generate-files.ts index c465764..713c0cb 100644 --- a/src/commands/generate-files.ts +++ b/src/commands/generate-files.ts @@ -2,6 +2,8 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import pc from "picocolors"; import { AgentFilesApiError, type AgentFilesBundle, fetchAgentFiles } from "../api/agent-files"; +import { type CommandError, reportError } from "../report/json-error"; +import { axCall, placeholder, printNext } from "../report/next-line"; import { EXIT } from "./audit"; export interface GenerateFilesCommandInput { @@ -112,9 +114,58 @@ function wrap(text: string, width: number, indent: string): string[] { return rows; } +/** The code ora gave a refusal (`errorCode` on this route), or null. */ +function servedCode(payload: unknown): string | null { + if (typeof payload !== "object" || payload === null) return null; + const { errorCode, code } = payload as { errorCode?: unknown; code?: unknown }; + if (typeof errorCode === "string" && errorCode) return errorCode; + return typeof code === "string" && code ? code : null; +} + +function fetchFailure(input: GenerateFilesCommandInput, cause: unknown): CommandError { + const message = cause instanceof Error ? cause.message : String(cause); + // ora's 400 means the domain itself is not acceptable: a usage mistake. + if (cause instanceof AgentFilesApiError && cause.status === 400) { + return { + code: servedCode(cause.payload) || "INVALID_DOMAIN", + message, + next: { + call: axCall(["generate-files", placeholder("domain"), ...(input.json ? ["--json"] : [])]), + why: "a public host such as example.com", + }, + }; + } + const words = [ + "generate-files", + input.domain, + ...(input.json ? ["--json"] : []), + ...(input.out ? ["--out", input.out] : []), + ...(input.force ? ["--force"] : []), + ]; + return { + code: + servedCode(cause instanceof AgentFilesApiError ? cause.payload : undefined) || "API_ERROR", + message, + next: { call: axCall(words), why: "retry; ora may be briefly unreachable" }, + }; +} + export async function generateFilesCommand(input: GenerateFilesCommandInput): Promise { if (!input.domain?.trim()) { - console.error("Usage: ax generate-files "); + reportError( + { + code: "INVALID_TARGET", + message: "a domain is required", + next: { + call: axCall([ + "generate-files", + placeholder("domain"), + ...(input.json ? ["--json"] : []), + ]), + }, + }, + { json: input.json, display: () => "Usage: ax generate-files " }, + ); return EXIT.USAGE; } @@ -122,19 +173,30 @@ export async function generateFilesCommand(input: GenerateFilesCommandInput): Pr try { bundle = await fetchAgentFiles(input.domain, { baseUrl: input.baseUrl }); } catch (cause) { - if (input.json && cause instanceof AgentFilesApiError && cause.payload !== undefined) { - process.stdout.write(`${JSON.stringify(cause.payload, null, 2)}\n`); - } else { - console.error( - `generate-files failed: ${cause instanceof Error ? cause.message : String(cause)}`, - ); - } - // ora's 400 means the domain itself is not acceptable: a usage mistake. + const error = fetchFailure(input, cause); + // Only an HTTP error body is ora's error contract. A 2xx bundle ax refused + // would read as success on stdout, so that one gets the envelope. + const served = + cause instanceof AgentFilesApiError && cause.status !== null && cause.status >= 400 + ? cause.payload + : undefined; + reportError(error, { + json: input.json, + display: (message) => `generate-files failed: ${message}`, + served, + }); return cause instanceof AgentFilesApiError && cause.status === 400 ? EXIT.USAGE : EXIT.API; } if (input.json) { process.stdout.write(`${JSON.stringify(bundle, null, 2)}\n`); + printNext( + { + call: axCall(["generate-files", bundle.domain, "--out", placeholder("dir")]), + why: "write the files instead of printing them", + }, + { stderr: true }, + ); return EXIT.OK; } @@ -151,8 +213,17 @@ export async function generateFilesCommand(input: GenerateFilesCommandInput): Pr if (existing.length > 0 && !input.force) { console.error("These files already exist, so nothing was written:"); for (const w of existing) console.error(` ${displayPath(w.target)}`); - console.error( - pc.dim("Pass --force to overwrite them, or --out to write somewhere else."), + printNext( + { + call: axCall([ + "generate-files", + input.domain, + ...(input.out ? ["--out", input.out] : []), + "--force", + ]), + why: "overwrite them, or pass --out to write somewhere else", + }, + { stderr: true }, ); return EXIT.USAGE; } @@ -203,5 +274,12 @@ export async function generateFilesCommand(input: GenerateFilesCommandInput): Pr ), ); console.log(""); + printNext( + { + call: axCall(["audit", bundle.domain, "--force"]), + why: "after you publish them, rescan to verify", + }, + { stderr: false }, + ); return EXIT.OK; } diff --git a/src/commands/journey-cli.test.ts b/src/commands/journey-cli.test.ts new file mode 100644 index 0000000..f2f3b0d --- /dev/null +++ b/src/commands/journey-cli.test.ts @@ -0,0 +1,427 @@ +import { runCommand } from "citty"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { deepJourneyCommand } from "./deep-journey"; +import { journeyCommand } from "./journey"; +import { firstPositional, journey, journeyRoute, journeyUsage } from "./journey-cli"; +import { journeySetRunCommand, journeySetViewCommand } from "./journey-set"; +import { journeySetCancelCommand, journeySetListCommand } from "./journey-set-records"; + +// The six command functions are mocked: these tests pin which one a typed `ax journey` reaches, +// how its arguments map onto that input, and that each exit code reaches process.exitCode. +// Arguments go through citty's own parser (runCommand), so positionals and flag values are the +// real ones. +vi.mock("./journey", () => ({ journeyCommand: vi.fn() })); +vi.mock("./deep-journey", () => ({ deepJourneyCommand: vi.fn() })); +vi.mock("./journey-set", () => ({ journeySetRunCommand: vi.fn(), journeySetViewCommand: vi.fn() })); +vi.mock("./journey-set-records", () => ({ + journeySetListCommand: vi.fn(), + journeySetCancelCommand: vi.fn(), +})); + +const oneMock = vi.mocked(journeyCommand); +const setMock = vi.mocked(journeySetRunCommand); +const viewMock = vi.mocked(journeySetViewCommand); +const listMock = vi.mocked(journeySetListCommand); +const cancelMock = vi.mocked(journeySetCancelCommand); +const freeMock = vi.mocked(deepJourneyCommand); + +const savedExitCode = process.exitCode; +const GOAL = "Find how to create an issue from the API"; + +beforeEach(() => { + oneMock.mockResolvedValue(0); + setMock.mockResolvedValue(0); + viewMock.mockResolvedValue(0); + listMock.mockReturnValue(0); + cancelMock.mockResolvedValue(0); + freeMock.mockResolvedValue(0); + // A partner key in the shell would send a lone goal to the free path. + vi.stubEnv("ORA_PARTNER_API_KEY", ""); + vi.stubEnv("ORA_SCAN_API_KEY", ""); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.clearAllMocks(); + vi.restoreAllMocks(); + process.exitCode = savedExitCode; +}); + +const argv = (...rawArgs: string[]) => runCommand(journey, { rawArgs }); +const oneInput = () => oneMock.mock.calls[0][0]; +const setInput = () => setMock.mock.calls[0][0]; +const errored = () => vi.mocked(console.error).mock.calls.map((call) => String(call[0])); + +function expectOnly(mock: { mock: { calls: unknown[] } }) { + for (const other of [oneMock, setMock, viewMock, listMock, cancelMock, freeMock]) { + expect(other.mock.calls).toHaveLength(other === mock ? 1 : 0); + } +} + +describe("one journey", () => { + it("runs for one goal with no set flags, on claude-code by default", async () => { + await argv(GOAL, "--domain", "stripe.com"); + expectOnly(oneMock); + expect(oneInput()).toEqual({ + intent: GOAL, + domain: "stripe.com", + harness: "claude-code", + model: undefined, + json: false, + }); + }); + + it("takes --harness, --model and --json as before", async () => { + await argv(GOAL, "--harness", "codex", "--model", "gpt-5.5", "--json"); + expect(oneInput()).toMatchObject({ harness: "codex", model: "gpt-5.5", json: true }); + }); + + it("reads a single --agents entry as the harness and the model", async () => { + await argv(GOAL, "--agents", "codex/gpt-5.5"); + expectOnly(oneMock); + expect(oneInput()).toMatchObject({ harness: "codex", model: "gpt-5.5" }); + await argv(GOAL, "--agents", "eve"); + expect(oneMock.mock.calls[1][0]).toMatchObject({ harness: "eve", model: undefined }); + }); + + it("stays one journey with --repeat 1, and ignores --yes and --wait", async () => { + await argv(GOAL, "--repeat", "1", "--yes", "--wait", "5"); + expectOnly(oneMock); + expect(Object.keys(oneInput()).sort()).toEqual([ + "domain", + "harness", + "intent", + "json", + "model", + ]); + }); + + it.each([0, 1, 2])("sets process.exitCode to the returned %i", async (code) => { + oneMock.mockResolvedValue(code); + process.exitCode = undefined; + await argv(GOAL); + expect(process.exitCode).toBe(code); + }); +}); + +describe("a set of journeys", () => { + it("runs for several goals, in the order typed, with every value as typed", async () => { + await argv( + ...[GOAL, "Plan a sprint", "--domain", "linear.app", "--brand", "Linear"], + ...["--docs", "https://d.example", "--exact", "https://e.example", "--repeat", "3"], + ...["--wait", "15", "--yes", "--json"], + ); + expectOnly(setMock); + expect(setInput()).toEqual({ + site: "linear.app", + goals: [GOAL, "Plan a sprint"], + agents: undefined, + levels: false, + brand: "Linear", + docs: "https://d.example", + exact: "https://e.example", + repeat: "3", + wait: "15", + yes: true, + json: true, + }); + }); + + it("runs for --levels, --brand, --docs or --exact, passing --levels as typed", async () => { + await argv(GOAL, "--domain", "linear.app", "--levels"); + expect(setInput()).toMatchObject({ levels: true, goals: [GOAL] }); + for (const flag of [ + ["--brand", "Linear"], + ["--docs", "https://linear.app/docs"], + ["--exact", "https://linear.app/developers"], + ]) { + setMock.mockClear(); + await argv(GOAL, "--domain", "linear.app", ...flag); + expect(setInput()).toMatchObject({ levels: false }); + } + expect(oneMock).not.toHaveBeenCalled(); + }); + + it("runs for several agents or a repeat above 1, passing --agents as typed", async () => { + await argv(GOAL, "--domain", "linear.app", "--agents", "claude-code/opus, codex"); + expect(setInput()).toMatchObject({ agents: "claude-code/opus, codex" }); + await argv(GOAL, "--domain", "linear.app", "--repeat", "2"); + expect(setMock.mock.calls[1][0]).toMatchObject({ repeat: "2", agents: undefined }); + expect(oneMock).not.toHaveBeenCalled(); + }); + + it("turns --harness and --model into the set's one agent", async () => { + await argv(GOAL, "--levels", "--harness", "codex", "--model", "gpt-5.5"); + await argv(GOAL, "--levels", "--harness", "codex"); + await argv(GOAL, "--levels", "--model", "claude-opus-5"); + expect(setMock.mock.calls.map(([input]) => input.agents)).toEqual([ + "codex/gpt-5.5", + "codex", + "claude-code/claude-opus-5", + ]); + }); + + it("passes a missing --domain on, for the set to refuse", async () => { + await argv(GOAL, "--levels"); + expect(setInput().site).toBeUndefined(); + }); + + it("passes no test seam to the command", async () => { + await argv(GOAL, "--levels", "--yes"); + for (const seam of ["setsDir", "client", "now", "sleep", "ask"]) { + expect(setInput()).not.toHaveProperty(seam); + } + }); + + it.each([0, 2, 3, 130])("sets process.exitCode to the returned %i", async (code) => { + setMock.mockResolvedValue(code); + process.exitCode = undefined; + await argv(GOAL, "--levels"); + expect(process.exitCode).toBe(code); + }); +}); + +describe("usage errors before either path", () => { + it("refuses --agents with --harness or --model, exit 2, with the call without them", async () => { + process.exitCode = undefined; + await argv(GOAL, "--domain", "linear.app", "--agents", "codex", "--harness", "eve"); + expect(process.exitCode).toBe(2); + expect(errored()).toEqual([ + "--agents cannot be combined with --harness: name each agent in --agents as harness or harness/model", + `Next: ax journey '${GOAL}' --domain linear.app --agents codex # --agents alone names the agents`, + ]); + await argv(GOAL, "--agents", "codex", "--model=gpt-5.5", "--json"); + expect(process.stdout.write).toHaveBeenCalledWith(expect.stringContaining('"INVALID_FLAG"')); + expect(errored()[2]).toMatch(/^--agents cannot be combined with --model:/); + expect(oneMock).not.toHaveBeenCalled(); + expect(setMock).not.toHaveBeenCalled(); + }); + + it("refuses a bad --agents or --repeat, since the mode cannot be told without them", async () => { + process.exitCode = undefined; + await argv(GOAL, "--repeat", "0"); + expect(process.exitCode).toBe(2); + expect(errored()[0]).toMatch(/^--repeat must be an integer between 1 and 10/); + expect(errored()[1]).toBe( + "Next: ax journey --domain --levels # with the value named above fixed", + ); + expect(oneMock).not.toHaveBeenCalled(); + expect(setMock).not.toHaveBeenCalled(); + }); +}); + +describe("view, list and cancel", () => { + it("view passes the id, --wait and --json", async () => { + await argv("view", "j-0123456789", "--wait", "5", "--json"); + expectOnly(viewMock); + expect(viewMock).toHaveBeenCalledWith({ id: "j-0123456789", wait: "5", json: true }); + }); + + it("view leaves --wait undefined and --json false when not typed", async () => { + await argv("view", "j-0123456789"); + expect(viewMock).toHaveBeenCalledWith({ id: "j-0123456789", wait: undefined, json: false }); + }); + + it("list passes --json, and sets the exit code from its synchronous return", async () => { + process.exitCode = 9; + await argv("--json", "list"); + expectOnly(listMock); + expect(listMock).toHaveBeenCalledWith({ json: true }); + expect(process.exitCode).toBe(0); + }); + + it("cancel passes the id and --json", async () => { + await argv("cancel", "j-0123456789", "--json"); + expectOnly(cancelMock); + expect(cancelMock).toHaveBeenCalledWith({ id: "j-0123456789", json: true }); + }); + + it("sets process.exitCode to what view and cancel return", async () => { + viewMock.mockResolvedValue(130); + cancelMock.mockResolvedValue(3); + process.exitCode = undefined; + await argv("view", "j-0123456789"); + expect(process.exitCode).toBe(130); + await argv("cancel", "j-0123456789"); + expect(process.exitCode).toBe(3); + }); + + it("reads the word only as the first positional, never as a flag's value or a later goal", async () => { + await argv("--domain", "list", GOAL); + await argv(GOAL, "view"); + expect(listMock).not.toHaveBeenCalled(); + expect(viewMock).not.toHaveBeenCalled(); + expect(oneInput()).toMatchObject({ intent: GOAL, domain: "list" }); + expect(setInput().goals).toEqual([GOAL, "view"]); + }); +}); + +describe("a free journey, with no goal of its own", () => { + const FREE = { + url: "stripe.com", + intent: undefined, + task: undefined, + apiKey: undefined, + agent: undefined, + json: false, + noStream: false, + via: "journey", + }; + const freeInput = () => freeMock.mock.calls[0][0]; + + it("runs ora's curated task on the public journeys when no goal is given", async () => { + process.exitCode = undefined; + freeMock.mockResolvedValue(1); + await argv("--domain", "stripe.com"); + expectOnly(freeMock); + expect(freeInput()).toEqual(FREE); + expect(process.exitCode).toBe(1); + }); + + it("maps the public flags onto the free run", async () => { + await argv( + "--domain=stripe.com", + "--intent", + "pricing", + "--agent", + "cas-sonnet", + "--no-stream", + "--json", + ); + expectOnly(freeMock); + expect(freeInput()).toEqual({ + ...FREE, + intent: "pricing", + agent: "cas-sonnet", + noStream: true, + json: true, + }); + }); + + it("sends a goal with a public flag to the free path as its task", async () => { + await argv(GOAL, "--domain", "stripe.com", "--api-key", "k"); + expectOnly(freeMock); + expect(freeInput()).toEqual({ ...FREE, task: GOAL, apiKey: "k" }); + }); + + it("takes a lone goal to the partner tier when only a partner key is set", async () => { + vi.stubEnv("ORA_API_KEY", ""); + vi.stubEnv("ORA_PARTNER_API_KEY", "partner"); + await argv(GOAL, "--domain", "stripe.com"); + expectOnly(freeMock); + expect(freeInput().task).toBe(GOAL); + }); + + it("keeps a goal on the Front account when a Front key is set too", async () => { + vi.stubEnv("ORA_API_KEY", "front"); + vi.stubEnv("ORA_PARTNER_API_KEY", "partner"); + await argv(GOAL, "--domain", "stripe.com"); + expectOnly(oneMock); + }); + + it("refuses public and Front flags together, exit 2, with the call for Front", async () => { + process.exitCode = undefined; + await argv(GOAL, "--domain", "stripe.com", "--intent", "pricing", "--no-stream", "--levels"); + expect(process.exitCode).toBe(2); + expect(errored()).toEqual([ + "--intent, --no-stream are for ora's free journeys and --levels for your Front account's; use one kind", + `Next: ax journey '${GOAL}' --domain stripe.com --levels # for your Front account`, + ]); + expect(freeMock).not.toHaveBeenCalled(); + expect(setMock).not.toHaveBeenCalled(); + }); + + it("needs --domain, exit 2", async () => { + process.exitCode = undefined; + await argv("--intent", "pricing"); + expect(process.exitCode).toBe(2); + expect(errored()).toEqual([ + "--domain names the site the agent works on", + "Next: ax journey --domain # ora picks the task", + ]); + expect(freeMock).not.toHaveBeenCalled(); + }); + + it("refuses several goals with a public flag, since a free run takes one", async () => { + process.exitCode = undefined; + await argv(GOAL, "Find the pricing page", "--domain", "stripe.com", "--agent", "cas-haiku"); + expect(process.exitCode).toBe(2); + expect(errored()[0]).toBe( + "a free run takes one goal; several goals run a set, on your Front account", + ); + expect(freeMock).not.toHaveBeenCalled(); + }); + + it("sends no goal with a Front flag to the Front path, where the set asks for a goal", async () => { + await argv("--domain", "stripe.com", "--levels"); + expectOnly(setMock); + expect(setInput().goals).toEqual([]); + }); +}); + +describe("journeyRoute", () => { + const base = { goals: [], publicFlags: [], frontFlags: [], frontKey: false, partnerKey: false }; + + it("routes by the flags first, then by the goal and the keys", () => { + expect(journeyRoute(base)).toEqual({ route: "public" }); + expect(journeyRoute({ ...base, goals: ["g"] })).toEqual({ route: "front" }); + expect(journeyRoute({ ...base, goals: ["g"], partnerKey: true })).toEqual({ route: "public" }); + expect(journeyRoute({ ...base, goals: ["g", "h"], partnerKey: true })).toEqual({ + route: "front", + }); + expect(journeyRoute({ ...base, goals: ["g"], frontKey: true, partnerKey: true })).toEqual({ + route: "front", + }); + expect(journeyRoute({ ...base, goals: ["g"], publicFlags: ["agent"] })).toEqual({ + route: "public", + }); + expect(journeyRoute({ ...base, frontFlags: ["levels"] })).toEqual({ route: "front" }); + }); +}); + +describe("firstPositional", () => { + it("skips flags and the values of string flags, even values that look like flags", () => { + expect(firstPositional(["--json", "--domain", "--x", "list"])).toBe(3); + expect(firstPositional(["--domain=linear.app", "view", "j-1"])).toBe(1); + expect(firstPositional(["--levels", "goal"])).toBe(1); + }); + + it("takes the argument after -- as positional, and -1 when there is none", () => { + expect(firstPositional(["--", "--json"])).toBe(1); + expect(firstPositional(["--json", "--"])).toBe(-1); + expect(firstPositional([])).toBe(-1); + }); +}); + +describe("journeyUsage", () => { + // biome-ignore lint/suspicious/noControlCharactersInRegex: strips the color codes picocolors may add + const usage = journeyUsage().replace(/\u001b\[[0-9;]*m/g, ""); + + it("shows the one journey form, the set form and the three commands", () => { + expect(usage).toContain('ax journey "" [--domain ] [--harness ] [--model ]'); + expect(usage).toContain('ax journey "" ["" ...] --domain [--agents ]'); + for (const command of ["view ", "list [--json]", "cancel "]) { + expect(usage).toContain(`ax journey ${command}`); + } + for (const flag of [ + "--levels", + "--brand", + "--docs", + "--exact", + "--repeat", + "--yes", + "--wait", + ]) { + expect(usage).toContain(flag); + } + }); + + it("states the mode rule, the reserved words and what one journey ignores", () => { + expect(usage).toContain("ax runs a set of journeys"); + expect(usage).toContain("a goal cannot be just view, list or cancel"); + expect(usage).toContain("--yes and --wait apply to a set; one journey ignores them."); + expect(usage).not.toMatch(/\btrials?\b/i); + }); +}); diff --git a/src/commands/journey-cli.ts b/src/commands/journey-cli.ts new file mode 100644 index 0000000..c0ea7ac --- /dev/null +++ b/src/commands/journey-cli.ts @@ -0,0 +1,497 @@ +import { defineCommand, runCommand } from "citty"; +import pc from "picocolors"; +import { SET_USAGE_CALL } from "../journey-set/report-flow"; +import { + DEFAULT_HARNESS, + DEFAULT_WAIT_MINUTES, + journeyMode, + MAX_AGENTS, + MAX_GOALS, + MAX_REPEAT, +} from "../journey-set/spec"; +import { reportError } from "../report/json-error"; +import { axCall, placeholder } from "../report/next-line"; +import { EXIT } from "./audit"; +import { deepJourneyCommand } from "./deep-journey"; +import { journeyCommand } from "./journey"; +import { journeySetRunCommand, journeySetViewCommand } from "./journey-set"; +import { journeySetCancelCommand, journeySetListCommand } from "./journey-set-records"; + +// The citty definition of `ax journey`. With no goal of its own it runs one of ora's curated +// tasks on ora.ai's public journeys, free (journeyRoute); with a goal, one journey drawn live or +// a set of journeys on your Front account, as journeyMode reads the flags; plus `view`, `list` +// and `cancel` for recorded sets. Those three +// are words the run dispatches on, not citty subcommands: citty refuses a first positional that +// names no subcommand, so a goal in plain words could not sit beside them. Every command function +// returns its exit code; the runs assign it to process.exitCode and never call process.exit() +// (ax invariant 10: a large --json document must drain before the process ends). The test seams +// of the command functions (setsDir, client, now, sleep, ask) get no flag. + +const JSON_ARG = { + type: "boolean", + description: "Print JSON on stdout instead of the report", + default: false, +} as const; + +const WAIT_ARG = { + type: "string", + description: `Minutes to follow the runs before printing what has finished (default ${DEFAULT_WAIT_MINUTES})`, +} as const; + +const SET_ID_ARG = { + type: "positional", + description: "Set id (see ax journey list)", + required: true, +} as const; + +const VIEW_DESCRIPTION = "Pick up a recorded set, follow its runs, and print the report"; +const LIST_DESCRIPTION = "List the sets recorded on this machine; makes no API call"; +const CANCEL_DESCRIPTION = "Stop the runs of a recorded set that are still going"; + +const view = defineCommand({ + meta: { name: "view", description: VIEW_DESCRIPTION }, + args: { id: SET_ID_ARG, wait: WAIT_ARG, json: JSON_ARG }, + async run({ args }) { + process.exitCode = await journeySetViewCommand({ + id: args.id as string, + wait: args.wait as string | undefined, + json: Boolean(args.json), + }); + }, +}); + +const list = defineCommand({ + meta: { name: "list", description: LIST_DESCRIPTION }, + args: { json: JSON_ARG }, + async run({ args }) { + process.exitCode = journeySetListCommand({ json: Boolean(args.json) }); + }, +}); + +const cancel = defineCommand({ + meta: { name: "cancel", description: CANCEL_DESCRIPTION }, + args: { + id: SET_ID_ARG, + json: { ...JSON_ARG, description: "Print the platform's answer as JSON" }, + }, + async run({ args }) { + process.exitCode = await journeySetCancelCommand({ + id: args.id as string, + json: Boolean(args.json), + }); + }, +}); + +const SET_COMMANDS = { + view: (rawArgs: string[]) => runCommand(view, { rawArgs }), + list: (rawArgs: string[]) => runCommand(list, { rawArgs }), + cancel: (rawArgs: string[]) => runCommand(cancel, { rawArgs }), +}; + +function isSetCommand(word: string | undefined): word is keyof typeof SET_COMMANDS { + return word !== undefined && Object.hasOwn(SET_COMMANDS, word); +} + +const JOURNEY_ARGS = { + goal: { + type: "positional", + description: `What the agent should do, in plain words; up to ${MAX_GOALS} goals run a set. Leave it out to run one of ora's curated tasks, free`, + required: false, + }, + domain: { + type: "string", + description: "Site the agent works on, a domain or an http(s) URL (a set needs one)", + }, + harness: { + type: "string", + description: `Agent harness for one agent (default ${DEFAULT_HARNESS})`, + }, + model: { type: "string", description: "Model for one agent (the harness default when omitted)" }, + agents: { + type: "string", + description: `Agents as harness or harness/model, comma separated (up to ${MAX_AGENTS})`, + }, + levels: { + type: "boolean", + description: "Run each goal at the levels of help and count them", + default: false, + }, + brand: { type: "string", description: "Brand for the brand level; turns levels on" }, + docs: { type: "string", description: "Documentation URL; adds the docs level" }, + exact: { type: "string", description: "URL the agent starts at; adds the exact level" }, + repeat: { + type: "string", + description: `Runs for each goal, level and agent (1 to ${MAX_REPEAT}; default 1)`, + }, + yes: { type: "boolean", description: "Launch a set without asking", default: false }, + wait: WAIT_ARG, + intent: { + type: "string", + description: "One of ora's curated tasks, e.g. pricing, signup, api-docs (a free run)", + }, + agent: { + type: "string", + description: + "Agent from ora's public roster for a free run (default cas-haiku, Claude Code on Haiku)", + }, + "api-key": { + type: "string", + description: + "ora partner API key: lets a free run take a goal of your own; also read from ORA_PARTNER_API_KEY, then ORA_SCAN_API_KEY", + }, + "no-stream": { + type: "boolean", + description: "A free run prints only its result, not the live view", + default: false, + }, + json: JSON_ARG, +} as const; + +/** + * Whether `--no-stream` was given. citty reads `--no-` as `: false`, so the flag + * arrives as `stream: false` and the declared `no-stream` arg keeps its default. + */ +export function noStreamFlag(args: Record): boolean { + return args.stream === false || args["no-stream"] === true; +} + +/** Flags only ora.ai's public journeys read. */ +const PUBLIC_FLAGS = ["intent", "agent", "api-key", "no-stream"] as const; +/** Flags only your Front account's journeys read. */ +const FRONT_FLAGS = [ + "harness", + "model", + "agents", + "levels", + "brand", + "docs", + "exact", + "repeat", + "yes", + "wait", +] as const; + +export type JourneyRoute = + | { route: "public" } + | { route: "front" } + | { error: string; drop: readonly string[] }; + +/** + * Which system a journey runs on. A public flag, or no goal at all, means ora.ai's public + * journeys: a curated task, free, or a goal of your own with a partner key. Otherwise a goal runs + * on your Front account, except that a goal with only a partner key set (no ORA_API_KEY) takes the + * partner tier, which is what `ax deep-journey --task` did. `drop` names the flags to remove when + * both kinds were given. + */ +export function journeyRoute(input: { + goals: readonly string[]; + publicFlags: readonly string[]; + frontFlags: readonly string[]; + frontKey: boolean; + partnerKey: boolean; +}): JourneyRoute { + const { goals, publicFlags, frontFlags } = input; + if (publicFlags.length > 0 && frontFlags.length > 0) { + const pub = publicFlags.map((f) => `--${f}`).join(", "); + const front = frontFlags.map((f) => `--${f}`).join(", "); + return { + error: `${pub} ${publicFlags.length === 1 ? "is" : "are"} for ora's free journeys and ${front} for your Front account's; use one kind`, + drop: publicFlags.map((f) => `--${f}`), + }; + } + if (publicFlags.length > 0) return { route: "public" }; + if (frontFlags.length > 0) return { route: "front" }; + if (goals.length === 0) return { route: "public" }; + if (goals.length === 1 && !input.frontKey && input.partnerKey) return { route: "public" }; + return { route: "front" }; +} + +/** The flag names present in the raw args, in the order of `names`. */ +function present(rawArgs: readonly string[], names: readonly string[]): string[] { + const typed = new Set(); + for (const arg of rawArgs) { + if (arg === "--") break; + if (arg.startsWith("--")) + typed.add( + arg + .slice(2) + .split("=")[0] + .replace(/^no-(?!stream)/, ""), + ); + } + return names.filter((name) => typed.has(name)); +} + +/** The flags that take a value, so the scans below skip it. */ +const VALUE_FLAGS: ReadonlySet = new Set( + Object.entries(JOURNEY_ARGS) + .filter(([, def]) => def.type === "string") + .map(([name]) => `--${name}`), +); + +/** + * Where the first positional sits in the raw args, found the way citty finds a subcommand: the + * value after a string flag is skipped even when it starts with `-`, `--name=value` is one + * argument, and after `--` every argument is positional. -1 when there is none. + */ +export function firstPositional(rawArgs: readonly string[]): number { + for (let i = 0; i < rawArgs.length; i++) { + const arg = rawArgs[i]; + if (arg === "--") return i + 1 < rawArgs.length ? i + 1 : -1; + if (!arg.startsWith("-")) return i; + if (!arg.includes("=") && VALUE_FLAGS.has(arg)) i++; + } + return -1; +} + +/** The raw args without the named flags and their values; the rest stays as typed. */ +function withoutFlags(rawArgs: readonly string[], names: readonly string[]): string[] { + const kept: string[] = []; + for (let i = 0; i < rawArgs.length; i++) { + const arg = rawArgs[i]; + if (arg === "--") { + kept.push(...rawArgs.slice(i)); + break; + } + const name = arg.includes("=") ? arg.slice(0, arg.indexOf("=")) : arg; + if (!names.includes(name)) kept.push(arg); + else if (!arg.includes("=") && VALUE_FLAGS.has(name)) i++; + } + return kept; +} + +interface JourneyArgs { + goals: string[]; + intent?: string; + agent?: string; + apiKey?: string; + noStream: boolean; + domain?: string; + harness?: string; + model?: string; + agents?: string; + levels: boolean; + brand?: string; + docs?: string; + exact?: string; + repeat?: string; + wait?: string; + yes: boolean; + json: boolean; +} + +/** --harness and --model as one --agents entry, for a set that names its agent the long way. */ +function agentEntry(harness: string | undefined, model: string | undefined): string | undefined { + if (harness === undefined && model === undefined) return undefined; + const named = harness ?? DEFAULT_HARNESS; + return model === undefined ? named : `${named}/${model}`; +} + +/** A free run on ora.ai's public journeys: a curated task, or a goal of your own with a partner key. */ +function runFree(input: JourneyArgs): Promise | number { + const { goals, json } = input; + const next = (why: string) => ({ + call: axCall(["journey", "--domain", placeholder("domain"), ...(json ? ["--json"] : [])]), + why, + }); + if (goals.length > 1) { + const message = "a free run takes one goal; several goals run a set, on your Front account"; + reportError( + { code: "INVALID_SPEC", message, next: { call: SET_USAGE_CALL, why: "for a set" } }, + { json }, + ); + return EXIT.USAGE; + } + if (!input.domain?.trim()) { + const message = "--domain names the site the agent works on"; + reportError({ code: "INVALID_TARGET", message, next: next("ora picks the task") }, { json }); + return EXIT.USAGE; + } + return deepJourneyCommand({ + url: input.domain, + intent: input.intent, + task: goals[0], + apiKey: input.apiKey, + agent: input.agent, + json, + noStream: input.noStream, + via: "journey", + }); +} + +/** One journey or a set; --yes and --wait only mean something to a set, so one journey ignores them. */ +async function runJourney(input: JourneyArgs, rawArgs: readonly string[]): Promise { + const { goals, harness, model, agents, json } = input; + const route = journeyRoute({ + goals, + publicFlags: present(rawArgs, PUBLIC_FLAGS), + frontFlags: present(rawArgs, FRONT_FLAGS), + frontKey: Boolean(process.env.ORA_API_KEY), + partnerKey: Boolean( + input.apiKey || process.env.ORA_PARTNER_API_KEY || process.env.ORA_SCAN_API_KEY, + ), + }); + if ("error" in route) { + const next = { + call: axCall(["journey", ...withoutFlags(rawArgs, route.drop)]), + why: "for your Front account", + }; + reportError({ code: "INVALID_FLAG", message: route.error, next }, { json }); + return EXIT.USAGE; + } + if (route.route === "public") return runFree(input); + if (agents !== undefined && (harness !== undefined || model !== undefined)) { + const flag = harness !== undefined ? "--harness" : "--model"; + const message = `--agents cannot be combined with ${flag}: name each agent in --agents as harness or harness/model`; + const next = { + call: axCall(["journey", ...withoutFlags(rawArgs, ["--harness", "--model"])]), + why: "--agents alone names the agents", + }; + reportError({ code: "INVALID_FLAG", message, next }, { json }); + return EXIT.USAGE; + } + const mode = journeyMode(input); + if ("error" in mode) { + const next = { call: SET_USAGE_CALL, why: "with the value named above fixed" }; + reportError({ code: "INVALID_SPEC", message: mode.error, next }, { json }); + return EXIT.USAGE; + } + if (!mode.value.set) { + const [agent] = mode.value.agents; + return journeyCommand({ + intent: goals[0], + domain: input.domain, + harness: agent?.harness ?? harness ?? DEFAULT_HARNESS, + model: agent?.model ?? model, + json, + }); + } + return journeySetRunCommand({ + site: input.domain, + goals, + agents: agents ?? agentEntry(harness, model), + levels: input.levels, + brand: input.brand, + docs: input.docs, + exact: input.exact, + repeat: input.repeat, + wait: input.wait, + yes: input.yes, + json, + }); +} + +const JOURNEY_DESCRIPTION = + "Send a real AI agent at a site and watch it live: free on one of ora's curated tasks, or on a goal of your own with a Front API key, one journey or a set with the verdicts counted"; + +export const journey = defineCommand({ + meta: { name: "journey", description: JOURNEY_DESCRIPTION }, + args: JOURNEY_ARGS, + async run({ args, rawArgs }) { + const at = firstPositional(rawArgs); + const word = rawArgs[at]; + if (isSetCommand(word)) { + const rest = [...rawArgs.slice(0, at), ...rawArgs.slice(at + 1)]; + await SET_COMMANDS[word](rest); + return; + } + // Same exitCode-over-exit rule as audit: let stdout drain before exiting. + process.exitCode = await runJourney( + { + goals: args._, + intent: args.intent as string | undefined, + agent: args.agent as string | undefined, + apiKey: args["api-key"] as string | undefined, + noStream: noStreamFlag(args), + domain: args.domain as string | undefined, + harness: args.harness as string | undefined, + model: args.model as string | undefined, + agents: args.agents as string | undefined, + levels: Boolean(args.levels), + brand: args.brand as string | undefined, + docs: args.docs as string | undefined, + exact: args.exact as string | undefined, + repeat: args.repeat as string | undefined, + wait: args.wait as string | undefined, + yes: Boolean(args.yes), + json: Boolean(args.json), + }, + rawArgs, + ); + }, +}); + +/** `ax journey --help`: citty's generated usage cannot show both forms and the three commands. */ +export function journeyUsage(): string { + const d = pc.dim; + const h = (title: string) => pc.underline(pc.bold(title)); + return [ + d(`${JOURNEY_DESCRIPTION} (ax journey)`), + "", + h("USAGE"), + "", + " ax journey --domain [--intent ] [--agent ] [--no-stream] [--json]", + ` ax journey "" [--domain ] [--harness ] [--model ] [--json]`, + ` ax journey "" ["" ...] --domain [--agents ] [--levels]`, + " [--brand ] [--docs ] [--exact ] [--repeat ] [--yes] [--wait ] [--json]", + " ax journey view [--wait ] [--json]", + " ax journey list [--json]", + " ax journey cancel [--json]", + "", + " With no goal, ax runs one of ora's curated tasks on ora.ai's public journeys: free, no key.", + " With a goal, the journey runs on your Front account.", + "", + " One goal and one agent, with no --repeat and no levels: ax runs one journey and draws it", + " live. Ask for more (several goals or agents, --repeat, or levels with --levels, --docs,", + " --exact or --brand) and ax runs a set of journeys: it previews the set, asks before", + " launching, waits for the platform's verdicts and counts them.", + "", + h("FREE, NO GOAL OF YOUR OWN"), + "", + ` --domain ${d("Site the agent works on (required)")}`, + ` --intent ${d("Curated task id, e.g. pricing, signup, api-docs (ora picks one when omitted)")}`, + ` --agent ${d("Agent from ora's public roster (default cas-haiku, Claude Code on Haiku)")}`, + ` --api-key ${d("ora partner API key: allows a goal of your own here; also read from ORA_PARTNER_API_KEY, then ORA_SCAN_API_KEY")}`, + ` --no-stream ${d("Print only the result, not the live view")}`, + ` --json ${d("Print the finished run as JSON")}`, + ` ${d("these flags cannot be combined with the Front flags below")}`, + ` ${d("exit codes: 0 the run finished (any verdict) · 1 the run failed · 2 usage · 3 API error")}`, + ` ${d("ax deep-journey is the older name for this; it still works and will be removed in a future major release")}`, + "", + h("ONE JOURNEY"), + "", + ` --domain ${d("Site the agent works on (e.g. stripe.com)")}`, + ` --harness ${d(`Agent harness: ${DEFAULT_HARNESS} (default), codex, openclaw, hermess, claude-agent, eve, ...`)}`, + ` --model ${d("Model override (the harness default when omitted)")}`, + ` --agents ${d("One agent as harness or harness/model, instead of --harness and --model")}`, + ` --json ${d("Print the run as JSON")}`, + ` ${d("exit codes: 0 the run succeeded · 1 it finished without success · 2 error")}`, + "", + h("A SET OF JOURNEYS"), + "", + ` ... ${d(`1 to ${MAX_GOALS} goals, each in quotes`)}`, + ` --domain ${d("Site the agents work on, a domain or an http(s) URL (required)")}`, + ` --agents ${d(`Agents as harness or harness/model, comma separated (up to ${MAX_AGENTS}; default: your plan's Claude Code agent, else the first it offers)`)}`, + ` --levels ${d("Run each goal at the levels of help: need only, brand named, docs linked, exact target")}`, + ` --brand ${d("Brand for the brand level (default: the site's name); turns levels on")}`, + ` --docs ${d("Documentation URL; adds the docs level and turns levels on")}`, + ` --exact ${d("URL the agent starts at; adds the exact level and turns levels on")}`, + ` --repeat ${d(`Runs for each goal, level and agent (1 to ${MAX_REPEAT}; default 1)`)}`, + ` --yes ${d("Launch without asking after the preview")}`, + ` --wait ${d(`Minutes to follow the runs before printing what has finished (default ${DEFAULT_WAIT_MINUTES})`)}`, + ` --json ${d("Print one JSON document on stdout instead of the report")}`, + ` ${d("--harness and --model name one agent for a set too; --agents cannot be combined with them")}`, + ` ${d("without levels, each goal runs once as written; the preview shows the runs before any start")}`, + ` ${d("exit codes: 0 report printed, even with runs unfinished · 2 usage or refusal before launch · 3 API error · 130 Ctrl-C")}`, + "", + h("COMMANDS"), + "", + ` view ${d(`${VIEW_DESCRIPTION}; takes --wait and --json`)}`, + ` list ${d(`${LIST_DESCRIPTION}; takes --json`)}`, + ` cancel ${d(`${CANCEL_DESCRIPTION}; takes --json`)}`, + ` ${d("a goal cannot be just view, list or cancel: ax runs the command of that name")}`, + "", + ` ${d("--yes and --wait apply to a set; one journey ignores them.")}`, + ` ${d("A goal of your own needs a Front API key as ORA_API_KEY; sign up at app.ora.ai (see the README's Get a key).")}`, + ].join("\n"); +} diff --git a/src/commands/journey-set-records.test.ts b/src/commands/journey-set-records.test.ts new file mode 100644 index 0000000..939303f --- /dev/null +++ b/src/commands/journey-set-records.test.ts @@ -0,0 +1,426 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import cancel from "../api/__fixtures__/studio/cancel.json"; +import missingScope403 from "../api/__fixtures__/studio/error-403-missing-scope.json"; +import { createStudioClient } from "../api/studio"; +import { + appendJourney, + createRecord, + type JourneyEntry, + type SetRecord, +} from "../journey-set/record"; +import { + GET_A_KEY, + KEY_DESTINATION_WHY, + PRODUCT_NOT_GRANTED_MESSAGE, +} from "../journey-set/render-preview"; +import { parseSetSpec, planCells } from "../journey-set/spec"; +import { EXIT } from "./audit"; +import { + type JourneySetCancelInput, + journeySetCancelCommand, + journeySetListCommand, +} from "./journey-set-records"; + +// Records are real files in a temp directory, written with createRecord and appendJourney. The +// cancel call goes through a real createStudioClient over a mocked fetch, so each error has the +// shape the client really throws. Fixtures: src/api/__fixtures__/studio/ (provenance in +// src/api/studio.test.ts): cancel.json answers the happy path, error-403-missing-scope.json the +// missing scope. The PRODUCT_NOT_GRANTED body is that 403 with its code swapped, and the 500 is a +// plain Studio error body. + +const BASE = "https://platform.test"; +const KEY_NEXT = "ORA_API_KEY= ax journey cancel"; + +const asJson = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + +const PRODUCT_403 = { error: "Forbidden", code: "PRODUCT_NOT_GRANTED" }; +const SERVER_500 = { error: "boom", code: "internal" }; + +type Reply = () => Response | Promise; + +/** Answers the key exchange, then records every other call; a call with no reply fails loudly. */ +function studioMock(reply?: Reply) { + const bodies: Array<{ url: string; method: string; body: unknown }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string | URL, init: RequestInit = {}) => { + const at = String(url); + if (at.endsWith("/auth/token")) return asJson({ token: "bearer" }); + bodies.push({ + url: at, + method: init.method ?? "GET", + body: init.body ? JSON.parse(String(init.body)) : undefined, + }); + if (!reply) throw new Error(`unrouted ${init.method} ${at}`); + return reply(); + }), + ); + return { calls: bodies }; +} + +const client = () => createStudioClient({ apiKey: "ora_sk_local", baseUrl: BASE }); + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ax-set-records-")); + vi.stubEnv("ORA_API_KEY", "ora_sk_local"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); +}); + +afterEach(() => { + process.exitCode = undefined; + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + rmSync(dir, { recursive: true, force: true }); +}); + +const stdoutWrites = () => + vi + .mocked(process.stdout.write) + .mock.calls.map((c) => String(c[0])) + .join(""); +const logged = () => vi.mocked(console.log).mock.calls.map((c) => String(c[0])); +const errored = () => vi.mocked(console.error).mock.calls.map((c) => String(c[0])); + +function newSet(site = "linear.app"): SetRecord { + const parsed = parseSetSpec({ + site, + levels: true, + goals: ["Create an issue", "Plan a sprint"], + exact: `https://${site}/developers`, + }); + if (!parsed.ok) throw new Error(parsed.error); + return createRecord( + { + spec: parsed.spec, + agents: [{ harness: "claude-code", model: "claude-sonnet-5-5" }], + cells: planCells(parsed.spec).cells, + }, + dir, + ); +} + +function entry(n: number, runIds: string[]): JourneyEntry { + return { + journeyId: `j-${n}`, + goalIndex: 0, + level: "brand", + repeat: 1, + runIds, + failed: [], + createdAt: `2026-11-13T10:00:0${n}.000Z`, + }; +} + +/** A set with two journeys holding the three run ids cancel.json answers for. */ +function launchedSet(): { record: SetRecord; runIds: string[] } { + const record = newSet(); + const runIds = [...cancel.canceled, ...cancel.alreadyTerminal, ...cancel.skipped]; + appendJourney(record.id, entry(1, runIds.slice(0, 2)), dir); + appendJourney(record.id, entry(2, runIds.slice(2)), dir); + return { record, runIds }; +} + +const cancelIt = ( + id: string, + json = false, + extra: { client?: JourneySetCancelInput["client"] } = {}, +) => journeySetCancelCommand({ id, json, setsDir: dir, client: extra.client ?? client }); + +describe("journeySetListCommand", () => { + it("prints one line per record and asks no server", () => { + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + const a = newSet("linear.app"); + appendJourney(a.id, entry(1, ["r1", "r2"]), dir); + const b = newSet("stripe.com"); + + expect(journeySetListCommand({ json: false, setsDir: dir })).toBe(EXIT.OK); + + const lines = logged(); + expect(lines).toHaveLength(3); + const lineA = lines.find((line) => line.startsWith(a.id)); + const lineB = lines.find((line) => line.startsWith(b.id)); + expect(lineA).toContain(a.createdAt); + expect(lineA).toContain("linear.app"); + expect(lineA).toContain("1 journey, 2 runs"); + expect(lineB).toContain("stripe.com"); + expect(lineB).toContain("0 journeys, 0 runs"); + expect(lines[2]).toMatch(/^Next: ax journey view j-[a-z0-9]{10} {2}# /); + expect(errored()).toEqual([]); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(stdoutWrites()).toBe(""); + }); + + it("--json prints ax's own summaries on stdout and nothing else there", () => { + const a = newSet(); + appendJourney(a.id, entry(1, ["r1", "r2", "r3"]), dir); + + expect(journeySetListCommand({ json: true, setsDir: dir })).toBe(EXIT.OK); + + expect(JSON.parse(stdoutWrites())).toEqual({ + records: [{ id: a.id, createdAt: a.createdAt, site: "linear.app", journeys: 1, runs: 3 }], + skipped: [], + }); + expect(console.log).not.toHaveBeenCalled(); + expect(errored()).toHaveLength(1); + expect(errored()[0]).toBe(`Next: ax journey view ${a.id} # read the newest set's report`); + }); + + it("says there are no sets yet, with a Next line to start one", () => { + expect(journeySetListCommand({ json: false, setsDir: join(dir, "missing") })).toBe(EXIT.OK); + expect(logged()).toEqual([ + "No sets of journeys on this machine yet.", + "Next: ax journey --domain --levels # start a set of journeys; it is recorded here", + ]); + expect(errored()).toEqual([]); + }); + + it("names a corrupt file on stderr and still lists the rest", () => { + const a = newSet(); + writeFileSync(join(dir, "j-abcdef0123.json"), "{not json"); + + expect(journeySetListCommand({ json: false, setsDir: dir })).toBe(EXIT.OK); + + expect(logged()).toHaveLength(2); + expect(errored()).toHaveLength(1); + expect(errored()[0]).toBe( + "Skipped 1 file that is not a readable set record: j-abcdef0123.json", + ); + + vi.mocked(process.stdout.write).mockClear(); + expect(journeySetListCommand({ json: true, setsDir: dir })).toBe(EXIT.OK); + expect(JSON.parse(stdoutWrites())).toMatchObject({ + records: [{ id: a.id }], + skipped: ["j-abcdef0123.json"], + }); + }); +}); + +describe("journeySetCancelCommand refusals", () => { + it("refuses an id that is not a set id (exit 2), without reading anything", async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + expect(await cancelIt("../../etc/passwd")).toBe(EXIT.USAGE); + expect(errored()[0]).toBe('"../../etc/passwd" is not a set id. Ids look like j-0123456789.'); + expect(errored()[1]).toMatch(/^Next: ax journey list {2}# /); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("refuses an unknown id (exit 2)", async () => { + expect(await cancelIt("j-0000000000")).toBe(EXIT.USAGE); + expect(errored()[0]).toBe("No set j-0000000000 on this machine."); + expect(errored()[1]).toMatch(/^Next: ax journey list {2}# /); + }); + + it("refuses a corrupt record and names the file (exit 2)", async () => { + writeFileSync(join(dir, "j-abcdef0123.json"), "{not json"); + expect(await cancelIt("j-abcdef0123")).toBe(EXIT.USAGE); + expect(errored()[0]).toContain("The record j-abcdef0123.json is damaged"); + expect(errored()[0]).toContain(join(dir, "j-abcdef0123.json")); + }); + + it("rethrows a read failure that is not a corrupt record", async () => { + // A directory where the record file should be: readFileSync fails with EISDIR, not ENOENT. + mkdirSync(join(dir, "j-abcdef0123.json")); + await expect(cancelIt("j-abcdef0123")).rejects.toThrow(/EISDIR/); + }); + + it("exits 2 with the key copy and a Next line when ORA_API_KEY is unset", async () => { + const { record } = launchedSet(); + vi.stubEnv("ORA_API_KEY", ""); + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + + expect(await cancelIt(record.id)).toBe(EXIT.USAGE); + + expect(errored()[0]).toBe(GET_A_KEY); + expect(errored()[1]).toMatch(new RegExp(`^Next: ${KEY_NEXT} ${record.id} {2}# `)); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("exits 2 without a call when the platform URL would carry the key over plain http", async () => { + const { record } = launchedSet(); + vi.stubEnv("ORA_PLATFORM_URL", "http://collector.example"); + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + + expect(await cancelIt(record.id)).toBe(EXIT.USAGE); + + expect(errored()[0]).toMatch(/ORA_PLATFORM_URL.*https/); + expect(errored()[1]).toBe(`Next: ax journey cancel ${record.id} # ${KEY_DESTINATION_WHY}`); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("says nothing started when the record has no run ids, and makes no call (exit 0)", async () => { + const record = newSet(); + appendJourney(record.id, entry(1, []), dir); + vi.stubEnv("ORA_API_KEY", ""); + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + + expect(await cancelIt(record.id)).toBe(EXIT.OK); + + expect(logged()).toEqual([`Set ${record.id} started no runs, so there is nothing to cancel.`]); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(stdoutWrites()).toBe(""); + }); + + it("--json with no run ids prints an empty report as the one document", async () => { + const record = newSet(); + expect(await cancelIt(record.id, true)).toBe(EXIT.OK); + expect(JSON.parse(stdoutWrites())).toEqual({ + requested: 0, + canceled: [], + alreadyTerminal: [], + skipped: [], + }); + expect(console.log).not.toHaveBeenCalled(); + expect(errored()).toEqual([`Set ${record.id} started no runs, so there is nothing to cancel.`]); + }); +}); + +describe("journeySetCancelCommand against Studio", () => { + it("sends every recorded run id in one call and prints the four counts (exit 0)", async () => { + const { record, runIds } = launchedSet(); + const mock = studioMock(() => asJson(cancel)); + + expect(await cancelIt(record.id)).toBe(EXIT.OK); + + expect(mock.calls).toEqual([ + { url: `${BASE}/studio/v1/runs/cancel`, method: "POST", body: { runIds } }, + ]); + expect(logged()).toEqual([ + `Asked to cancel 3 runs for set ${record.id}.`, + " canceled 1", + " already finished 1", + " skipped 1", + `Next: ax journey view ${record.id} # read the report for what finished`, + ]); + expect(errored()).toEqual([]); + expect(stdoutWrites()).toBe(""); + }); + + it("--json prints Studio's answer as the fixture, with every person line on stderr", async () => { + const { record } = launchedSet(); + studioMock(() => asJson(cancel)); + + expect(await cancelIt(record.id, true)).toBe(EXIT.OK); + + expect(JSON.parse(stdoutWrites())).toEqual(cancel); + expect(console.log).not.toHaveBeenCalled(); + expect(errored()).toEqual([ + `Next: ax journey view ${record.id} --json # read the report for what finished`, + ]); + }); + + it("names the missing scope and prints the key copy on a 403 (exit 2)", async () => { + const { record } = launchedSet(); + const missing = { ...missingScope403, details: { missing: ["runs:write"] } }; + studioMock(() => asJson(missing, 403)); + + expect(await cancelIt(record.id)).toBe(EXIT.USAGE); + + expect(errored()[0]).toBe(`Your API key is missing this scope: runs:write.\n${GET_A_KEY}`); + expect(errored()[1]).toMatch(new RegExp(`^Next: ${KEY_NEXT} ${record.id} {2}# `)); + }); + + it("--json on a 403 prints Studio's error body untouched", async () => { + const { record } = launchedSet(); + const missing = { ...missingScope403, details: { missing: ["runs:write"] } }; + studioMock(() => asJson(missing, 403)); + + expect(await cancelIt(record.id, true)).toBe(EXIT.USAGE); + + expect(JSON.parse(stdoutWrites())).toEqual(missing); + expect(console.log).not.toHaveBeenCalled(); + }); + + it("prints the product message on a PRODUCT_NOT_GRANTED 403 (exit 2)", async () => { + const { record } = launchedSet(); + studioMock(() => asJson(PRODUCT_403, 403)); + + expect(await cancelIt(record.id)).toBe(EXIT.USAGE); + + expect(errored()[0]).toBe(PRODUCT_NOT_GRANTED_MESSAGE); + expect(errored()[1]).toMatch(new RegExp(`^Next: ${KEY_NEXT} ${record.id} {2}# `)); + }); + + it("exits 3 with the same cancel call as the Next line on a 500", async () => { + const { record } = launchedSet(); + studioMock(() => asJson(SERVER_500, 500)); + + expect(await cancelIt(record.id)).toBe(EXIT.API); + + expect(errored()[0]).toMatch(/^Could not cancel the runs: /); + expect(errored()[1]).toBe( + `Next: ax journey cancel ${record.id} # retry; runs that already stopped are reported, not canceled twice`, + ); + expect(stdoutWrites()).toBe(""); + }); + + it("--json on a 500 keeps stdout to Studio's error body and the retry keeps --json", async () => { + const { record } = launchedSet(); + studioMock(() => asJson(SERVER_500, 500)); + + expect(await cancelIt(record.id, true)).toBe(EXIT.API); + + expect(JSON.parse(stdoutWrites())).toEqual(SERVER_500); + expect(errored()[1]).toContain(`Next: ax journey cancel ${record.id} --json # retry`); + }); + + it("exits 3 with the envelope under --json when Studio cannot be reached", async () => { + const { record } = launchedSet(); + studioMock(() => { + throw new TypeError("fetch failed"); + }); + + expect(await cancelIt(record.id, true)).toBe(EXIT.API); + + expect(JSON.parse(stdoutWrites())).toEqual({ + error: { + code: "API_UNREACHABLE", + message: expect.stringContaining("could not reach ora at"), + next: `ax journey cancel ${record.id} --json`, + }, + }); + }); + + it("exits 3 when the key exchange fails", async () => { + const { record } = launchedSet(); + vi.stubGlobal( + "fetch", + vi.fn(async () => asJson({ error: "Unauthorized" }, 401)), + ); + expect(await cancelIt(record.id)).toBe(EXIT.API); + expect(errored()[1]).toMatch(/^Next: ax journey cancel /); + }); + + it("lets an error that is not the platform's propagate", async () => { + const { record } = launchedSet(); + const broken = () => ({ + cancelRuns: async () => { + throw new TypeError("a bug in ax"); + }, + }); + await expect(cancelIt(record.id, false, { client: broken })).rejects.toThrow("a bug in ax"); + }); + + it("blanks control characters in what Studio says before it reaches the terminal", async () => { + const { record } = launchedSet(); + studioMock(() => asJson({ error: "bad\u001b[31m news", code: "internal" }, 500)); + expect(await cancelIt(record.id)).toBe(EXIT.API); + expect(errored()[0]).not.toContain("\u001b"); + }); +}); diff --git a/src/commands/journey-set-records.ts b/src/commands/journey-set-records.ts new file mode 100644 index 0000000..bdfa1dc --- /dev/null +++ b/src/commands/journey-set-records.ts @@ -0,0 +1,207 @@ +import { PlatformApiError, platformKeyProblem } from "../api/platform"; +import { createStudioClient, isStudioFailure, StudioApiError } from "../api/studio"; +import type { CancelResult, StudioClient } from "../api/studio-types"; +import { listRecords } from "../journey-set/record"; +import { + GET_A_KEY, + isProductNotGranted, + KEY_DESTINATION_WHY, + PRODUCT_NOT_GRANTED_MESSAGE, +} from "../journey-set/render-preview"; +import { failureCode, pickUpRecord, SET_USAGE_CALL } from "../journey-set/report-flow"; +import { reportError } from "../report/json-error"; +import { axCall, type NextStep, printNext } from "../report/next-line"; +import { plain } from "../ui/ansi"; +import { EXIT } from "./audit"; + +// `ax journey list` and `ax journey cancel`: the two commands that work from the local record +// in ~/.ax/journeys/. `list` reads only that directory and asks no server. `cancel` sends every +// run id the record holds to Studio in one call and prints Studio's answer: counts for a person, +// the body untouched under --json (a single server payload, so invariant 4 applies). + +export interface JourneySetListInput { + json: boolean; + /** Record directory; tests only, there is no flag for it. */ + setsDir?: string; +} + +export interface JourneySetCancelInput { + id: string; + json: boolean; + /** Record directory; tests only, there is no flag for it. */ + setsDir?: string; + /** Builds the Studio client; tests inject one over a mocked fetch. */ + client?: () => Pick; +} + +const noun = (n: number, word: string) => `${n} ${word}${n === 1 ? "" : "s"}`; + +function jsonWords(json: boolean): string[] { + return json ? ["--json"] : []; +} + +/** After a result: stdout for a person's report, stderr when stdout carries the JSON document. */ +function next(step: NextStep, json: boolean): void { + printNext(step, { stderr: json }); +} + +// --- list --- + +export function journeySetListCommand(input: JourneySetListInput): number { + const { records, skipped } = listRecords(input.setsDir); + if (skipped.length > 0) { + console.error( + `Skipped ${noun(skipped.length, "file")} that ${skipped.length === 1 ? "is" : "are"} not a readable set record: ${skipped.map(plain).join(", ")}`, + ); + } + if (input.json) process.stdout.write(`${JSON.stringify({ records, skipped }, null, 2)}\n`); + + if (records.length === 0) { + if (!input.json) { + console.log( + skipped.length > 0 + ? "No readable sets of journeys on this machine." + : "No sets of journeys on this machine yet.", + ); + } + next( + { + call: SET_USAGE_CALL, + why: "start a set of journeys; it is recorded here", + }, + input.json, + ); + return EXIT.OK; + } + + if (!input.json) { + const site = Math.max(...records.map((r) => plain(r.site).length)); + for (const r of records) { + const counts = `${noun(r.journeys, "journey")}, ${noun(r.runs, "run")}`; + console.log(`${r.id} ${plain(r.createdAt)} ${plain(r.site).padEnd(site)} ${counts}`); + } + } + next( + { call: axCall(["journey", "view", records[0].id]), why: "read the newest set's report" }, + input.json, + ); + return EXIT.OK; +} + +// --- cancel --- + +function fail( + code: string, + message: string, + step: NextStep, + exit: number, + options: { json: boolean; served?: unknown }, +): number { + reportError({ code, message, next: step }, options); + return exit; +} + +/** The scopes a 403 named, in one sentence. */ +function missingScopesMessage(scopes: readonly string[]): string { + const names = scopes.map((scope) => plain(scope)).join(", "); + const lead = `Your API key is missing ${scopes.length === 1 ? "this scope" : "these scopes"}: ${names}.`; + return `${lead}\n${GET_A_KEY}`; +} + +function platformFailure(error: unknown, id: string, json: boolean): number { + const retry: NextStep = { + call: axCall(["journey", "cancel", id, ...jsonWords(json)]), + why: "retry; runs that already stopped are reported, not canceled twice", + }; + const keyStep = (why: string): NextStep => ({ + call: `ORA_API_KEY= ${retry.call}`, + why, + }); + const served = error instanceof PlatformApiError ? error.payload : undefined; + if (isProductNotGranted(error)) { + const step = keyStep("run it again with a key made in your Front account settings"); + return fail("PRODUCT_NOT_GRANTED", PRODUCT_NOT_GRANTED_MESSAGE, step, EXIT.USAGE, { + json, + served, + }); + } + if (error instanceof StudioApiError && error.status === 403 && error.missingScopes.length > 0) { + return fail( + error.code || "INSUFFICIENT_SCOPE", + missingScopesMessage(error.missingScopes), + keyStep("run it again with a key that has the scope"), + EXIT.USAGE, + { json, served }, + ); + } + const message = `Could not cancel the runs: ${plain(error instanceof Error ? error.message : String(error))}`; + return fail(failureCode(error), message, retry, EXIT.API, { json, served }); +} + +function printCounts(id: string, result: CancelResult): void { + const rows: Array<[string, number]> = [ + ["canceled", result.canceled.length], + ["already finished", result.alreadyTerminal.length], + ["skipped", result.skipped.length], + ]; + console.log(`Asked to cancel ${noun(result.requested, "run")} for set ${id}.`); + for (const [label, n] of rows) console.log(` ${label.padEnd(16)} ${n}`); +} + +export async function journeySetCancelCommand(input: JourneySetCancelInput): Promise { + const { id, json } = input; + const loaded = pickUpRecord(id, json, input.setsDir); + if (!loaded) return EXIT.USAGE; + + const runIds = loaded.journeys.flatMap((journey) => journey.runIds); + if (runIds.length === 0) { + // Nothing started, so there is no server answer to pass through: --json gets an empty report. + const none: CancelResult = { requested: 0, canceled: [], alreadyTerminal: [], skipped: [] }; + if (json) process.stdout.write(`${JSON.stringify(none, null, 2)}\n`); + console[json ? "error" : "log"](`Set ${id} started no runs, so there is nothing to cancel.`); + return EXIT.OK; + } + + if (!process.env.ORA_API_KEY) { + return fail( + "MISSING_API_KEY", + GET_A_KEY, + { + call: `ORA_API_KEY= ${axCall(["journey", "cancel", id, ...jsonWords(json)])}`, + why: "canceling runs needs your Front API key", + }, + EXIT.USAGE, + { json }, + ); + } + const refused = platformKeyProblem(); + if (refused !== undefined) { + const call = axCall(["journey", "cancel", id, ...jsonWords(json)]); + const next = { call, why: KEY_DESTINATION_WHY }; + return fail("KEY_DESTINATION_REFUSED", refused, next, EXIT.USAGE, { json }); + } + + let result: CancelResult; + try { + const client = input.client ?? (() => createStudioClient()); + result = await client().cancelRuns(runIds); + } catch (error) { + // Only the platform's failures are reported here; a bug in ax propagates. + if (!isStudioFailure(error)) throw error; + return platformFailure(error, id, json); + } + + if (json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + } else { + printCounts(id, result); + } + next( + { + call: axCall(["journey", "view", id, ...jsonWords(json)]), + why: "read the report for what finished", + }, + json, + ); + return EXIT.OK; +} diff --git a/src/commands/journey-set.test.ts b/src/commands/journey-set.test.ts new file mode 100644 index 0000000..f213721 --- /dev/null +++ b/src/commands/journey-set.test.ts @@ -0,0 +1,810 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import agentsFixture from "../api/__fixtures__/studio/agents.json"; +import quota402 from "../api/__fixtures__/studio/error-402-quota.json"; +import missingScope403 from "../api/__fixtures__/studio/error-403-missing-scope.json"; +import created from "../api/__fixtures__/studio/journey-create.json"; +import createdPartial from "../api/__fixtures__/studio/journey-create-partial.json"; +import detailFinal from "../api/__fixtures__/studio/journey-detail-final.json"; +import detailRunning from "../api/__fixtures__/studio/journey-detail-running.json"; +import trajectory from "../api/__fixtures__/studio/trajectory.json"; +import usageFixture from "../api/__fixtures__/studio/usage.json"; +import { createStudioClient } from "../api/studio"; +import { STUDIO_SCOPES } from "../api/studio-types"; +import { listRecords, readRecord, type SetRecord } from "../journey-set/record"; +import { + confirmPrompt, + GET_A_KEY, + KEY_DESTINATION_WHY, + PRODUCT_NOT_GRANTED_MESSAGE, +} from "../journey-set/render-preview"; +import { NEED_LEVEL_NOTE } from "../journey-set/render-report"; +import { MISSING_DOMAIN, parseSetSpec } from "../journey-set/spec"; +import { EXIT } from "./audit"; +import { + type JourneySetRunInput, + type JourneySetViewInput, + journeySetRunCommand, + journeySetViewCommand, +} from "./journey-set"; + +// Every case drives the command through a real createStudioClient over a mocked fetch that answers +// from src/api/__fixtures__/studio/ (provenance in src/api/studio.test.ts), the key exchange +// included, and writes real records to a temp dir. The clock and the sleep are injected, so no +// case waits. Bodies with no fixture file are the named deltas below. + +const BASE = "https://platform.test"; +const KEY = "ora_sk_local"; +const SITE = "linear.app"; +/** Names neither the domain nor the brand, so the need level runs: 2 levels x 2 agents, 4 runs. */ +const GOAL = "Find how to create an issue from the API"; +/** The two agents agents.json offers, the pair journey-create.json answers for. */ +const AGENTS = "claude-code/claude-haiku-4-5-20251001,claude-code/claude-sonnet-5-5"; +const RUN_CALL = `ax journey '${GOAL}' --domain linear.app --agents ${AGENTS} --levels`; +const JOURNEY = created.data.journeyId; + +const asJson = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); + +/** A token shaped like the key exchange mints for an ora_sk_ key, as studio.test.ts builds it. */ +function jwt(claims: Record): string { + const part = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url"); + return [part({ alg: "RS256", typ: "JWT" }), part(claims), "signature"].join("."); +} + +type Reply = () => Response | Promise; + +const grantOf = + (scopes: readonly string[]): Reply => + () => + asJson({ token: jwt({ type: "api_key", sub: "key:k1", scopes }), expiresIn: 900 }); + +/** The nth create of a set: journey-create.json with its journey and run ids made unique. */ +function createdN(n: number) { + const runIds = created.data.runIds.map((id) => `${id}-${n}`); + return { ...created, data: { ...created.data, journeyId: `${JOURNEY}-${n}`, runIds } }; +} + +/** A read of one of those journeys: the detail fixture under that create's ids. */ +function readOf(detail: D, id: string) { + const suffix = id.slice(JOURNEY.length); + const agents = detail.data.agents.map((run) => ({ ...run, runId: `${run.runId}${suffix}` })); + return asJson({ ...detail, data: { ...detail.data, id, agents } }); +} + +const ok = + (n: number): Reply => + () => + asJson(createdN(n), 201); +// Studio's own 502 on create, nothing started: launch.test.ts's delta on journey-create-partial.json. +const NONE_STARTED_502 = { + ...createdPartial, + success: false, + data: { + ...createdPartial.data, + runIds: [], + streamUrls: [], + failed: [ + { ...createdPartial.data.failed[0], model: "claude-haiku-4-5-20251001" }, + createdPartial.data.failed[0], + ], + }, +}; +const studio502: Reply = () => asJson(NONE_STARTED_502, 502); +const gateway502: Reply = () => new Response("bad gateway", { status: 502 }); +const quota: Reply = () => asJson(quota402, 402); +// A plain Studio error body, and the 403 with its code swapped, as set-records.test.ts has them. +const SERVER_500 = { error: "boom", code: "internal" }; +const PRODUCT_403 = { error: "Forbidden", code: "PRODUCT_NOT_GRANTED" }; + +interface Platform { + /** The key exchange; a token with every scope a set needs by default. */ + grant?: Reply; + agents?: Reply; + usage?: Reply; + /** One reply per create, in turn; by default the nth create answers createdN(n). */ + creates?: Reply[]; + /** The nth read (from 1) of one journey; the final detail by default. */ + journey?: (id: string, n: number) => Response; + /** Every run's trajectory read; the recorded trajectory by default. */ + trajectory?: Reply; +} + +function platform(script: Platform = {}) { + const calls: string[] = []; + const creates: Array> = []; + const reads = new Map(); + vi.stubGlobal( + "fetch", + vi.fn(async (url: string | URL, init: RequestInit = {}) => { + const method = init.method ?? "GET"; + const path = String(url).slice(BASE.length).split("?")[0]; + calls.push(`${method} ${path}`); + if (path === "/auth/token") return (script.grant ?? grantOf(STUDIO_SCOPES))(); + if (path === "/studio/v1/agents") return (script.agents ?? (() => asJson(agentsFixture)))(); + if (path === "/studio/v1/usage") return (script.usage ?? (() => asJson(usageFixture)))(); + if (method === "POST" && path === "/studio/v1/journeys") { + creates.push(JSON.parse(String(init.body))); + const n = creates.length; + const reply = script.creates ? script.creates[n - 1] : ok(n); + if (!reply) throw new Error(`create ${n} is past the script`); + return reply(); + } + const journey = /^\/studio\/v1\/journeys\/([^/]+)$/.exec(path); + if (journey) { + const id = decodeURIComponent(journey[1]); + const n = (reads.get(id) ?? 0) + 1; + reads.set(id, n); + return (script.journey ?? ((at: string) => readOf(detailFinal, at)))(id, n); + } + if (/^\/studio\/v1\/runs\/[^/]+\/trajectory$/.test(path)) { + return (script.trajectory ?? (() => asJson(trajectory)))(); + } + throw new Error(`unrouted ${method} ${path}`); + }), + ); + const journeyReads = () => calls.filter((call) => call.startsWith("GET /studio/v1/journeys/")); + return { calls, creates, journeyReads }; +} + +const client = () => createStudioClient({ apiKey: KEY, baseUrl: BASE }); +let clock = 0; +const now = () => clock; +const sleep = async (ms: number) => { + clock += ms; +}; + +let dir: string; +let listeners: number; +const tty = { + stdin: Object.getOwnPropertyDescriptor(process.stdin, "isTTY"), + stderr: Object.getOwnPropertyDescriptor(process.stderr, "isTTY"), +}; + +function setTerminal(value: boolean) { + Object.defineProperty(process.stdin, "isTTY", { value, configurable: true }); + Object.defineProperty(process.stderr, "isTTY", { value, configurable: true }); +} + +function restore(stream: NodeJS.ReadStream | NodeJS.WriteStream, prior?: PropertyDescriptor) { + if (prior) Object.defineProperty(stream, "isTTY", prior); + else Reflect.deleteProperty(stream, "isTTY"); +} + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ax-set-")); + clock = Date.parse("2026-11-13T14:00:00.000Z"); + listeners = process.listenerCount("SIGINT"); + setTerminal(false); + vi.stubEnv("ORA_API_KEY", KEY); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); +}); + +afterEach(() => { + expect(process.listenerCount("SIGINT")).toBe(listeners); + process.exitCode = undefined; + restore(process.stdin, tty.stdin); + restore(process.stderr, tty.stderr); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + rmSync(dir, { recursive: true, force: true }); +}); + +// biome-ignore lint/suspicious/noControlCharactersInRegex: strips the color codes picocolors may add +const COLOR = /\u001b\[[0-9;]*m/g; +const text = (calls: unknown[][]) => calls.map((call) => String(call[0]).replace(COLOR, "")); +/** One entry per printed line: a message of several lines is printed as one call. */ +const printed = (calls: unknown[][]) => text(calls).flatMap((call) => call.split("\n")); +/** stdout as written: console.log lines, then raw writes (the --json document). */ +const stdout = () => [ + ...printed(vi.mocked(console.log).mock.calls), + ...text(vi.mocked(process.stdout.write).mock.calls), +]; +/** stderr as written: console.error lines, then raw writes (the live status line). */ +const stderr = () => [ + ...printed(vi.mocked(console.error).mock.calls), + ...text(vi.mocked(process.stderr.write).mock.calls), +]; +/** The one --json document; fails when anything else reached stdout. */ +function jsonDocument(): Record { + expect(console.log).not.toHaveBeenCalled(); + expect(process.stdout.write).toHaveBeenCalledTimes(1); + return JSON.parse(String(vi.mocked(process.stdout.write).mock.calls[0][0])); +} + +const nextLine = (call: string, why: string) => `Next: ${call} # ${why}`; +const keyed = (call: string) => + nextLine( + `ORA_API_KEY= ${call}`, + "a Front API key with the scopes a set of journeys needs", + ); +const viewLine = (id: string, json = "") => + nextLine(`ax journey view ${id}${json}`, "picks the set up and prints its report"); +const cancelLine = (id: string) => + nextLine(`ax journey cancel ${id}`, "stops the runs that started"); +const auditLine = (json = "") => + nextLine( + `ax audit linear.app${json}`, + "the readiness audit: which checks explain what the agents hit", + ); + +const run = (input: Partial = {}) => + journeySetRunCommand({ + ...{ site: SITE, goals: [GOAL], agents: AGENTS, levels: true, yes: true, json: false }, + ...{ setsDir: dir, client, now, sleep }, + ...input, + }); +const view = (id: string, input: Partial = {}) => + journeySetViewCommand({ id, json: false, setsDir: dir, client, now, sleep, ...input }); + +async function exits(command: Promise, code: number) { + expect(await command).toBe(code); +} + +/** The one record the run wrote, read back from disk. */ +function theRecord(): SetRecord { + const { records } = listRecords(dir); + expect(records).toHaveLength(1); + const record = readRecord(records[0].id, dir); + if (!record) throw new Error("the record is gone"); + return record; +} + +const noRecord = () => expect(listRecords(dir)).toEqual({ records: [], skipped: [] }); + +describe("a set from ax journey, before anything launches", () => { + it("refuses a spec with no goal, exit 2, before any call", async () => { + const mock = platform(); + const parsed = parseSetSpec({ site: SITE, goals: [], levels: true }); + if (parsed.ok) throw new Error("an empty goal list parsed"); + + await exits(run({ goals: [] }), EXIT.USAGE); + + expect(mock.calls).toEqual([]); + expect(stdout()).toEqual([]); + expect(stderr()).toEqual([ + parsed.error, + nextLine("ax journey --domain --levels", "with the value named above fixed"), + ]); + }); + + it("refuses a set with no --domain before any call, with the call to complete, exit 2", async () => { + const mock = platform(); + + await exits(run({ site: undefined }), EXIT.USAGE); + + expect(mock.calls).toEqual([]); + expect(stderr()).toEqual([ + MISSING_DOMAIN, + nextLine( + `ax journey '${GOAL}' --domain --agents ${AGENTS} --levels --yes`, + "with the site the agents work on", + ), + ]); + noRecord(); + }); + + it("without ORA_API_KEY prints the key copy and the same call keyed, exit 2", async () => { + const mock = platform(); + vi.stubEnv("ORA_API_KEY", ""); + + await exits(run(), EXIT.USAGE); + + expect(mock.calls).toEqual([]); + expect(stdout()).toEqual([]); + expect(stderr()).toEqual([GET_A_KEY, keyed(`${RUN_CALL} --yes`)]); + }); + + it("refuses to send the key to a platform URL over plain http, before any call, exit 2", async () => { + const mock = platform(); + vi.stubEnv("ORA_PLATFORM_URL", "http://collector.example"); + + await exits(run(), EXIT.USAGE); + + expect(mock.calls).toEqual([]); + expect(stdout()).toEqual([]); + const [message, next] = stderr(); + expect(message).toMatch(/ORA_PLATFORM_URL.*https/); + expect(next).toBe(nextLine(`${RUN_CALL} --yes`, KEY_DESTINATION_WHY)); + noRecord(); + }); + + it("refuses a key whose token lacks scopes without reading the catalog, exit 2", async () => { + const mock = platform({ grant: grantOf(["journeys:read", "runs:read", "intents:read"]) }); + + await exits(run(), EXIT.USAGE); + + expect(mock.calls).toEqual(["POST /auth/token"]); + expect(stdout()).toEqual([]); + expect(stderr()).toEqual([ + "Your API key is missing these scopes: journeys:write, runs:write.", + GET_A_KEY, + keyed(`${RUN_CALL} --yes`), + ]); + noRecord(); + }); + + it("names the scope a 403 on the catalog read says is missing, exit 2", async () => { + platform({ agents: () => asJson(missingScope403, 403) }); + + await exits(run(), EXIT.USAGE); + + expect(stderr()).toEqual([ + "Your API key is missing these scopes: journeys:write.", + GET_A_KEY, + keyed(`${RUN_CALL} --yes`), + ]); + noRecord(); + }); + + it("passes a PRODUCT_NOT_GRANTED body through under --json, exit 2", async () => { + platform({ agents: () => asJson(PRODUCT_403, 403) }); + + await exits(run({ json: true }), EXIT.USAGE); + + expect(jsonDocument()).toEqual(PRODUCT_403); + expect(stderr()).toEqual([PRODUCT_NOT_GRANTED_MESSAGE, keyed(`${RUN_CALL} --yes --json`)]); + }); + + it("a 500 on the usage read exits 3 with the same call to retry", async () => { + platform({ usage: () => asJson(SERVER_500, 500) }); + + await exits(run(), EXIT.API); + + expect(stdout()).toEqual([]); + expect(stderr()).toEqual([ + "ora request failed (500 internal): boom", + nextLine(`${RUN_CALL} --yes`, "retry"), + ]); + noRecord(); + }); + + it("a key exchange that cannot connect exits 3 with the envelope under --json", async () => { + platform({ + grant: () => { + throw new TypeError("fetch failed"); + }, + }); + + await exits(run({ json: true }), EXIT.API); + + const message = "could not reach ora at https://platform.test: fetch failed"; + const call = `${RUN_CALL} --yes --json`; + expect(jsonDocument()).toEqual({ error: { code: "API_UNREACHABLE", message, next: call } }); + expect(stderr()).toEqual([message, nextLine(call, "retry")]); + noRecord(); + }); + + it("a key the exchange rejects exits 3 and names the key", async () => { + platform({ grant: () => asJson({}, 401) }); + + await exits(run(), EXIT.API); + + expect(stderr()).toEqual([ + "ora rejected the API key (401). Check ORA_API_KEY.", + keyed(`${RUN_CALL} --yes`), + ]); + }); + + it("over the allowance, the Next line carries the repeat that fits, exit 2", async () => { + const mock = platform(); + + await exits(run({ repeat: "10" }), EXIT.USAGE); + + expect(stderr()).toEqual([ + "This set needs 40 runs and your plan has 38 runs left this month (resets Dec 1).", + "9 repeats fit: 36 runs. Try --repeat 9.", + nextLine( + `${RUN_CALL} --repeat 9 --yes`, + "the largest repeat that fits what your plan has left", + ), + ]); + expect(mock.creates).toEqual([]); + noRecord(); + }); + + it("over the allowance when only --repeat made it a set, the Next line is one journey", async () => { + const usage = { ...usageFixture.data, runs: { used: 49, limit: 50, remaining: 1 } }; + const mock = platform({ usage: () => asJson({ ...usageFixture, data: usage }) }); + const agent = "claude-code/claude-haiku-4-5-20251001"; + + await exits(run({ agents: agent, levels: false, repeat: "5" }), EXIT.USAGE); + + expect(stderr()).toEqual([ + "This set needs 5 runs and your plan has 1 run left this month (resets Dec 1).", + "1 run fits. Without --repeat, ax runs it as one journey and draws it live.", + nextLine( + `ax journey '${GOAL}' --domain linear.app --agents ${agent}`, + "one run fits what your plan has left; without --repeat this is one journey", + ), + ]); + expect(mock.creates).toEqual([]); + noRecord(); + }); + + it("with no terminal and no --yes, prints the preview and refuses, exit 2", async () => { + const mock = platform(); + + await exits(run({ yes: false }), EXIT.USAGE); + + const lines = stderr(); + expect(lines[0]).toBe("This set uses 4 runs (1 goal x 2 levels x 2 agents x 1 repeat)."); + expect(lines).toContain("Your plan has 38 runs left this month (resets Dec 1)."); + expect(lines.slice(-2)).toEqual([ + "There is no terminal to confirm the launch in. Nothing was launched.", + nextLine(`${RUN_CALL} --yes`, "launches without asking"), + ]); + expect(stdout()).toEqual([]); + expect(mock.creates).toEqual([]); + noRecord(); + }); + + it("a no at the prompt launches nothing and writes no record, exit 0", async () => { + const mock = platform(); + setTerminal(true); + const ask = vi.fn(async () => "n"); + + await exits(run({ yes: false, ask }), EXIT.OK); + + expect(ask).toHaveBeenCalledWith(confirmPrompt(4)); + expect(stderr().slice(-2)).toEqual([ + "Nothing was launched.", + nextLine(RUN_CALL, "the same set, when you want to launch it"), + ]); + expect(stdout()).toEqual([]); + expect(mock.creates).toEqual([]); + noRecord(); + }); + + it("Ctrl-C at the prompt launches nothing, exit 130", async () => { + const mock = platform(); + setTerminal(true); + + await exits(run({ yes: false, ask: async () => null }), 130); + + expect(stderr().slice(-2)[0]).toBe("Nothing was launched."); + expect(mock.creates).toEqual([]); + noRecord(); + }); +}); + +describe("a set from ax journey, launched", () => { + it("writes the record, launches, follows and prints the report, exit 0", async () => { + let recordsAtFirstCreate: unknown; + const mock = platform({ + creates: [ + () => { + recordsAtFirstCreate = listRecords(dir).records.length; + return ok(1)(); + }, + ok(2), + ], + }); + setTerminal(true); + + await exits(run({ yes: false, ask: async () => "y" }), EXIT.OK); + + expect(recordsAtFirstCreate).toBe(1); + const record = theRecord(); + expect(record.journeys.map((journey) => journey.journeyId)).toEqual([ + `${JOURNEY}-1`, + `${JOURNEY}-2`, + ]); + expect(mock.creates.map((body) => body.goal)).toEqual([GOAL, `${GOAL}. Use Linear.`]); + expect(mock.creates[0]).toMatchObject({ targetDomain: SITE }); + + const out = stdout(); + expect(out[0]).toContain(`linear.app set ${record.id}`); + expect(out).toContain(NEED_LEVEL_NOTE); + expect(out.join("\n")).not.toContain("This report is not complete"); + expect(out.at(-1)).toBe(auditLine()); + + const err = stderr(); + expect(err).toContain( + `You can close the terminal; \`ax journey view ${record.id}\` picks it up.`, + ); + expect(err.join("")).toContain("2 of 2 journeys created"); + expect(err.join("")).toContain("0 queued, 0 running, 4 finished"); + }); + + it("without levels, sends each goal once as written and reports that level", async () => { + const mock = platform(); + + await exits(run({ levels: false }), EXIT.OK); + + expect(mock.creates.map((body) => body.goal)).toEqual([GOAL]); + const out = stdout(); + expect(out.some((line) => line.trim().startsWith("As written"))).toBe(true); + expect(out).not.toContain(NEED_LEVEL_NOTE); + expect(stderr()[0]).toBe("This set uses 2 runs (1 goal x 1 level x 2 agents x 1 repeat)."); + }); + + it("says on stderr when steps could not be read, since those runs carry no flags", async () => { + platform({ trajectory: () => asJson(SERVER_500, 500) }); + + await exits(run(), EXIT.OK); + + expect(stderr()).toContain( + "The steps of 4 runs could not be read, so the report does not flag them as blocked or never visited.", + ); + expect(stdout().at(-1)).toBe(auditLine()); + }); + + it("under --json, stdout carries the one report document", async () => { + platform(); + + await exits(run({ json: true }), EXIT.OK); + + const record = theRecord(); + const document = jsonDocument(); + expect(document).toMatchObject({ setId: record.id, site: SITE, complete: true }); + expect(document.ended).toBe("settled"); + expect(stderr().at(-1)).toBe(auditLine(" --json")); + }); + + it("after a 402 mid-launch, follows what started and reports it incomplete, exit 0", async () => { + platform({ creates: [ok(1), quota] }); + + await exits(run({ json: true }), EXIT.OK); + + expect(jsonDocument()).toMatchObject({ + complete: false, + stoppedReason: "quota", + notLaunched: 2, + }); + const err = stderr(); + expect(err).toContain( + "Your plan's run limit was reached, so launching stopped. 2 runs were not launched.", + ); + expect(err).toContain("Following the 2 runs that started."); + expect(err.at(-1)).toBe(auditLine(" --json")); + }); + + it("a gateway 502 stops launching: exit 3, the view and cancel lines, no following", async () => { + const mock = platform({ creates: [ok(1), gateway502] }); + + await exits(run(), EXIT.API); + + const { id } = theRecord(); + expect(mock.journeyReads()).toEqual([]); + expect(stdout()).toEqual([]); + expect(stderr().slice(-4)).toEqual([ + "The platform is likely down, so launching stopped. Runs that started keep running.", + "The last journey may have started its runs anyway; they will show in your account's journeys list.", + viewLine(id), + cancelLine(id), + ]); + }); + + it("three Studio 502s in a row stop launching with only the view line, exit 3", async () => { + const mock = platform({ creates: [studio502, studio502, studio502] }); + + await exits(run({ repeat: "2" }), EXIT.API); + + const record = theRecord(); + expect(record.stoppedReason).toBe("platform_down"); + expect(mock.journeyReads()).toEqual([]); + expect(stderr().slice(-2)).toEqual([ + "The platform is likely down, so launching stopped. Runs that started keep running.", + viewLine(record.id), + ]); + }); + + it("an error answer stops launching: exit 3, the error quoted, view and cancel", async () => { + const mock = platform({ creates: [ok(1), () => asJson(SERVER_500, 500)] }); + + await exits(run(), EXIT.API); + + const { id } = theRecord(); + expect(mock.journeyReads()).toEqual([]); + expect(stderr().slice(-4)).toEqual([ + "The platform answered with an error, so launching stopped: ora request failed (500 internal): boom", + "2 runs were not launched. Runs that started keep running.", + viewLine(id), + cancelLine(id), + ]); + }); + + it("Ctrl-C while launching: exit 130 with the view and cancel lines", async () => { + const mock = platform({ + creates: [ + () => { + process.emit("SIGINT"); + return ok(1)(); + }, + ], + }); + + await exits(run(), 130); + + const record = theRecord(); + expect(record.stoppedReason).toBe("interrupted"); + expect(mock.journeyReads()).toEqual([]); + expect(stdout()).toEqual([]); + expect(stderr().slice(-3)).toEqual([ + "Launching stopped. 2 runs were not launched. Runs that started keep running.", + viewLine(record.id), + cancelLine(record.id), + ]); + }); +}); + +describe("a set from ax journey, following", () => { + it("Ctrl-C while following: exit 130, the view line, no report", async () => { + const mock = platform({ + journey: (id) => { + process.emit("SIGINT"); + return readOf(detailRunning, id); + }, + }); + + await exits(run(), 130); + + const { id } = theRecord(); + expect(mock.calls.some((call) => call.endsWith("/trajectory"))).toBe(false); + expect(stdout()).toEqual([]); + expect(stderr().slice(-2)).toEqual([ + "Stopped following. Runs that started keep running.", + viewLine(id), + ]); + }); + + it("Ctrl-C in the round that settles every journey still prints the report, exit 0", async () => { + platform({ + journey: (id) => { + if (id.endsWith("-2")) process.emit("SIGINT"); + return readOf(detailFinal, id); + }, + }); + + await exits(run(), EXIT.OK); + + expect(stdout().join("\n")).not.toContain("This report is not complete"); + expect(stdout().at(-1)).toBe(auditLine()); + }); + + it("the wait limit ends following: the report says so, Next is view, exit 0", async () => { + platform({ journey: (id) => readOf(detailRunning, id) }); + + await exits(run({ wait: "1" }), EXIT.OK); + + const { id } = theRecord(); + const out = stdout(); + expect(out).toContain( + "This report is not complete: the wait limit ran out with 4 runs still going.", + ); + expect(out.at(-1)).toBe(viewLine(id)); + expect(stderr().join("")).toContain("2 queued, 2 running, 0 finished"); + }); + + it("reads failing after a good round: the last error, then the report, exit 0", async () => { + platform({ + journey: (id, n) => (n === 1 ? readOf(detailRunning, id) : asJson(SERVER_500, 500)), + }); + + await exits(run(), EXIT.OK); + + const { id } = theRecord(); + expect(stderr()).toContain("Reading from the platform kept failing."); + expect(stderr()).toContain("The last error: ora request failed (500 internal): boom"); + const out = stdout(); + expect(out).toContain( + "This report is not complete: the platform could not be read, so some runs may be missing.", + ); + expect(out.at(-1)).toBe(viewLine(id)); + }); + + it("reads failing from the first round: exit 3 with the view line and no report", async () => { + platform({ journey: () => asJson(SERVER_500, 500) }); + + await exits(run({ json: true }), EXIT.API); + + const { id } = theRecord(); + const message = [ + "Reading from the platform kept failing.", + "The last error: ora request failed (500 internal): boom", + ].join("\n"); + const call = `ax journey view ${id} --json`; + expect(jsonDocument()).toEqual({ error: { code: "internal", message, next: call } }); + expect(stderr().slice(-3)).toEqual([...message.split("\n"), viewLine(id, " --json")]); + }); +}); + +describe("ax journey view", () => { + it("picks a recorded set up and prints the same document the run printed, exit 0", async () => { + platform(); + await run({ json: true }); + const fromRun = jsonDocument(); + vi.mocked(process.stdout.write).mockClear(); + vi.mocked(console.error).mockClear(); + const mock = platform(); + + await exits(view(String(fromRun.setId), { json: true }), EXIT.OK); + + expect(jsonDocument()).toEqual(fromRun); + expect(mock.creates).toEqual([]); + expect(stderr().at(-1)).toBe(auditLine(" --json")); + }); + + it("refuses an id that is not a set id without reading anything, exit 2", async () => { + const mock = platform(); + + await exits(view("../etc/passwd"), EXIT.USAGE); + + expect(mock.calls).toEqual([]); + expect(stderr()).toEqual([ + '"../etc/passwd" is not a set id. Ids look like j-0123456789.', + nextLine("ax journey list", "see the sets recorded on this machine"), + ]); + }); + + it("refuses an id with no record on this machine, exit 2", async () => { + platform(); + + await exits(view("j-0000000000", { json: true }), EXIT.USAGE); + + const message = "No set j-0000000000 on this machine."; + expect(jsonDocument()).toEqual({ + error: { code: "UNKNOWN_SET", message, next: "ax journey list" }, + }); + expect(stderr()[0]).toBe(message); + }); + + it("names a damaged record and where it is, exit 2", async () => { + platform(); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "j-abcdef0123.json"), "{ not json"); + + await exits(view("j-abcdef0123"), EXIT.USAGE); + + const where = join(dir, "j-abcdef0123.json"); + expect(stderr()[0]).toBe( + `The record j-abcdef0123.json is damaged, so ax cannot read it. It is at ${where}.`, + ); + }); + + it("refuses a --wait that is not a whole number of minutes, exit 2", async () => { + platform(); + await run(); + const { id } = theRecord(); + vi.mocked(console.error).mockClear(); + + await exits(view(id, { wait: "soon" }), EXIT.USAGE); + + expect(stderr().at(-1)).toBe( + nextLine(`ax journey view ${id} --wait `, "a whole number of minutes"), + ); + }); + + it("without ORA_API_KEY prints the key copy and the view call keyed, exit 2", async () => { + platform(); + await run(); + const { id } = theRecord(); + vi.mocked(console.error).mockClear(); + vi.stubEnv("ORA_API_KEY", ""); + + await exits(view(id, { wait: "5" }), EXIT.USAGE); + + expect(stderr()).toEqual([GET_A_KEY, keyed(`ax journey view ${id} --wait 5`)]); + }); + + it("refuses to send the key to a platform URL over plain http, exit 2", async () => { + const mock = platform(); + await run(); + const { id } = theRecord(); + const before = mock.calls.length; + vi.mocked(console.error).mockClear(); + vi.stubEnv("ORA_PLATFORM_URL", "http://collector.example"); + + await exits(view(id, { wait: "5" }), EXIT.USAGE); + + expect(mock.calls).toHaveLength(before); + expect(stderr()[1]).toBe(nextLine(`ax journey view ${id} --wait 5`, KEY_DESTINATION_WHY)); + }); +}); diff --git a/src/commands/journey-set.ts b/src/commands/journey-set.ts new file mode 100644 index 0000000..e698ce5 --- /dev/null +++ b/src/commands/journey-set.ts @@ -0,0 +1,348 @@ +import { createInterface } from "node:readline"; +import { PlatformApiError, platformKeyProblem } from "../api/platform"; +import { createStudioClient, isStudioFailure, StudioApiError } from "../api/studio"; +import { STUDIO_SCOPES, type StudioClient } from "../api/studio-types"; +import { abortableSleep } from "../journey-set/follow"; +import { type PreviewResult, previewSet, type Refusal } from "../journey-set/preview"; +import { createRecord } from "../journey-set/record"; +import { + closeTerminalLine, + confirmPrompt, + GET_A_KEY, + isProductNotGranted, + KEY_DESTINATION_WHY, + PRODUCT_NOT_GRANTED_MESSAGE, + renderPreview, + renderRefusal, + renderRefusals, +} from "../journey-set/render-preview"; +import { + type FlowEnd, + failureCode, + followAndReport, + launchAndReport, + pickUpRecord, + SET_USAGE_CALL, +} from "../journey-set/report-flow"; +import { + type CellPlan, + parseSetSpec, + parseWaitMinutes, + planCells, + type SetSpec, +} from "../journey-set/spec"; +import { type CommandError, reportError } from "../report/json-error"; +import { axCall, type NextStep, placeholder, printNext, type Word } from "../report/next-line"; +import { plain } from "../ui/ansi"; +import { EXIT } from "./audit"; + +// A set of journeys from `ax journey` (src/commands/journey-cli.ts decides when the flags ask for +// one), and `ax journey view`. Run checks the spec and the key, reads the key's scopes, the agent +// catalog and the month's usage, prints the preview, asks, writes the record, then launches, +// follows and reports through src/journey-set/report-flow.ts; view picks a recorded set up and +// follows and reports it the same way. Lines for a person go to stderr; stdout carries only the +// report. + +/** Ctrl-C, as `ax audit` exits on it. */ +const INTERRUPTED = 130; + +/** Tests only; there is no flag for any of these. */ +interface Seams { + setsDir?: string; + /** Builds the Studio client; tests build one over a mocked fetch. */ + client?: () => StudioClient; + /** Following's clock and sleep. */ + now?: () => number; + sleep?: (ms: number, signal?: AbortSignal) => Promise; +} + +export interface JourneySetRunInput extends Seams { + /** --domain; a set refuses to run without one. */ + site?: string; + goals: readonly string[]; + agents?: string; + /** --levels as typed; --brand, --docs and --exact turn levels on without it. */ + levels: boolean; + brand?: string; + docs?: string; + exact?: string; + repeat?: string; + wait?: string; + yes: boolean; + json: boolean; + /** Asks on the terminal; null when Ctrl-C answers. Tests answer for the person. */ + ask?: (question: string) => Promise; +} + +export interface JourneySetViewInput extends Seams { + id: string; + wait?: string; + json: boolean; +} + +const NOTHING_LAUNCHED = "Nothing was launched."; + +function fail(error: CommandError, exit: number, json: boolean, served?: unknown): number { + reportError(error, { json, served }); + return exit; +} + +function messageOf(error: unknown): string { + return plain(error instanceof Error ? error.message : String(error)); +} + +function exitOf(end: FlowEnd): number { + if (end === "interrupted") return INTERRUPTED; + return end === "failed" ? EXIT.API : EXIT.OK; +} + +/** The Studio client and following's clock, real unless a test injected them. */ +function seams(input: Seams) { + const client = (input.client ?? (() => createStudioClient()))(); + return { client, now: input.now ?? Date.now, sleep: input.sleep ?? abortableSleep }; +} + +const keyStep = (call: string): NextStep => ({ + call: `ORA_API_KEY= ${call}`, + why: "a Front API key with the scopes a set of journeys needs", +}); + +/** Exit 2 before any call when the key must not go to the platform URL; undefined when it may. */ +function keyRefusal(call: string, json: boolean): number | undefined { + const problem = platformKeyProblem(); + if (problem === undefined) return undefined; + const next = { call, why: KEY_DESTINATION_WHY }; + return fail({ code: "KEY_DESTINATION_REFUSED", message: problem, next }, EXIT.USAGE, json); +} + +/** The key exchange or a read failed on the platform: exit 3, with the key named on a 401. */ +function platformFailure(error: unknown, call: string, json: boolean): number { + const rejected = error instanceof PlatformApiError && error.status === 401; + const served = error instanceof PlatformApiError ? error.payload : undefined; + const next = rejected ? keyStep(call) : { call, why: "retry" }; + const failure = { code: failureCode(error), message: messageOf(error), next }; + return fail(failure, EXIT.API, json, served); +} + +// --- run --- + +interface Swap { + site?: Word; + repeat?: Word; + agents?: Word; + yes?: true; +} + +/** The set as the person typed it, with one value swapped for a Next line. */ +function runCall(input: JourneySetRunInput, swap: Swap = {}) { + const flag = (name: string, value: Word | undefined) => + value === undefined ? [] : [name, value]; + return axCall([ + "journey", + ...input.goals, + ...flag("--domain", swap.site ?? input.site), + ...flag("--agents", swap.agents ?? input.agents), + ...(input.levels ? ["--levels"] : []), + ...flag("--brand", input.brand), + ...flag("--docs", input.docs), + ...flag("--exact", input.exact), + ...flag("--repeat", swap.repeat ?? input.repeat), + ...flag("--wait", input.wait), + ...(input.yes || swap.yes ? ["--yes"] : []), + ...jsonWords(input), + ]); +} + +/** The set's goal, site and agent as one journey: no --repeat, and no set-only flags. */ +function oneJourneyCall(input: JourneySetRunInput) { + return axCall([ + "journey", + ...input.goals, + ...(input.site === undefined ? [] : ["--domain", input.site]), + ...(input.agents === undefined ? [] : ["--agents", input.agents]), + ...jsonWords(input), + ]); +} + +function jsonWords(input: { json: boolean }): string[] { + return input.json ? ["--json"] : []; +} + +/** A 403 on the agents or usage read is about the key, so it exits 2; anything else exits 3. */ +function readFailure(error: unknown, input: JourneySetRunInput): number { + const next = keyStep(runCall(input)); + const served = error instanceof PlatformApiError ? error.payload : undefined; + if (isProductNotGranted(error)) { + const refusal = { code: "PRODUCT_NOT_GRANTED", message: PRODUCT_NOT_GRANTED_MESSAGE, next }; + return fail(refusal, EXIT.USAGE, input.json, served); + } + if (error instanceof StudioApiError && error.code === "INSUFFICIENT_SCOPE") { + const missing = STUDIO_SCOPES.filter((scope) => error.missingScopes.includes(scope)); + const lines = missing.length + ? renderRefusal({ kind: "missing_scopes", missing }) + : [messageOf(error), GET_A_KEY]; + const refusal = { code: error.code, message: lines.join("\n"), next }; + return fail(refusal, EXIT.USAGE, input.json, served); + } + return platformFailure(error, runCall(input), input.json); +} + +/** The scopes, then the catalog and usage unless a scope is missing; or the exit code reported. */ +async function preview( + studio: StudioClient, + input: JourneySetRunInput, + base: { spec: SetSpec; cells: CellPlan["cells"] }, +): Promise { + let scopes: string[]; + try { + scopes = await studio.scopes(); + } catch (error) { + if (!isStudioFailure(error)) throw error; + return platformFailure(error, runCall(input), input.json); + } + const lacking = STUDIO_SCOPES.some((scope) => !scopes.includes(scope)); + if (lacking) return previewSet({ ...base, scopes }); + try { + const agents = await studio.agents(); + return previewSet({ ...base, scopes, agents, usage: await studio.usage() }); + } catch (error) { + if (!isStudioFailure(error)) throw error; + return readFailure(error, input); + } +} + +/** The Next line for one refusal: the call with the value to change filled in or left to fill. */ +function refusalNext(lead: Refusal, input: JourneySetRunInput): NextStep { + switch (lead.kind) { + case "missing_scopes": + return keyStep(runCall(input)); + case "over_allowance": { + if (!lead.fits) return { call: runCall(input), why: "after your plan's runs reset" }; + if (lead.oneJourney) { + const why = "one run fits what your plan has left; without --repeat this is one journey"; + return { call: oneJourneyCall(input), why }; + } + const repeat = String(lead.fits.repeat); + const why = "the largest repeat that fits what your plan has left"; + return { call: runCall(input, { repeat }), why }; + } + case "over_cap": { + const repeat = placeholder("repeats"); + return { call: runCall(input, { repeat }), why: "fewer repeats, goals or agents" }; + } + default: { + const agents = placeholder("harness/model"); + return { call: runCall(input, { agents }), why: "an agent your plan offers" }; + } + } +} + +/** The typed answer; null on Ctrl-C. End of input answers like an empty line: the default, no. */ +function askOnTerminal(question: string): Promise { + const rl = createInterface({ input: process.stdin, output: process.stderr }); + return new Promise((resolve) => { + rl.once("SIGINT", () => { + process.stderr.write("\n"); + resolve(null); + rl.close(); + }); + rl.once("close", () => resolve("")); + rl.question(question, (answer) => { + resolve(answer); + rl.close(); + }); + }); +} + +/** True to launch; otherwise the exit code of a refusal or a no, already reported. */ +async function confirm(input: JourneySetRunInput, runs: number): Promise { + if (input.yes) return true; + if (!process.stdin.isTTY || !process.stderr.isTTY) { + const next = { call: runCall(input, { yes: true }), why: "launches without asking" }; + const message = `There is no terminal to confirm the launch in. ${NOTHING_LAUNCHED}`; + return fail({ code: "CONFIRM_NEEDS_TERMINAL", message, next }, EXIT.USAGE, input.json); + } + const answer = await (input.ask ?? askOnTerminal)(confirmPrompt(runs)); + if (answer !== null && /^y(es)?$/i.test(answer.trim())) return true; + console.error(NOTHING_LAUNCHED); + const why = "the same set, when you want to launch it"; + printNext({ call: runCall(input), why }, { stderr: true }); + return answer === null ? INTERRUPTED : EXIT.OK; +} + +export async function journeySetRunCommand(input: JourneySetRunInput): Promise { + const { json } = input; + const parsed = parseSetSpec(input); + if (!parsed.ok) { + const next = + input.site === undefined + ? { + call: runCall(input, { site: placeholder("site") }), + why: "with the site the agents work on", + } + : { call: SET_USAGE_CALL, why: "with the value named above fixed" }; + return fail({ code: "INVALID_SPEC", message: parsed.error, next }, EXIT.USAGE, json); + } + if (!process.env.ORA_API_KEY) { + const next = keyStep(runCall(input)); + return fail({ code: "MISSING_API_KEY", message: GET_A_KEY, next }, EXIT.USAGE, json); + } + const refused = keyRefusal(runCall(input), json); + if (refused !== undefined) return refused; + const { spec } = parsed; + const { cells, skipped } = planCells(spec); + const { client, now, sleep } = seams(input); + const decided = await preview(client, input, { spec, cells }); + if (typeof decided === "number") return decided; + if (!decided.ok) { + const { refusals } = decided; + const lead = refusals.find((r) => r.kind === "over_allowance" && r.fits) ?? refusals[0]; + const message = renderRefusals(refusals).join("\n"); + const refusal = { code: lead.kind.toUpperCase(), message, next: refusalNext(lead, input) }; + return fail(refusal, EXIT.USAGE, json); + } + for (const line of renderPreview({ spec, cells, skipped, preview: decided })) console.error(line); + console.error(""); + const go = await confirm(input, decided.totals.runs); + if (go !== true) return go; + + const agents = decided.agents.map(({ harness, model }) => ({ harness, model })); + const record = createRecord({ spec, agents, cells }, input.setsDir); + console.error(closeTerminalLine(record.id)); + const { waitMinutes } = spec; + const flow = { client, record, agents, waitMinutes, json, now, sleep, dir: input.setsDir }; + return exitOf(await launchAndReport({ ...flow, runCall: runCall(input) })); +} + +// --- view --- + +export async function journeySetViewCommand(input: JourneySetViewInput): Promise { + const { id, json } = input; + const record = pickUpRecord(id, json, input.setsDir); + if (!record) return EXIT.USAGE; + const wait = parseWaitMinutes(input.wait); + if ("error" in wait) { + const minutes = placeholder("minutes"); + const call = axCall(["journey", "view", id, "--wait", minutes, ...jsonWords(input)]); + const next = { call, why: "a whole number of minutes" }; + return fail({ code: "INVALID_WAIT", message: wait.error, next }, EXIT.USAGE, json); + } + const waitWords = input.wait ? ["--wait", input.wait] : []; + const call = axCall(["journey", "view", id, ...waitWords, ...jsonWords(input)]); + if (!process.env.ORA_API_KEY) { + const missing = { code: "MISSING_API_KEY", message: GET_A_KEY, next: keyStep(call) }; + return fail(missing, EXIT.USAGE, json); + } + const refused = keyRefusal(call, json); + if (refused !== undefined) return refused; + const { client, now, sleep } = seams(input); + try { + // The key exchange up front, so a rejected key fails here and not after three read rounds. + await client.scopes(); + } catch (error) { + if (!isStudioFailure(error)) throw error; + return platformFailure(error, call, json); + } + const flow = { client, record, waitMinutes: wait.value, json, now, sleep }; + return exitOf(await followAndReport(flow)); +} diff --git a/src/commands/journey.ts b/src/commands/journey.ts index ec01a34..5bbc886 100644 --- a/src/commands/journey.ts +++ b/src/commands/journey.ts @@ -1,12 +1,17 @@ import pc from "picocolors"; +import { KeyDestinationError } from "../api/key-destination"; import { type JourneyInsight, type JourneyRun, type JourneyStep, launchJourney, + PlatformApiError, type RunSpend, type RunSummary, } from "../api/platform"; +import { KEY_DESTINATION_WHY } from "../journey-set/render-preview"; +import { type CommandError, reportError } from "../report/json-error"; +import { axCall, type NextStep, placeholder, printNext } from "../report/next-line"; import { flow, squeeze, stdoutWidth } from "../ui/ansi"; import { buildJourneyTree, @@ -147,6 +152,65 @@ function journeyRunJson(run: JourneyRun): string { ); } +/** A caller reading JSON keeps reading JSON on the next call. */ +const jsonWords = (input: JourneyCommandInput) => (input.json ? ["--json"] : []); + +/** After a run: audit the site it targeted, or name one when the run had none. */ +function resultNext(input: JourneyCommandInput): NextStep { + if (input.domain) { + return { + call: axCall(["audit", input.domain, ...jsonWords(input)]), + why: "the readiness audit: which checks explain what the agent hit", + }; + } + return { + call: axCall(["journey", input.intent, "--domain", placeholder("domain"), ...jsonWords(input)]), + why: "name the site so the run can be traced back to it", + }; +} + +/** + * A failed launch. `launchJourney` reads the key from ORA_API_KEY, so an unset + * variable is the missing-key failure; anything else is retried as the same call. + */ +function launchFailure(input: JourneyCommandInput, cause: unknown): CommandError { + const words = [ + "journey", + input.intent, + ...(input.domain ? ["--domain", input.domain] : []), + "--harness", + input.harness, + ...(input.model ? ["--model", input.model] : []), + ...jsonWords(input), + ]; + const message = cause instanceof Error ? cause.message : String(cause); + if (!process.env.ORA_API_KEY) { + return { + code: "MISSING_API_KEY", + message, + next: { + call: `ORA_API_KEY= ${axCall(words)}`, + why: "a goal of your own runs on your Front account and needs a Front API key (see the README's Get a key); with no goal, ax journey --domain runs one of ora's curated tasks free", + }, + }; + } + if (cause instanceof KeyDestinationError) { + return { + code: "KEY_DESTINATION_REFUSED", + message, + next: { call: axCall(words), why: KEY_DESTINATION_WHY }, + }; + } + return { + code: "PLATFORM_ERROR", + message, + next: { + call: axCall(words), + why: "retry; if it fails the same way, check ORA_API_KEY and ORA_PLATFORM_URL", + }, + }; +} + // Exit codes: 0 run outcome "success" · 1 finished otherwise · 2 error. export async function journeyCommand(input: JourneyCommandInput): Promise { const request = { @@ -155,15 +219,24 @@ export async function journeyCommand(input: JourneyCommandInput): Promise { + const error = launchFailure(input, cause); + reportError(error, { + json: input.json, + display: (message) => `Journey failed: ${message}`, + served: cause instanceof PlatformApiError ? cause.payload : undefined, + }); + return 2; + }; if (input.json) { try { const run = await launchJourney(request); process.stdout.write(`${journeyRunJson(run)}\n`); + printNext(resultNext(input), { stderr: true }); return run.result?.outcome === "success" ? 0 : 1; } catch (cause) { - console.error(`Journey failed: ${cause instanceof Error ? cause.message : String(cause)}`); - return 2; + return fail(cause); } } @@ -210,10 +283,10 @@ export async function journeyCommand(input: JourneyCommandInput): Promise + String(url).endsWith("index.json") + ? new Response(JSON.stringify(index), { status: 200 }) + : new Response(bytes, { status: 200 }), + ); +} + +beforeEach(() => { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +const NEXT = "Next: ax audit # step one: score the site"; + +describe("learnCommand", () => { + it("prints the skill's Workflows section on stdout and the Next line on stderr", async () => { + vi.stubGlobal("fetch", registry(WITH_WORKFLOWS)); + expect(await learnCommand()).toBe(EXIT.OK); + const printed = String(vi.mocked(process.stdout.write).mock.calls[0][0]); + expect(printed.startsWith("## Workflows\n")).toBe(true); + expect(printed).toContain("3. `ax deep-journey {domain}` sends a real agent"); + expect(printed).not.toContain("Discovering Products"); + expect(console.error).toHaveBeenCalledWith(NEXT); + }); + + it("names the docs URL and exits 0 when the skill has no Workflows section", async () => { + vi.stubGlobal("fetch", registry(PUBLISHED)); + expect(await learnCommand({ baseUrl: "http://ora.localhost:1355" })).toBe(EXIT.OK); + expect(console.log).toHaveBeenCalledWith( + "The ora skill has no Workflows section yet. The ax workflow is in the CLI guide: http://ora.localhost:1355/docs/cli", + ); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith(NEXT); + }); + + it("fetches from ORA_API_URL", async () => { + vi.stubEnv("ORA_API_URL", "http://ora.localhost:1355"); + const fetchMock = registry(WITH_WORKFLOWS); + vi.stubGlobal("fetch", fetchMock); + expect(await learnCommand()).toBe(EXIT.OK); + expect(String(fetchMock.mock.calls[0][0])).toBe( + "http://ora.localhost:1355/.well-known/agent-skills/index.json", + ); + }); + + it("exits 3 with a retry when the registry cannot be reached", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => { + throw new Error("getaddrinfo ENOTFOUND"); + }), + ); + expect(await learnCommand()).toBe(EXIT.API); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith( + "Next: ax learn # retry, or read https://ora.ai/docs/cli", + ); + }); +}); diff --git a/src/commands/learn.ts b/src/commands/learn.ts new file mode 100644 index 0000000..a596149 --- /dev/null +++ b/src/commands/learn.ts @@ -0,0 +1,53 @@ +import { fetchSkill, resolveBase, SkillApiError, skillSection } from "../api/skills"; +import { reportError } from "../report/json-error"; +import { axCall, placeholder, printNext } from "../report/next-line"; +import { EXIT } from "./audit"; + +// `ax learn`: print the Workflows section of ora's own skill, fetched and +// digest-verified through the registry like `ax skill`, so the CLI teaches the +// same steps the published skill does and never a bundled copy. + +export const LEARN_SKILL = "ora"; +const SECTION = "Workflows"; + +export interface LearnCommandInput { + /** Base URL override; otherwise $ORA_API_URL, otherwise https://ora.ai. */ + baseUrl?: string; +} + +const FIRST_STEP = { + call: axCall(["audit", placeholder("domain")]), + why: "step one: score the site", +}; + +export async function learnCommand(input: LearnCommandInput = {}): Promise { + const docsUrl = `${resolveBase(input.baseUrl)}/docs/cli`; + let markdown: string; + try { + const { bytes } = await fetchSkill(LEARN_SKILL, input.baseUrl); + markdown = new TextDecoder().decode(bytes); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + reportError( + { + code: cause instanceof SkillApiError ? cause.code : "REGISTRY_ERROR", + message, + next: { call: axCall(["learn"]), why: `retry, or read ${docsUrl}` }, + }, + { json: false, display: (cleaned) => `ax learn: ${cleaned}` }, + ); + return EXIT.API; + } + + const section = skillSection(markdown, SECTION); + if (!section) { + console.log( + `The ${LEARN_SKILL} skill has no ${SECTION} section yet. The ax workflow is in the CLI guide: ${docsUrl}`, + ); + printNext(FIRST_STEP, { stderr: true }); + return EXIT.OK; + } + process.stdout.write(`${section}\n`); + printNext(FIRST_STEP, { stderr: true }); + return EXIT.OK; +} diff --git a/src/commands/skill.test.ts b/src/commands/skill.test.ts new file mode 100644 index 0000000..607479b --- /dev/null +++ b/src/commands/skill.test.ts @@ -0,0 +1,68 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { EXIT } from "./audit"; +import { skillCommand } from "./skill"; + +const unreachable = () => + vi.fn(async () => { + throw new Error("getaddrinfo ENOTFOUND"); + }); + +const stdout = () => + vi + .mocked(process.stdout.write) + .mock.calls.map((c) => String(c[0])) + .join(""); + +beforeEach(() => { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("skillCommand Next lines after a failure", () => { + it("--json keeps --json in the retry when the registry cannot be reached", async () => { + vi.stubGlobal("fetch", unreachable()); + expect(await skillCommand({ install: false, json: true })).toBe(EXIT.API); + expect(JSON.parse(stdout()).error).toMatchObject({ + code: "REGISTRY_UNREACHABLE", + next: "ax skill --json", + }); + expect(console.error).toHaveBeenLastCalledWith( + "Next: ax skill --json # retry; the registry may be briefly unreachable", + ); + }); + + it("--json keeps --json in the retry of a named skill", async () => { + vi.stubGlobal("fetch", unreachable()); + expect(await skillCommand({ name: "ora", install: false, json: true })).toBe(EXIT.API); + expect(JSON.parse(stdout()).error.next).toBe("ax skill ora --json"); + }); + + it("--json keeps --json in the listing an unknown skill points at", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ skills: [{ name: "ora", url: "/.well-known/agent-skills/ora/SKILL.md" }] }), + ), + ); + expect(await skillCommand({ name: "nope", install: false, json: true })).toBe(EXIT.USAGE); + expect(JSON.parse(stdout()).error).toMatchObject({ + code: "UNKNOWN_SKILL", + next: "ax skill --json", + }); + }); + + it("without --json the retry carries no --json and stdout stays empty", async () => { + vi.stubGlobal("fetch", unreachable()); + expect(await skillCommand({ install: false, json: false })).toBe(EXIT.API); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenLastCalledWith( + "Next: ax skill # retry; the registry may be briefly unreachable", + ); + }); +}); diff --git a/src/commands/skill.ts b/src/commands/skill.ts index cd0342b..4ab649c 100644 --- a/src/commands/skill.ts +++ b/src/commands/skill.ts @@ -2,6 +2,8 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import pc from "picocolors"; import { fetchSkill, fetchSkillIndex, SkillApiError } from "../api/skills"; +import { reportError } from "../report/json-error"; +import { axCall, printNext } from "../report/next-line"; import { EXIT } from "./audit"; export interface SkillCommandInput { @@ -35,8 +37,12 @@ export async function skillCommand(input: SkillCommandInput): Promise { try { if (!input.name) { const index = await fetchSkillIndex(); + const next = index[0] + ? { call: axCall(["skill", index[0].name, "--install"]), why: "or any name in the list" } + : { call: axCall(["learn"]), why: "the registry is empty; print the ax workflow instead" }; if (input.json) { process.stdout.write(`${JSON.stringify(index, null, 2)}\n`); + printNext(next, { stderr: true }); return EXIT.OK; } console.log(""); @@ -48,8 +54,7 @@ export async function skillCommand(input: SkillCommandInput): Promise { } console.log(""); } - console.log(pc.dim(" Install one with: ax skill --install")); - console.log(""); + printNext(next, { stderr: false }); return EXIT.OK; } @@ -61,15 +66,51 @@ export async function skillCommand(input: SkillCommandInput): Promise { writeFileSync(path, bytes); console.log(`Installed ${entry.name} to ${path}`); if (entry.digest) console.log(pc.dim(`verified ${entry.digest}`)); + printNext( + { + call: axCall(["learn"]), + why: "the workflow: scan, read the report, run a journey, read its result", + }, + { stderr: false }, + ); return EXIT.OK; } + // stdout is the SKILL.md bytes alone, so `ax skill > SKILL.md` works. process.stdout.write(bytes); + printNext( + { + call: axCall(["skill", entry.name, "--install"]), + why: `write it to ${DEFAULT_INSTALL_DIR}`, + }, + { stderr: true }, + ); return EXIT.OK; } catch (cause) { const message = cause instanceof Error ? cause.message : String(cause); - console.error(`Skill command failed: ${message}`); // An unknown name is a usage mistake; everything else is registry trouble. - if (cause instanceof SkillApiError && message.includes("unknown skill")) return EXIT.USAGE; - return EXIT.API; + const unknown = cause instanceof SkillApiError && cause.code === "UNKNOWN_SKILL"; + const json = input.json ? ["--json"] : []; + const retry = axCall([ + "skill", + ...(input.name ? [input.name] : []), + ...json, + ...(input.install ? ["--install"] : []), + ...(input.dir ? ["--dir", input.dir] : []), + ]); + reportError( + { + code: cause instanceof SkillApiError ? cause.code : "REGISTRY_ERROR", + message, + next: unknown + ? { call: axCall(["skill", ...json]), why: "list the skills the registry has" } + : { call: retry, why: "retry; the registry may be briefly unreachable" }, + }, + { + json: input.json, + display: (cleaned) => `Skill command failed: ${cleaned}`, + served: cause instanceof SkillApiError ? cause.payload : undefined, + }, + ); + return unknown ? EXIT.USAGE : EXIT.API; } } diff --git a/src/commands/webmcp-audit.test.ts b/src/commands/webmcp-audit.test.ts index d62eece..7b740de 100644 --- a/src/commands/webmcp-audit.test.ts +++ b/src/commands/webmcp-audit.test.ts @@ -49,6 +49,7 @@ beforeEach(() => { vi.mocked(ingestCapture).mockResolvedValue(AUDIT); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); }); afterEach(() => { @@ -206,3 +207,67 @@ describe("warnings the reader must not miss", () => { expect(err.mock.calls.map((c) => String(c[0])).join("\n")).not.toMatch(/WebMCPTesting/); }); }); + +describe("Next lines and the --json error envelope", () => { + const envelope = () => JSON.parse(String(vi.mocked(process.stdout.write).mock.calls[0][0])); + const stderrLines = () => vi.mocked(console.error).mock.calls.map((c) => String(c[0])); + + it("--json prints the envelope on a URL it cannot drive", async () => { + expect(await webmcpAuditCommand({ ...BASE, url: "not a url" })).toBe(EXIT.USAGE); + expect(envelope()).toEqual({ + error: { + code: "INVALID_TARGET", + message: 'Not an auditable URL: "not a url"', + next: "ax webmcp-audit http://localhost:3000 --json", + }, + }); + }); + + it("an outdated capture shim points at the upgrade, not a retry", async () => { + vi.mocked(ingestCapture).mockRejectedValue( + new WebmcpIngestError("upgrade ax", { code: "SHIM_VERSION_MISMATCH" }), + ); + expect(await webmcpAuditCommand({ ...BASE })).toBe(EXIT.API); + expect(envelope().error).toMatchObject({ + code: "SHIM_VERSION_MISMATCH", + next: "npm i -g ax@latest", + }); + }); + + it("--json prints ora's refusal body unchanged and keeps the upgrade Next line on stderr", async () => { + const served = { error: "capture shim 3 is no longer scored", code: "SHIM_VERSION_MISMATCH" }; + vi.mocked(ingestCapture).mockRejectedValue( + new WebmcpIngestError("upgrade ax", { code: "SHIM_VERSION_MISMATCH", payload: served }), + ); + expect(await webmcpAuditCommand({ ...BASE })).toBe(EXIT.API); + expect(process.stdout.write).toHaveBeenCalledTimes(1); + expect(process.stdout.write).toHaveBeenCalledWith(`${JSON.stringify(served, null, 2)}\n`); + expect(stderrLines()).toEqual([ + "upgrade ax", + "Next: npm i -g ax@latest # this ax captures with a shim ora no longer scores; then run it again", + ]); + }); + + it("keeps stdout the raw payload and suggests a re-run for a local page on stderr", async () => { + expect(await webmcpAuditCommand({ ...BASE })).toBe(EXIT.OK); + expect(console.log).toHaveBeenCalledTimes(1); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(stderrLines()).toContain( + "Next: ax webmcp-audit http://localhost:3000 --json # after you fix the failing checks above, run it again", + ); + }); + + it("moves a public page that cleared the gate on to the whole site", async () => { + expect(await webmcpAuditCommand({ ...BASE, url: "https://shop.example.com/" })).toBe(EXIT.OK); + expect(stderrLines()).toContain( + "Next: ax audit shop.example.com --json # the readiness audit of the whole site", + ); + }); + + it("prints no envelope when ora returns no score, because the payload is already out", async () => { + vi.mocked(ingestCapture).mockResolvedValue({ ...AUDIT, score: null, grade: null }); + expect(await webmcpAuditCommand({ ...BASE, minScore: "70" })).toBe(EXIT.API); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(stderrLines().some((line) => line.startsWith("Next: ax webmcp-audit"))).toBe(true); + }); +}); diff --git a/src/commands/webmcp-audit.ts b/src/commands/webmcp-audit.ts index cd9cf32..aeb704e 100644 --- a/src/commands/webmcp-audit.ts +++ b/src/commands/webmcp-audit.ts @@ -1,3 +1,6 @@ +import { type CommandError, reportError } from "../report/json-error"; +import { axCall, type NextStep, placeholder, printNext, type Word } from "../report/next-line"; +import { isLocalTarget } from "../tunnel"; import { plain } from "../ui/ansi"; import { spinner } from "../ui/spinner"; import { captureLocally } from "../webmcp/capture"; @@ -62,26 +65,107 @@ function parseMinScore(raw: string | undefined): { value?: number; error?: strin return { value }; } +/** The same audit as a call, with a key passed as a flag shown as a placeholder. */ +function sameCallWords(input: WebmcpAuditCommandInput, minScore = input.minScore): Word[] { + return [ + "webmcp-audit", + input.url.trim(), + ...(input.json ? ["--json"] : []), + ...(input.showPassing ? ["--show-passing"] : []), + ...(minScore !== undefined ? ["--min-score", minScore] : []), + ...(input.chromeEndpoint ? ["--chrome-endpoint", input.chromeEndpoint] : []), + ...(input.apiKey ? ["--api-key", placeholder("key")] : []), + ]; +} + +/** A failed ingest: an outdated shim is an upgrade, never a retry. */ +function ingestNext(input: WebmcpAuditCommandInput, err: unknown): NextStep { + if (err instanceof WebmcpIngestError && err.code === "SHIM_VERSION_MISMATCH") { + return { + call: "npm i -g ax@latest", + why: "this ax captures with a shim ora no longer scores; then run it again", + }; + } + if (err instanceof WebmcpIngestError && err.code === "RATE_LIMITED") { + return { call: axCall(sameCallWords(input)), why: "after the wait the message names" }; + } + return { call: axCall(sameCallWords(input)), why: "retry; ora may be briefly unreachable" }; +} + +/** + * After a result: a local page or one under the gate is re-audited once the + * fixes are in; a public page that cleared it moves on to the whole site. + */ +function resultNext(input: WebmcpAuditCommandInput, target: string, belowGate: boolean): NextStep { + if (isLocalTarget(target) || belowGate) { + return { + call: axCall(sameCallWords(input)), + why: "after you fix the failing checks above, run it again", + }; + } + return { + call: axCall(["audit", new URL(target).host, ...(input.json ? ["--json"] : [])]), + why: "the readiness audit of the whole site", + }; +} + export async function webmcpAuditCommand(input: WebmcpAuditCommandInput): Promise { + const fail = ( + error: CommandError, + exit: number, + display?: (message: string) => string, + served?: unknown, + ): number => { + reportError(error, { json: input.json, display, served }); + return exit; + }; const target = normalizeTarget(input.url); if (!target) { - console.error(`Not an auditable URL: ${JSON.stringify(input.url)}`); - console.error("Pass the page you want audited, e.g. http://localhost:3000"); - return EXIT.USAGE; + return fail( + { + code: "INVALID_TARGET", + message: `Not an auditable URL: ${JSON.stringify(input.url)}`, + next: { + call: axCall([ + "webmcp-audit", + "http://localhost:3000", + ...(input.json ? ["--json"] : []), + ]), + }, + }, + EXIT.USAGE, + (cleaned) => `${cleaned}\nPass the page you want audited, e.g. http://localhost:3000`, + ); } const minScore = parseMinScore(input.minScore); if (minScore.error) { - console.error(minScore.error); - return EXIT.USAGE; + return fail( + { + code: "INVALID_FLAG", + message: minScore.error, + next: { call: axCall(sameCallWords(input, "70")) }, + }, + EXIT.USAGE, + ); } // A Chrome we cannot reach is a usage problem with a fix the developer can // act on, not an API failure - and never a prompt to download a browser. const chrome = await resolveChrome(input.chromeEndpoint); if (!chrome.ok) { - console.error(chrome.message); - return EXIT.USAGE; + const words = sameCallWords({ ...input, chromeEndpoint: undefined }); + return fail( + { + code: "NO_CHROME", + message: chrome.message, + next: { + call: axCall([...words, "--chrome-endpoint", placeholder("host:port")]), + why: "a Chrome started with --remote-debugging-port, or install Chrome", + }, + }, + EXIT.USAGE, + ); } if (!input.json) { @@ -101,8 +185,16 @@ export async function webmcpAuditCommand(input: WebmcpAuditCommandInput): Promis }); } catch (err) { spinner.stop(); - console.error(`Capture failed: ${err instanceof Error ? err.message : String(err)}`); - return EXIT.API; + const message = err instanceof Error ? err.message : String(err); + return fail( + { + code: "CAPTURE_FAILED", + message, + next: { call: axCall(sameCallWords(input)), why: "retry the capture" }, + }, + EXIT.API, + (cleaned) => `Capture failed: ${cleaned}`, + ); } finally { // A Chrome we started is ours to stop, on every path out of the capture. // Leaking one holds a profile directory and a port until reboot, and this @@ -116,16 +208,28 @@ export async function webmcpAuditCommand(input: WebmcpAuditCommandInput): Promis const page = capture.pages[0]; if (page?.error) { spinner.stop(); - console.error(`Could not capture ${target}: ${page.error}`); - console.error("Is the dev server running?"); - return EXIT.USAGE; + return fail( + { + code: "PAGE_UNREACHABLE", + message: `Could not capture ${target}: ${page.error}`, + next: { call: axCall(sameCallWords(input)), why: "start the dev server first" }, + }, + EXIT.USAGE, + (cleaned) => `${cleaned}\nIs the dev server running?`, + ); } const rejection = preflightCapture(capture); if (rejection) { spinner.stop(); - console.error(`ora would refuse this capture: ${rejection}`); - return EXIT.USAGE; + return fail( + { + code: "CAPTURE_REFUSED", + message: `ora would refuse this capture: ${rejection}`, + next: { call: axCall(sameCallWords(input)), why: "after you fix what the message names" }, + }, + EXIT.USAGE, + ); } const { capture: payload, droppedScreenshot } = prepareForIngest(capture); @@ -147,12 +251,18 @@ export async function webmcpAuditCommand(input: WebmcpAuditCommandInput): Promis }); } catch (err) { spinner.stop(); - if (err instanceof WebmcpIngestError) { - console.error(err.message); - return EXIT.API; - } - console.error(`Scoring failed: ${err instanceof Error ? err.message : String(err)}`); - return EXIT.API; + const message = err instanceof Error ? err.message : String(err); + const ingest = err instanceof WebmcpIngestError; + return fail( + { + code: (ingest && err.code) || "SCORING_FAILED", + message, + next: ingestNext(input, err), + }, + EXIT.API, + (cleaned) => (ingest ? cleaned : `Scoring failed: ${cleaned}`), + ingest ? err.payload : undefined, + ); } spinner.stop(); @@ -183,20 +293,30 @@ export async function webmcpAuditCommand(input: WebmcpAuditCommandInput): Promis // Compared against the score, never the grade: below the coverage // threshold the grade is withheld and a check keyed on it would pass // everything. + // The payload is already on stdout, so this is a note, not an envelope. if (audit.score === null) { if (!input.json) { console.error("\n ora returned no score, so --min-score cannot be evaluated."); } + printNext( + { + call: axCall(sameCallWords(input)), + why: "ora returned no score to gate on; retry, or drop --min-score", + }, + { stderr: true }, + ); return EXIT.API; } if (audit.score < minScore.value) { if (!input.json) { console.log(`\n Score ${audit.score} is below the required minimum ${minScore.value}\n`); } + printNext(resultNext(input, target, true), { stderr: input.json }); return EXIT.BELOW_MIN_SCORE; } } + printNext(resultNext(input, target, false), { stderr: input.json }); return EXIT.OK; } diff --git a/src/contract/index.ts b/src/contract/index.ts index a90443b..20f059f 100644 --- a/src/contract/index.ts +++ b/src/contract/index.ts @@ -16,6 +16,7 @@ export type JourneyCappedRun = components["schemas"]["JourneyCappedRun"]; export type JourneyRunDetail = components["schemas"]["JourneyRunDetail"]; export type JourneyRunResult = components["schemas"]["JourneyRunResult"]; export type JourneyTrajectoryStep = components["schemas"]["JourneyTrajectoryStep"]; +export type JourneyNextAction = components["schemas"]["JourneyNextAction"]; export { BUILT_AGAINST }; diff --git a/src/contract/types.gen.ts b/src/contract/types.gen.ts index 30c9448..d8a70bf 100644 --- a/src/contract/types.gen.ts +++ b/src/contract/types.gen.ts @@ -160,7 +160,7 @@ export interface paths { "/api/agent-files/{domain}": { /** * Draft a domain's own agent discovery files (ard.json, ai-catalog.json, poppy.json) - * @description Drafts the agent discovery files a domain publishes about itself: /.well-known/ard.json (Agentic Resource Discovery), /.well-known/ai-catalog.json (the same catalog at the AI Catalog Standard's path) and /.well-known/poppy.json (Personal Agent Protocol, Draft 0.2). Built from what ora already holds for the domain - no live crawl - so a domain ora never scanned still gets a valid minimal set (a catalog listing its website, a poppy.json for signed-out browsing) with scannedAt null. Every catalog entry is anchored to the domain itself (urn:air::..., identity https://), never to ora, and carries only https URLs on the domain or its subdomains: anything the scan saw on another host is named in the file's notes instead. MCP servers are listed by their server card (the card the scan found, else the reserved /server-card location, which the notes ask the owner to serve). The poppy.json is the minimal valid document; what ora saw that a fuller one could name (an OAuth issuer, MCP servers, an OpenAPI description) is listed in that file's notes. Pass ?include= for llms.txt, an edit of the site's robots.txt, an RFC 9727 api-catalog and draft MCP server cards. Review every file before publishing it. A leading www. is dropped, and any spelling other than the canonical domain answers 308 to /api/agent-files/. Sends Access-Control-Allow-Origin: * and is CDN-cached. Rate limited to 60 requests per minute per IP - returns 429 if exceeded. Also used by the generator page at directory.ora.ai/generate and by `npx ax generate-files `. + * @description Drafts the agent discovery files a domain publishes about itself: /.well-known/ard.json (Agentic Resource Discovery), /.well-known/ai-catalog.json (the same catalog at the AI Catalog Standard's path) and /.well-known/poppy.json (Personal Agent Protocol, Draft 1.0, in progress, the version the spec's authors ask builders to use). Built from what ora already holds for the domain - no live crawl - so a domain ora never scanned still gets a valid minimal set (a catalog listing its website, a poppy.json for signed-out browsing) with scannedAt null. Every catalog entry is anchored to the domain itself (urn:air::..., identity https://), never to ora, and carries only https URLs on the domain or its subdomains: anything the scan saw on another host is named in the file's notes instead. MCP servers are listed by their server card (the card the scan found, else the reserved /server-card location, which the notes ask the owner to serve). The poppy.json is the minimal valid document; what ora saw that a fuller one could name (an OAuth issuer, MCP servers, an OpenAPI description) is listed in that file's notes. Pass ?include= for llms.txt, an edit of the site's robots.txt, an RFC 9727 api-catalog and draft MCP server cards. Review every file before publishing it. A leading www. is dropped, and any spelling other than the canonical domain answers 308 to /api/agent-files/. Sends Access-Control-Allow-Origin: * and is CDN-cached. Rate limited to 60 requests per minute per IP - returns 429 if exceeded. Also used by the generator page at directory.ora.ai/generate and by `npx ax generate-files `. */ get: operations["generateAgentFiles"]; }; @@ -188,14 +188,14 @@ export interface paths { "/api/journey/runs/{id}": { /** * Get a journey run record (non-streaming) - * @description Returns the run record by id. While the run executes, `status` is 'running' - open the stream or poll. Once finished and persisted, `status` is 'succeeded' and the body carries `verdict`, `step_count` (billable steps - the same counter run pricing uses), and the full `result` (trajectory + insight + signals). `status` 'failed' is terminal with no result. `result` is present iff status is 'succeeded'. + * @description Returns the run record by id. While the run executes, `status` is 'running' - open the stream or poll. Once finished and persisted, `status` is 'succeeded' and the body carries `verdict`, `step_count` (billable steps - the same counter run pricing uses), `flags` (two notes beside the verdict that never change it: `blocked` when the site refused the agent, `answered_from_memory` when half or more of the answer came from the model's memory), and the full `result` (trajectory + insight + signals). `status` 'failed' is terminal with no result. `result` is present iff status is 'succeeded'. `next_action` names the one request to send next: the run's stream while it is running, and after it failed, since reopening it reconnects a run that only lost its connection. A succeeded run carries none. */ get: operations["getJourneyRun"]; }; "/api/journey/domains/{host}": { /** * Get or create a domain's agent journey - * @description Returns the agent journey for a domain, running one only if there is not one already. Send only the domain and no body: ora selects the intent and the agent. Requires an ora-issued partner API key ('Authorization: Bearer ', issued manually - contact ora); without one, 401 PARTNER_KEY_REQUIRED. 200 means nothing was dispatched and an existing run answered, either finished (carrying its `result`) or still running (open `stream_url` to watch it). 201 means a new run was dispatched. Branch on `dispatched`. Two response views (see the `view` parameter): `graph` (JourneyDomainRunGraph, the default - `result` narrowed to what drawing the journey needs) and `full` (JourneyDomainRun, the complete projection, on `?view=full`). A finished run is served indefinitely; `run_age_seconds` gives its age. There is no per-caller rate limit on this endpoint, except a monthly journey usage limit when the key carries one. Each domain is capped at 100 runs per rolling 24h, and one domain cannot start two runs at once - both are per-domain, neither limits how many domains you may ask for or how fast. + * @description Returns the agent journey for a domain, running one only if there is not one already. Send only the domain and no body: ora selects the intent and the agent. Requires an ora-issued partner API key ('Authorization: Bearer ', issued manually - contact ora); without one, 401 PARTNER_KEY_REQUIRED. 200 means nothing was dispatched and an existing run answered, either finished (carrying its `result`) or still running (open `stream_url` to watch it). 201 means a new run was dispatched. Branch on `dispatched`. Two response views (see the `view` parameter): `graph` (JourneyDomainRunGraph, the default - `result` narrowed to what drawing the journey needs) and `full` (JourneyDomainRun, the complete projection, on `?view=full`). A finished run is served indefinitely; `run_age_seconds` gives its age. There is no per-caller rate limit on this endpoint, except a daily or monthly journey usage limit when the key carries one. Each domain is capped at 100 runs per rolling 24h, and one domain cannot start two runs at once - both are per-domain, neither limits how many domains you may ask for or how fast. */ post: operations["getOrCreateDomainJourney"]; }; @@ -223,7 +223,7 @@ export interface paths { "/api/usage": { /** * Read your partner API key's usage and usage limits - * @description Returns the calling key's own usage for the current period, per kind (scans, journeys) and per period (only month today, the calendar month in UTC): how many new dispatches the key started since period_start, its limit, what remains, and when the count resets. Cached and stored answers are not counted. limit and remaining are null when the key has no limit for that kind; a key with no limit is never refused for usage. Requires an ora-issued partner API key ('Authorization: Bearer ', issued manually - contact ora); the key is read from the token, so a caller can only see its own numbers. Every response carries Cache-Control: no-store. Rate limited to 60 requests per minute per key. + * @description Returns the calling key's own usage for the current period, per kind (scans, journeys) and per period (`day`, the calendar day in UTC, and `month`, the calendar month in UTC): how many new dispatches the key started since period_start, its limit, what remains, and when the count resets. Cached and stored answers are not counted. limit and remaining are null when the key has no limit for that kind; a key with no limit is never refused for usage. Requires an ora-issued partner API key ('Authorization: Bearer ', issued manually - contact ora); the key is read from the token, so a caller can only see its own numbers. Every response carries Cache-Control: no-store. Rate limited to 60 requests per minute per key. */ get: operations["getKeyUsage"]; }; @@ -591,7 +591,7 @@ export interface components { * @description The contract version this payload conforms to. SemVer: a major means a response-envelope break - a stable field removed, renamed, or changed in meaning, or the default response format flipping - and is safe to pin. Additive changes and check-catalog membership changes ship on a minor. See docs/api.md -> Contract and versioning. * @enum {string} */ - contractVersion: "1.40.0"; + contractVersion: "1.42.0"; /** * @description Present only when this body is a stored result served by the freshness gate instead of a fresh scan. Absent on a live scan. * @enum {boolean} @@ -810,7 +810,7 @@ export interface components { * @description The contract version this payload conforms to. SemVer: a major means a response-envelope break - a stable field removed, renamed, or changed in meaning, or the default response format flipping - and is safe to pin. Additive changes and check-catalog membership changes ship on a minor. See docs/api.md -> Contract and versioning. * @enum {string} */ - contractVersion: "1.40.0"; + contractVersion: "1.42.0"; /** @description Wall-clock duration of the scan that produced this stored result */ durationMs: number | null; /** @@ -867,7 +867,7 @@ export interface components { * @description The contract version this catalog conforms to - identical to the OpenAPI info.version and the MCP server version. SemVer: a major means a response-envelope break (a stable field, or a layer id, removed or renamed or changed in meaning, or the default response format flipping) and is safe to pin; check-catalog membership changes ship on a minor. The full versioning policy is published in the API description at /api/openapi.json. * @enum {string} */ - contractVersion: "1.40.0"; + contractVersion: "1.42.0"; /** @description The four scored layers in scoring order, with display name and current weight. */ layers: { /** @description Stable layer id: discovery, accessibility, usability, or payments. Removing or renaming a layer id is a major version change. Note one intentional divergence: the id 'accessibility' carries the display name 'Access'. */ @@ -979,7 +979,7 @@ export interface components { * @description The contract version this response conforms to - identical to the OpenAPI info.version and the MCP server version. The full versioning policy is published in the API description at /api/openapi.json. * @enum {string} */ - contractVersion: "1.40.0"; + contractVersion: "1.42.0"; /** @description The apex domain derived from the requested URL. */ domain: string; /** @description The normalized URL the run targeted. */ @@ -1057,6 +1057,23 @@ export interface components { verdict?: "satisfied" | "partial" | "unsatisfied" | "not_gradable"; /** @description Billable steps the run took (tool calls, excluding narration and filesystem ops), present once the run finished. This is the same counter run pricing uses. */ step_count?: number; + /** @description Two notes that sit beside the verdict and never change it, present with the verdict once the run finished. */ + flags?: { + /** @description The site refused the agent: its first request to the site that got an HTTP status was a 401, 403 or 429, or more than half of those requests were. A 404 never counts. */ + blocked: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + /** @description Half or more of the answer came from the model's memory rather than from pages it fetched. */ + answered_from_memory: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + }; /** @description Journey/audit contract version (shared SemVer; see docs) */ contractVersion: string; }; @@ -1090,10 +1107,45 @@ export interface components { verdict?: "satisfied" | "partial" | "unsatisfied" | "not_gradable"; /** @description Billable steps the run took (tool calls, excluding narration and filesystem ops), present once the run finished. This is the same counter run pricing uses. */ step_count?: number; + /** @description Two notes that sit beside the verdict and never change it, present with the verdict once the run finished. */ + flags?: { + /** @description The site refused the agent: its first request to the site that got an HTTP status was a 401, 403 or 429, or more than half of those requests were. A 404 never counts. */ + blocked: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + /** @description Half or more of the answer came from the model's memory rather than from pages it fetched. */ + answered_from_memory: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + }; /** @description Journey/audit contract version (shared SemVer; see docs) */ contractVersion: string; /** @description Echo of the custom intent text the caller sent on this request. Create-response-only: it is never returned by GET /api/journey/runs/{id} or the capped 200, and it is absent on curated runs. Experimental: may change or disappear without a major version. */ intent?: string; + /** @description The one request to send next: the run's stream, held open to the end, because only a followed stream saves the result */ + next_action?: { + /** + * @description HTTP method of the next request + * @enum {string} + */ + method: "GET" | "POST"; + /** @description Where to send it: an API path on this origin, or an absolute URL for a docs page */ + url: string; + /** @description JSON body to send with it, when it takes one: the caller's own request, replayed */ + body?: { + [key: string]: unknown; + }; + /** @description Seconds to wait before sending it */ + wait_seconds?: number; + /** @description Why this request is the recommended next step */ + reason: string; + }; }; JourneyCappedRun: { /** @description Run id - the handle for GET /api/journey/runs/{id} and the stream */ @@ -1125,6 +1177,23 @@ export interface components { verdict?: "satisfied" | "partial" | "unsatisfied" | "not_gradable"; /** @description Billable steps the run took (tool calls, excluding narration and filesystem ops), present once the run finished. This is the same counter run pricing uses. */ step_count?: number; + /** @description Two notes that sit beside the verdict and never change it, present with the verdict once the run finished. */ + flags?: { + /** @description The site refused the agent: its first request to the site that got an HTTP status was a 401, 403 or 429, or more than half of those requests were. A 404 never counts. */ + blocked: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + /** @description Half or more of the answer came from the model's memory rather than from pages it fetched. */ + answered_from_memory: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + }; /** @description Journey/audit contract version (shared SemVer; see docs) */ contractVersion: string; /** @@ -1141,6 +1210,24 @@ export interface components { }; /** @description Milliseconds until a per-target slot frees */ retry_after_ms: number; + /** @description The one request to send next for the served run: its stream while it is running or after it failed, or the run itself for its full result once it succeeded */ + next_action?: { + /** + * @description HTTP method of the next request + * @enum {string} + */ + method: "GET" | "POST"; + /** @description Where to send it: an API path on this origin, or an absolute URL for a docs page */ + url: string; + /** @description JSON body to send with it, when it takes one: the caller's own request, replayed */ + body?: { + [key: string]: unknown; + }; + /** @description Seconds to wait before sending it */ + wait_seconds?: number; + /** @description Why this request is the recommended next step */ + reason: string; + }; }; JourneyRunDetail: { /** @description Run id - the handle for GET /api/journey/runs/{id} and the stream */ @@ -1172,6 +1259,23 @@ export interface components { verdict?: "satisfied" | "partial" | "unsatisfied" | "not_gradable"; /** @description Billable steps the run took (tool calls, excluding narration and filesystem ops), present once the run finished. This is the same counter run pricing uses. */ step_count?: number; + /** @description Two notes that sit beside the verdict and never change it, present with the verdict once the run finished. */ + flags?: { + /** @description The site refused the agent: its first request to the site that got an HTTP status was a 401, 403 or 429, or more than half of those requests were. A 404 never counts. */ + blocked: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + /** @description Half or more of the answer came from the model's memory rather than from pages it fetched. */ + answered_from_memory: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + }; /** @description Journey/audit contract version (shared SemVer; see docs) */ contractVersion: string; /** @description The full run result. Present iff status is 'succeeded'. */ @@ -1428,6 +1532,41 @@ export interface components { journey_layers?: ("discovery" | "identity" | "access" | "payments" | "experience")[]; }; }; + /** @description The one request to send next: the run's stream while it is running, or after it failed, since reopening it reconnects a run that only lost its connection. Absent once the run succeeded, because the result is in this body. */ + next_action?: { + /** + * @description HTTP method of the next request + * @enum {string} + */ + method: "GET" | "POST"; + /** @description Where to send it: an API path on this origin, or an absolute URL for a docs page */ + url: string; + /** @description JSON body to send with it, when it takes one: the caller's own request, replayed */ + body?: { + [key: string]: unknown; + }; + /** @description Seconds to wait before sending it */ + wait_seconds?: number; + /** @description Why this request is the recommended next step */ + reason: string; + }; + }; + JourneyNextAction: { + /** + * @description HTTP method of the next request + * @enum {string} + */ + method: "GET" | "POST"; + /** @description Where to send it: an API path on this origin, or an absolute URL for a docs page */ + url: string; + /** @description JSON body to send with it, when it takes one: the caller's own request, replayed */ + body?: { + [key: string]: unknown; + }; + /** @description Seconds to wait before sending it */ + wait_seconds?: number; + /** @description Why this request is the recommended next step */ + reason: string; }; JourneyDomainRun: { /** @description Run id - the handle for GET /api/journey/runs/{id} and the stream */ @@ -1459,6 +1598,23 @@ export interface components { verdict?: "satisfied" | "partial" | "unsatisfied" | "not_gradable"; /** @description Billable steps the run took (tool calls, excluding narration and filesystem ops), present once the run finished. This is the same counter run pricing uses. */ step_count?: number; + /** @description Two notes that sit beside the verdict and never change it, present with the verdict once the run finished. */ + flags?: { + /** @description The site refused the agent: its first request to the site that got an HTTP status was a 401, 403 or 429, or more than half of those requests were. A 404 never counts. */ + blocked: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + /** @description Half or more of the answer came from the model's memory rather than from pages it fetched. */ + answered_from_memory: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + }; /** @description Journey/audit contract version (shared SemVer; see docs) */ contractVersion: string; /** @description The full run result. Present iff status is 'succeeded'. */ @@ -1752,6 +1908,23 @@ export interface components { verdict?: "satisfied" | "partial" | "unsatisfied" | "not_gradable"; /** @description Billable steps the run took (tool calls, excluding narration and filesystem ops), present once the run finished. This is the same counter run pricing uses. */ step_count?: number; + /** @description Two notes that sit beside the verdict and never change it, present with the verdict once the run finished. */ + flags?: { + /** @description The site refused the agent: its first request to the site that got an HTTP status was a 401, 403 or 429, or more than half of those requests were. A 404 never counts. */ + blocked: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + /** @description Half or more of the answer came from the model's memory rather than from pages it fetched. */ + answered_from_memory: { + /** @description Whether the note applies to this run */ + flagged: boolean; + /** @description The note in plain words, present only when flagged */ + reason?: string; + }; + }; /** @description Journey/audit contract version (shared SemVer; see docs) */ contractVersion: string; /** @description The graph-view run result. Present iff status is 'succeeded'. */ @@ -2473,10 +2646,17 @@ export interface components { next?: string; }; /** - * @description A partner key's usage per kind, returned by GET /api/usage. Each kind holds one entry per period; month is the only period today, and a future period is a sibling key next to it. + * @description A partner key's usage per kind, returned by GET /api/usage. Each kind holds one entry per period: day (calendar day, UTC) and month (calendar month, UTC). A future period is a sibling key next to them. * @example { * "key": "acme", * "scans": { + * "day": { + * "used": 40, + * "limit": 200, + * "remaining": 160, + * "period_start": "2026-10-14T00:00:00.000Z", + * "resets_at": "2026-10-15T00:00:00.000Z" + * }, * "month": { * "used": 1250, * "limit": 5000, @@ -2486,6 +2666,13 @@ export interface components { * } * }, * "journeys": { + * "day": { + * "used": 1, + * "limit": null, + * "remaining": null, + * "period_start": "2026-10-14T00:00:00.000Z", + * "resets_at": "2026-10-15T00:00:00.000Z" + * }, * "month": { * "used": 12, * "limit": null, @@ -2501,10 +2688,12 @@ export interface components { key: string; /** @description Scan dispatches: POST /api/scan, GET /api/scan/stream, POST /api/scan/checks, POST /api/v2/scan and the MCP scan_domain and run_checks tools. */ scans: { + day: components["schemas"]["KeyUsagePeriod"]; month: components["schemas"]["KeyUsagePeriod"]; }; /** @description Journey dispatches: POST /api/journey/runs and POST /api/journey/domains/{host}. */ journeys: { + day: components["schemas"]["KeyUsagePeriod"]; month: components["schemas"]["KeyUsagePeriod"]; }; }; @@ -2518,12 +2707,12 @@ export interface components { remaining: number | null; /** * Format: date-time - * @description Start of the current period (the 1st of the month, 00:00 UTC). + * @description Start of the current period: 00:00 UTC today for day, the 1st of the month at 00:00 UTC for month. */ period_start: string; /** * Format: date-time - * @description When the count resets (the 1st of next month, 00:00 UTC). + * @description When the count resets: 00:00 UTC tomorrow for day, the 1st of next month at 00:00 UTC for month. */ resets_at: string; }; @@ -2878,7 +3067,7 @@ export interface operations { "application/json": components["schemas"]["McpAuthRequiredResponse"]; }; }; - /** @description Rate limit exceeded - either the 10-per-minute burst cap or the durable daily quota (30 scans per rolling 24h per IP; 6 per day for force=true). Cache-served responses never count against the daily quota. The response includes a Retry-After header indicating seconds until the next request is allowed, and a JSON body with error and retry_after_ms. A keyed caller whose key has used its monthly scan limit gets 429 with error `usage_limit_reached` and `kind`, `period`, `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to the next 1st of the month (UTC). That refusal is key-wide; GET /api/usage shows the numbers. */ + /** @description Rate limit exceeded - either the 10-per-minute burst cap or the durable daily quota (30 scans per rolling 24h per IP; 6 per day for force=true). Cache-served responses never count against the daily quota. The response includes a Retry-After header indicating seconds until the next request is allowed, and a JSON body with error and retry_after_ms. A keyed caller whose key has used its daily or monthly scan limit gets 429 with error `usage_limit_reached` and `kind`, `period` (`day` or `month`), `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to `resets_at`: the next 00:00 UTC for a daily limit, the next 1st of the month (UTC) for a monthly one, and the later of the two when both are used up. That refusal is key-wide; GET /api/usage shows the numbers. */ 429: { headers: { /** @description Seconds until next allowed request */ @@ -2956,7 +3145,7 @@ export interface operations { "application/json": components["schemas"]["ErrorResponse"]; }; }; - /** @description Rate limit exceeded - either the 10-per-minute burst cap or the durable daily quota (30 scans per rolling 24h per IP), which this endpoint spends from the same budget as the v1 scan routes so a caller cannot drain it twice by switching endpoints. The response carries a Retry-After header; a durable deny also carries retry_after_ms in the body. A keyed caller whose key has used its monthly scan limit gets 429 with error `usage_limit_reached` and `kind`, `period`, `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to the next 1st of the month (UTC). That refusal is key-wide; GET /api/usage shows the numbers. */ + /** @description Rate limit exceeded - either the 10-per-minute burst cap or the durable daily quota (30 scans per rolling 24h per IP), which this endpoint spends from the same budget as the v1 scan routes so a caller cannot drain it twice by switching endpoints. The response carries a Retry-After header; a durable deny also carries retry_after_ms in the body. A keyed caller whose key has used its daily or monthly scan limit gets 429 with error `usage_limit_reached` and `kind`, `period` (`day` or `month`), `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to `resets_at`: the next 00:00 UTC for a daily limit, the next 1st of the month (UTC) for a monthly one, and the later of the two when both are used up. That refusal is key-wide; GET /api/usage shows the numbers. */ 429: { headers: { /** @description Seconds until next allowed request */ @@ -3018,7 +3207,7 @@ export interface operations { 400: { content: never; }; - /** @description Rate limit exceeded - either the 10-per-minute burst cap or the durable daily quota (30 scans per rolling 24h per IP; 6 per day for force=1). The response carries a Retry-After header with the seconds until the caller's window frees up. A keyed caller whose key has used its monthly scan limit gets 429 with error `usage_limit_reached` and `kind`, `period`, `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to the next 1st of the month (UTC). That refusal is key-wide; GET /api/usage shows the numbers. */ + /** @description Rate limit exceeded - either the 10-per-minute burst cap or the durable daily quota (30 scans per rolling 24h per IP; 6 per day for force=1). The response carries a Retry-After header with the seconds until the caller's window frees up. A keyed caller whose key has used its daily or monthly scan limit gets 429 with error `usage_limit_reached` and `kind`, `period` (`day` or `month`), `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to `resets_at`: the next 00:00 UTC for a daily limit, the next 1st of the month (UTC) for a monthly one, and the later of the two when both are used up. That refusal is key-wide; GET /api/usage shows the numbers. */ 429: { headers: { /** @description Seconds until next allowed request */ @@ -3051,7 +3240,7 @@ export interface operations { "application/json": components["schemas"]["ErrorResponse"]; }; }; - /** @description Rate limit exceeded - either the 10-per-minute burst cap (body `{ error }`) or the durable daily scan budget shared with POST /api/scan (body also carries `retry_after_ms`). Both carry a Retry-After header with the seconds until the caller's window frees up. A keyed caller whose key has used its monthly scan limit gets 429 with error `usage_limit_reached` and `kind`, `period`, `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to the next 1st of the month (UTC). That refusal is key-wide; GET /api/usage shows the numbers. */ + /** @description Rate limit exceeded - either the 10-per-minute burst cap (body `{ error }`) or the durable daily scan budget shared with POST /api/scan (body also carries `retry_after_ms`). Both carry a Retry-After header with the seconds until the caller's window frees up. A keyed caller whose key has used its daily or monthly scan limit gets 429 with error `usage_limit_reached` and `kind`, `period` (`day` or `month`), `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to `resets_at`: the next 00:00 UTC for a daily limit, the next 1st of the month (UTC) for a monthly one, and the later of the two when both are used up. That refusal is key-wide; GET /api/usage shows the numbers. */ 429: { headers: { /** @description Seconds until next allowed request */ @@ -3667,7 +3856,7 @@ export interface operations { }; /** * Draft a domain's own agent discovery files (ard.json, ai-catalog.json, poppy.json) - * @description Drafts the agent discovery files a domain publishes about itself: /.well-known/ard.json (Agentic Resource Discovery), /.well-known/ai-catalog.json (the same catalog at the AI Catalog Standard's path) and /.well-known/poppy.json (Personal Agent Protocol, Draft 0.2). Built from what ora already holds for the domain - no live crawl - so a domain ora never scanned still gets a valid minimal set (a catalog listing its website, a poppy.json for signed-out browsing) with scannedAt null. Every catalog entry is anchored to the domain itself (urn:air::..., identity https://), never to ora, and carries only https URLs on the domain or its subdomains: anything the scan saw on another host is named in the file's notes instead. MCP servers are listed by their server card (the card the scan found, else the reserved /server-card location, which the notes ask the owner to serve). The poppy.json is the minimal valid document; what ora saw that a fuller one could name (an OAuth issuer, MCP servers, an OpenAPI description) is listed in that file's notes. Pass ?include= for llms.txt, an edit of the site's robots.txt, an RFC 9727 api-catalog and draft MCP server cards. Review every file before publishing it. A leading www. is dropped, and any spelling other than the canonical domain answers 308 to /api/agent-files/. Sends Access-Control-Allow-Origin: * and is CDN-cached. Rate limited to 60 requests per minute per IP - returns 429 if exceeded. Also used by the generator page at directory.ora.ai/generate and by `npx ax generate-files `. + * @description Drafts the agent discovery files a domain publishes about itself: /.well-known/ard.json (Agentic Resource Discovery), /.well-known/ai-catalog.json (the same catalog at the AI Catalog Standard's path) and /.well-known/poppy.json (Personal Agent Protocol, Draft 1.0, in progress, the version the spec's authors ask builders to use). Built from what ora already holds for the domain - no live crawl - so a domain ora never scanned still gets a valid minimal set (a catalog listing its website, a poppy.json for signed-out browsing) with scannedAt null. Every catalog entry is anchored to the domain itself (urn:air::..., identity https://), never to ora, and carries only https URLs on the domain or its subdomains: anything the scan saw on another host is named in the file's notes instead. MCP servers are listed by their server card (the card the scan found, else the reserved /server-card location, which the notes ask the owner to serve). The poppy.json is the minimal valid document; what ora saw that a fuller one could name (an OAuth issuer, MCP servers, an OpenAPI description) is listed in that file's notes. Pass ?include= for llms.txt, an edit of the site's robots.txt, an RFC 9727 api-catalog and draft MCP server cards. Review every file before publishing it. A leading www. is dropped, and any spelling other than the canonical domain answers 308 to /api/agent-files/. Sends Access-Control-Allow-Origin: * and is CDN-cached. Rate limited to 60 requests per minute per IP - returns 429 if exceeded. Also used by the generator page at directory.ora.ai/generate and by `npx ax generate-files `. */ generateAgentFiles: { parameters: { @@ -3785,7 +3974,7 @@ export interface operations { }; }; responses: { - /** @description Per-target cap hit - the freshness analog, not an error: the body is the most recent stored run for this exact (domain, intent, harness, model) target, marked `rate_limited: true`, so CI and agents get the newest result without spending a run. When the served run has finished, the body also carries `verdict` and `step_count`. Replay it via its `stream_url`. Anonymous tier only: a keyed caller skips the per-target cap and never receives this response. */ + /** @description Per-target cap hit - the freshness analog, not an error: the body is the most recent stored run for this exact (domain, intent, harness, model) target, marked `rate_limited: true`, so CI and agents get the newest result without spending a run. When the served run has finished, the body also carries `verdict`, `step_count` and `flags`. Replay it via its `stream_url`. `next_action` names the one request to send next: the run itself, for its full result, once it succeeded; its stream while it is running or after it failed. Anonymous tier only: a keyed caller skips the per-target cap and never receives this response. */ 200: { headers: { /** @description Seconds until the target's oldest in-window run ages out */ @@ -3801,7 +3990,7 @@ export interface operations { "application/json": components["schemas"]["JourneyCappedRun"]; }; }; - /** @description Run created and dispatched. Open `stream_url` (SSE) and hold it to its `result` or `error` frame: the stream is what saves the finished run, which GET /api/journey/runs/{id} then reads. A keyed custom run gets its own free text back as `intent` here - the create response is the only place any intent text is published, so GET /api/journey/runs/{id} and the capped 200 never carry it, and a curated run has no `intent` at all. */ + /** @description Run created and dispatched. Open `stream_url` (SSE) and hold it to its `result` or `error` frame: the stream is what saves the finished run, which GET /api/journey/runs/{id} then reads. `next_action` names that same stream as the one request to send next. A keyed custom run gets its own free text back as `intent` here - the create response is the only place any intent text is published, so GET /api/journey/runs/{id} and the capped 200 never carry it, and a curated run has no `intent` at all. */ 201: { headers: { /** @description Size of the one window this response describes: the per-target window (100 runs per rolling 24h) for an anonymous caller, the per-key caller window (1000 runs per rolling 24h) for a keyed one */ @@ -3815,39 +4004,51 @@ export interface operations { "application/json": components["schemas"]["JourneyCreatedRun"]; }; }; - /** @description Invalid input: schema failure (including unknown body fields - the contract is strict, and a body matching neither intent arm or both of them lands here), `code: "INVALID_DOMAIN"` for a domain the scanner would reject, or `code: "UNSUPPORTED_AGENT"` for a (harness, model) pair that is not publicly runnable. */ + /** @description Invalid input: schema failure (including unknown body fields - the contract is strict, and a body matching neither intent arm or both of them lands here), `code: "INVALID_DOMAIN"` for a domain the scanner would reject, or `code: "UNSUPPORTED_AGENT"` for a (harness, model) pair that is not publicly runnable. The two coded refusals carry `next_action`: the request to send again with a public host, or GET /api/journey/agents. */ 400: { content: { - "application/json": components["schemas"]["ErrorResponse"]; + "application/json": components["schemas"]["ErrorResponse"] & { + /** @description The one request to send next. 400 INVALID_DOMAIN: the same request with a public host as intent.domain. 400 UNSUPPORTED_AGENT: GET /api/journey/agents, to pick an agent that runs here. 401: the docs page that says how to get a partner key. 429 and 503: the same request again after `wait_seconds`. Absent on a schema failure and on a `usage_limit_reached` 429. */ + next_action?: components["schemas"]["JourneyNextAction"]; + }; }; }; - /** @description `code: "CUSTOM_INTENT_REQUIRES_KEY"` - the request sent a custom free-text intent without a recognized ora-issued partner API key. An absent, malformed, and unrecognized key all land here alike, so the status never doubles as a key-validity oracle. Two ways forward: run a curated intent instead (GET /api/journey/intents lists them, and that arm needs no key), or ask ora for a partner key - they are issued manually on request, so contact us at the address in this document's info.contact. */ + /** @description `code: "CUSTOM_INTENT_REQUIRES_KEY"` - the request sent a custom free-text intent without a recognized ora-issued partner API key. An absent, malformed, and unrecognized key all land here alike, so the status never doubles as a key-validity oracle. Two ways forward: run a curated intent instead (GET /api/journey/intents lists them, and that arm needs no key), or ask ora for a partner key - they are issued manually on request, so contact us at the address in this document's info.contact. `next_action` points at the docs page that says how to get one. */ 401: { content: { - "application/json": components["schemas"]["ErrorResponse"]; + "application/json": components["schemas"]["ErrorResponse"] & { + /** @description The one request to send next. 400 INVALID_DOMAIN: the same request with a public host as intent.domain. 400 UNSUPPORTED_AGENT: GET /api/journey/agents, to pick an agent that runs here. 401: the docs page that says how to get a partner key. 429 and 503: the same request again after `wait_seconds`. Absent on a schema failure and on a `usage_limit_reached` 429. */ + next_action?: components["schemas"]["JourneyNextAction"]; + }; }; }; - /** @description Caller limit exceeded - the 20-per-minute burst cap (anonymous callers only; a keyed caller skips it), or the durable per-caller budget (20 runs per rolling 24h per IP anonymous, plus 3000 per rolling 24h shared by all anonymous callers, 1000 per rolling 24h per key; body carries `retry_after_ms`). Per-TARGET saturation is the 200 above, not this. A keyed caller whose key has used its monthly journey limit gets 429 with error `usage_limit_reached` and `kind`, `period`, `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to the next 1st of the month (UTC). That refusal is key-wide; GET /api/usage shows the numbers. */ + /** @description Caller limit exceeded - the 20-per-minute burst cap (anonymous callers only; a keyed caller skips it), or the durable per-caller budget (20 runs per rolling 24h per IP anonymous, plus 3000 per rolling 24h shared by all anonymous callers, 1000 per rolling 24h per key; body carries `retry_after_ms`). Per-TARGET saturation is the 200 above, not this. A keyed caller whose key has used its daily or monthly journey limit gets 429 with error `usage_limit_reached` and `kind`, `period` (`day` or `month`), `used`, `limit`, `resets_at` and `retry_after_ms`; its Retry-After counts to `resets_at`: the next 00:00 UTC for a daily limit, the next 1st of the month (UTC) for a monthly one, and the later of the two when both are used up. That refusal is key-wide; GET /api/usage shows the numbers. The burst and per-caller refusals carry `next_action`: the same request again after `wait_seconds`; the `usage_limit_reached` refusal carries none. */ 429: { headers: { /** @description Seconds until next allowed request */ "Retry-After"?: number; }; content: { - "application/json": components["schemas"]["ErrorResponse"]; + "application/json": components["schemas"]["ErrorResponse"] & { + /** @description The one request to send next. 400 INVALID_DOMAIN: the same request with a public host as intent.domain. 400 UNSUPPORTED_AGENT: GET /api/journey/agents, to pick an agent that runs here. 401: the docs page that says how to get a partner key. 429 and 503: the same request again after `wait_seconds`. Absent on a schema failure and on a `usage_limit_reached` 429. */ + next_action?: components["schemas"]["JourneyNextAction"]; + }; }; }; - /** @description The run engine could not be reached; nothing was created or consumed. */ + /** @description The run engine could not be reached; nothing was created or consumed. `next_action` is the same request again after `wait_seconds` (about a minute). */ 503: { content: { - "application/json": components["schemas"]["ErrorResponse"]; + "application/json": components["schemas"]["ErrorResponse"] & { + /** @description The one request to send next. 400 INVALID_DOMAIN: the same request with a public host as intent.domain. 400 UNSUPPORTED_AGENT: GET /api/journey/agents, to pick an agent that runs here. 401: the docs page that says how to get a partner key. 429 and 503: the same request again after `wait_seconds`. Absent on a schema failure and on a `usage_limit_reached` 429. */ + next_action?: components["schemas"]["JourneyNextAction"]; + }; }; }; }; }; /** * Get a journey run record (non-streaming) - * @description Returns the run record by id. While the run executes, `status` is 'running' - open the stream or poll. Once finished and persisted, `status` is 'succeeded' and the body carries `verdict`, `step_count` (billable steps - the same counter run pricing uses), and the full `result` (trajectory + insight + signals). `status` 'failed' is terminal with no result. `result` is present iff status is 'succeeded'. + * @description Returns the run record by id. While the run executes, `status` is 'running' - open the stream or poll. Once finished and persisted, `status` is 'succeeded' and the body carries `verdict`, `step_count` (billable steps - the same counter run pricing uses), `flags` (two notes beside the verdict that never change it: `blocked` when the site refused the agent, `answered_from_memory` when half or more of the answer came from the model's memory), and the full `result` (trajectory + insight + signals). `status` 'failed' is terminal with no result. `result` is present iff status is 'succeeded'. `next_action` names the one request to send next: the run's stream while it is running, and after it failed, since reopening it reconnects a run that only lost its connection. A succeeded run carries none. */ getJourneyRun: { parameters: { @@ -3857,7 +4058,7 @@ export interface operations { }; }; responses: { - /** @description The run record; plus `result` once succeeded. */ + /** @description The run record; plus `result` and `flags` once succeeded, and `next_action` until then. */ 200: { content: { "application/json": components["schemas"]["JourneyRunDetail"]; @@ -3871,7 +4072,7 @@ export interface operations { }; /** * Get or create a domain's agent journey - * @description Returns the agent journey for a domain, running one only if there is not one already. Send only the domain and no body: ora selects the intent and the agent. Requires an ora-issued partner API key ('Authorization: Bearer ', issued manually - contact ora); without one, 401 PARTNER_KEY_REQUIRED. 200 means nothing was dispatched and an existing run answered, either finished (carrying its `result`) or still running (open `stream_url` to watch it). 201 means a new run was dispatched. Branch on `dispatched`. Two response views (see the `view` parameter): `graph` (JourneyDomainRunGraph, the default - `result` narrowed to what drawing the journey needs) and `full` (JourneyDomainRun, the complete projection, on `?view=full`). A finished run is served indefinitely; `run_age_seconds` gives its age. There is no per-caller rate limit on this endpoint, except a monthly journey usage limit when the key carries one. Each domain is capped at 100 runs per rolling 24h, and one domain cannot start two runs at once - both are per-domain, neither limits how many domains you may ask for or how fast. + * @description Returns the agent journey for a domain, running one only if there is not one already. Send only the domain and no body: ora selects the intent and the agent. Requires an ora-issued partner API key ('Authorization: Bearer ', issued manually - contact ora); without one, 401 PARTNER_KEY_REQUIRED. 200 means nothing was dispatched and an existing run answered, either finished (carrying its `result`) or still running (open `stream_url` to watch it). 201 means a new run was dispatched. Branch on `dispatched`. Two response views (see the `view` parameter): `graph` (JourneyDomainRunGraph, the default - `result` narrowed to what drawing the journey needs) and `full` (JourneyDomainRun, the complete projection, on `?view=full`). A finished run is served indefinitely; `run_age_seconds` gives its age. There is no per-caller rate limit on this endpoint, except a daily or monthly journey usage limit when the key carries one. Each domain is capped at 100 runs per rolling 24h, and one domain cannot start two runs at once - both are per-domain, neither limits how many domains you may ask for or how fast. */ getOrCreateDomainJourney: { parameters: { @@ -3885,7 +4086,7 @@ export interface operations { }; }; responses: { - /** @description An existing run answered; nothing was dispatched (`dispatched` is false). Either a finished run with `result` and `run_age_seconds` (final: no backoff hint), one still running (carries `retry_after_ms` and a `Retry-After` header: poll no faster than that), or - when the domain is over its 100-per-24h cap - its newest stored run with `retry_after_ms`. The body is JourneyDomainRun, or JourneyDomainRunGraph when the effective view is `graph` (see the `view` parameter). */ + /** @description An existing run answered; nothing was dispatched (`dispatched` is false). Either a finished run with `result`, `flags` and `run_age_seconds` (final: no backoff hint), one still running (carries `retry_after_ms` and a `Retry-After` header: poll no faster than that), or - when the domain is over its 100-per-24h cap - its newest stored run with `retry_after_ms`. The body is JourneyDomainRun, or JourneyDomainRunGraph when the effective view is `graph` (see the `view` parameter). */ 200: { content: { "application/json": components["schemas"]["JourneyDomainRun"] | components["schemas"]["JourneyDomainRunGraph"]; @@ -3909,7 +4110,7 @@ export interface operations { "application/json": components["schemas"]["ErrorResponse"]; }; }; - /** @description Back off and retry the SAME DOMAIN; the body carries `retry_after_ms` and a `Retry-After` header. All three codes here are per-domain, never key-wide, so do not throttle your other traffic. `POLL_TOO_FAST`: more than 3 requests for this domain inside 2 seconds; back off by this response's `retry_after_ms`. The in-flight `200` also carries `retry_after_ms`, and a poller that honours it never sees this code. `DISPATCH_IN_PROGRESS`: another request is already starting this domain. `TARGET_CAPPED`: this domain is over its 100-per-24h cap. One refusal here is key-wide instead: `error: "usage_limit_reached"` (no `code`) means the key has used its monthly journey limit, with `kind`, `period`, `used`, `limit`, `resets_at` and `retry_after_ms` in the body and a Retry-After that counts to the next 1st of the month (UTC); stop dispatching journeys until `resets_at`. GET /api/usage shows the numbers. */ + /** @description Back off and retry the SAME DOMAIN; the body carries `retry_after_ms` and a `Retry-After` header. All three codes here are per-domain, never key-wide, so do not throttle your other traffic. `POLL_TOO_FAST`: more than 3 requests for this domain inside 2 seconds; back off by this response's `retry_after_ms`. The in-flight `200` also carries `retry_after_ms`, and a poller that honours it never sees this code. `DISPATCH_IN_PROGRESS`: another request is already starting this domain. `TARGET_CAPPED`: this domain is over its 100-per-24h cap. One refusal here is key-wide instead: `error: "usage_limit_reached"` (no `code`) means the key has used its daily or monthly journey limit, with `kind`, `period` (`day` or `month`), `used`, `limit`, `resets_at` and `retry_after_ms` in the body and a Retry-After that counts to `resets_at`; stop dispatching journeys until then. GET /api/usage shows the numbers. */ 429: { headers: { /** @description Seconds until next allowed request */ @@ -4021,7 +4222,7 @@ export interface operations { }; /** * Read your partner API key's usage and usage limits - * @description Returns the calling key's own usage for the current period, per kind (scans, journeys) and per period (only month today, the calendar month in UTC): how many new dispatches the key started since period_start, its limit, what remains, and when the count resets. Cached and stored answers are not counted. limit and remaining are null when the key has no limit for that kind; a key with no limit is never refused for usage. Requires an ora-issued partner API key ('Authorization: Bearer ', issued manually - contact ora); the key is read from the token, so a caller can only see its own numbers. Every response carries Cache-Control: no-store. Rate limited to 60 requests per minute per key. + * @description Returns the calling key's own usage for the current period, per kind (scans, journeys) and per period (`day`, the calendar day in UTC, and `month`, the calendar month in UTC): how many new dispatches the key started since period_start, its limit, what remains, and when the count resets. Cached and stored answers are not counted. limit and remaining are null when the key has no limit for that kind; a key with no limit is never refused for usage. Requires an ora-issued partner API key ('Authorization: Bearer ', issued manually - contact ora); the key is read from the token, so a caller can only see its own numbers. Every response carries Cache-Control: no-store. Rate limited to 60 requests per minute per key. */ getKeyUsage: { responses: { diff --git a/src/contract/version.ts b/src/contract/version.ts index 8e38b11..e09c4e2 100644 --- a/src/contract/version.ts +++ b/src/contract/version.ts @@ -2,4 +2,4 @@ // Regenerate: pnpm contract:gen [--url ] /** The ora contract version (spec info.version) these types were generated from. */ -export const BUILT_AGAINST = "1.40.0"; +export const BUILT_AGAINST = "1.42.0"; diff --git a/src/env.test.ts b/src/env.test.ts index ed4e4d5..25624cd 100644 --- a/src/env.test.ts +++ b/src/env.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { loadLocalEnv } from "./env"; +import { fromLocalEnvFile, loadLocalEnv } from "./env"; // Every case runs inside a throwaway directory: `.env` is resolved from cwd, and // the repo's own .env holds a real key that must never leak into a test. @@ -46,6 +46,57 @@ describe("loadLocalEnv", () => { expect(process.env.ORA_API_KEY).toBe("from_environment"); }); + it("remembers which variables the file supplied", () => { + inTempDir("ORA_API_KEY=from_file\n"); + loadLocalEnv(); + expect(fromLocalEnvFile("ORA_API_KEY")).toBe(true); + expect(fromLocalEnvFile("PATH")).toBe(false); + }); + + it("does not count a variable the shell exported, even when the file sets it too", () => { + process.env.ORA_API_KEY = "from_environment"; + inTempDir("ORA_API_KEY=from_file\n"); + loadLocalEnv(); + expect(fromLocalEnvFile("ORA_API_KEY")).toBe(false); + }); + + describe("on Windows, where variable names ignore case", () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + + afterEach(() => { + if (platform) Object.defineProperty(process, "platform", platform); + delete process.env.ora_api_key; + }); + + function onWindows(): void { + Object.defineProperty(process, "platform", { value: "win32" }); + } + + it("counts a lowercase name in the file as the variable it sets", () => { + delete process.env.ORA_API_KEY; + onWindows(); + inTempDir("ora_api_key=from_file\n"); + loadLocalEnv(); + expect(fromLocalEnvFile("ORA_API_KEY")).toBe(true); + }); + + it("does not count it when the shell exported the variable in another case", () => { + process.env.ORA_API_KEY = "from_environment"; + onWindows(); + inTempDir("ora_api_key=from_file\n"); + loadLocalEnv(); + expect(fromLocalEnvFile("ORA_API_KEY")).toBe(false); + }); + }); + + it("forgets the file's variables when a later load finds no file", () => { + inTempDir("ORA_API_KEY=from_file\n"); + loadLocalEnv(); + rmSync(join(process.cwd(), ".env")); + loadLocalEnv(); + expect(fromLocalEnvFile("ORA_API_KEY")).toBe(false); + }); + it("reports no load when the directory has no .env", () => { inTempDir(); expect(loadLocalEnv()).toBe(false); diff --git a/src/env.ts b/src/env.ts index 23a4c34..48fca33 100644 --- a/src/env.ts +++ b/src/env.ts @@ -14,19 +14,40 @@ * dotenv does by default. Exported variables win over the file, so a key set * for the shell or by CI beats a stale `.env` left in the directory. * + * Remembers which variables the file supplied (see `fromLocalEnvFile`): a file + * in whatever directory ax runs from is not the user's word on where a key goes. + * * @returns whether a file was actually loaded. */ export function loadLocalEnv(): boolean { + fromFile.clear(); // Node gained loadEnvFile in 20.12. `engines` asks for that, but engines is // advisory — an older runtime should still start and fail later on the // missing key, not die here with a TypeError. if (typeof process.loadEnvFile !== "function") return false; + const exported = new Set(Object.keys(process.env).map(envName)); try { process.loadEnvFile(); + for (const name of Object.keys(process.env).map(envName)) { + if (!exported.has(name)) fromFile.add(name); + } return true; } catch { // No .env here. The common case: `npx ax audit` needs no config. return false; } } + +const fromFile = new Set(); + +// Windows reads variables without regard to case, so `ora_api_url` in the file is +// what `process.env.ORA_API_URL` returns there, while `Object.keys` keeps the file's case. +function envName(name: string): string { + return process.platform === "win32" ? name.toUpperCase() : name; +} + +/** Whether `name` came from the last `loadLocalEnv`'s file rather than the shell. */ +export function fromLocalEnvFile(name: string): boolean { + return fromFile.has(envName(name)); +} diff --git a/src/journey-set/__fixtures__/studio-trajectories/openclaw-f6f23ac8.json b/src/journey-set/__fixtures__/studio-trajectories/openclaw-f6f23ac8.json new file mode 100644 index 0000000..71e476a --- /dev/null +++ b/src/journey-set/__fixtures__/studio-trajectories/openclaw-f6f23ac8.json @@ -0,0 +1,150 @@ +{ + "steps": [ + { + "id": 0, + "type": "text" + }, + { + "id": 1, + "type": "text" + }, + { + "id": 2, + "type": "tool_call" + }, + { + "id": 3, + "type": "tool_call", + "url_host": "ora-api.readme.io" + }, + { + "id": 4, + "type": "tool_call", + "url_host": "ora-api.readme.io" + }, + { + "id": 5, + "type": "tool_call" + }, + { + "id": 6, + "type": "text" + }, + { + "id": 7, + "type": "tool_call" + }, + { + "id": 8, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 9, + "type": "tool_call", + "url_host": "ora-api.readme.io" + }, + { + "id": 10, + "type": "tool_call", + "url_host": "api.ora.ai", + "status": 404 + }, + { + "id": 11, + "type": "text" + }, + { + "id": 12, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 13, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 14, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 15, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 16, + "type": "tool_call", + "url_host": "ora.ai" + }, + { + "id": 17, + "type": "tool_call", + "url_host": "ora.ai" + }, + { + "id": 18, + "type": "tool_call" + }, + { + "id": 19, + "type": "text" + }, + { + "id": 20, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 21, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 22, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 23, + "type": "tool_call", + "url_host": "ora.ai", + "status": 200 + }, + { + "id": 24, + "type": "tool_call", + "url_host": "api.ora.ai", + "status": 404 + }, + { + "id": 25, + "type": "tool_call", + "url_host": "ora-api.readme.io" + }, + { + "id": 26, + "type": "tool_call", + "url_host": "ora-api.readme.io" + }, + { + "id": 27, + "type": "tool_call", + "url_host": "ora-api.readme.io" + }, + { + "id": 28, + "type": "text" + } + ] +} diff --git a/src/journey-set/__fixtures__/studio-trajectories/run-49ab0c88.json b/src/journey-set/__fixtures__/studio-trajectories/run-49ab0c88.json new file mode 100644 index 0000000..6585809 --- /dev/null +++ b/src/journey-set/__fixtures__/studio-trajectories/run-49ab0c88.json @@ -0,0 +1,99 @@ +{ + "steps": [ + { + "id": 0, + "type": "text" + }, + { + "id": 1, + "type": "tool_call" + }, + { + "id": 2, + "type": "text" + }, + { + "id": 3, + "type": "tool_call", + "url_host": "ora.ai" + }, + { + "id": 4, + "type": "text" + }, + { + "id": 5, + "type": "tool_call", + "url_host": "ora.ai" + }, + { + "id": 6, + "type": "text" + }, + { + "id": 7, + "type": "tool_call" + }, + { + "id": 8, + "type": "text" + }, + { + "id": 9, + "type": "tool_call", + "url_host": "ora.ai" + }, + { + "id": 10, + "type": "text" + }, + { + "id": 11, + "type": "tool_call", + "url_host": "ora.ai" + }, + { + "id": 12, + "type": "text" + }, + { + "id": 13, + "type": "tool_call", + "url_host": "ora.run", + "status": 308 + }, + { + "id": 14, + "type": "text" + }, + { + "id": 15, + "type": "tool_call", + "url_host": "ora.ai" + }, + { + "id": 16, + "type": "text" + }, + { + "id": 17, + "type": "tool_call" + }, + { + "id": 18, + "type": "text" + }, + { + "id": 19, + "type": "tool_call" + }, + { + "id": 20, + "type": "text" + }, + { + "id": 21, + "type": "text" + } + ] +} diff --git a/src/journey-set/collect.test.ts b/src/journey-set/collect.test.ts new file mode 100644 index 0000000..0c89249 --- /dev/null +++ b/src/journey-set/collect.test.ts @@ -0,0 +1,462 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import detailFinal from "../api/__fixtures__/studio/journey-detail-final.json"; +import trajectory from "../api/__fixtures__/studio/trajectory.json"; +import { PlatformApiError } from "../api/platform"; +import { StudioApiError, StudioConnectionError, StudioShapeError } from "../api/studio"; +import type { JourneyAgentRun, JourneyDetail } from "../api/studio-types"; +import { collectSet, UNKNOWN_AGENT } from "./collect"; +import { countSet } from "./count"; +import { EXCLUDED_STATUSES } from "./outcome"; +import { appendJourney, createRecord, type JourneyEntry, type SetRecord } from "./record"; +import { type AgentChoice, parseSetSpec, planCells } from "./spec"; +import blockedFirst403 from "./vendor/__fixtures__/run-flags/blocked-first-403.json"; + +// Journey details are api/__fixtures__/studio/journey-detail-final.json (the `{ success, data }` +// envelope unwrapped, as api/studio.test.ts reads it) with only the fields a case needs changed, +// named below as deltas. Trajectories are api/__fixtures__/studio/trajectory.json, except the +// blocked case, whose body carries the steps of vendor/__fixtures__/run-flags/blocked-first-403.json. +// Records are written with createRecord and appendJourney in a temp directory, from a spec and +// cells parsed and planned by spec.ts. + +const FINAL = detailFinal.data as JourneyDetail; +const [HAIKU_ROW, SONNET_ROW] = FINAL.agents; +const HAIKU: AgentChoice = { harness: "claude-code", model: HAIKU_ROW.model as string }; +const SONNET: AgentChoice = { harness: "claude-code", model: SONNET_ROW.model as string }; +const HAIKU_LABEL = "claude-code/claude-haiku-4-5-20251001"; +const SONNET_LABEL = "claude-code/claude-sonnet-5-5"; +const CODEX: AgentChoice = { harness: "codex", model: "gpt-5" }; +const EXACT = "https://linear.app/developers"; + +/** Studio's start failure, in the words the create's failed[] carries (see launch.test.ts). */ +const FAILED_TO_START = "run gate unavailable"; + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ax-collect-")); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +interface SetInput { + agents: AgentChoice[]; + journeys: Omit[]; + site?: string; + goals?: string[]; + exact?: string; + repeat?: string; +} + +function makeRecord(input: SetInput): SetRecord { + const parsed = parseSetSpec({ + levels: true, + site: input.site ?? "linear.app", + goals: input.goals ?? ["Find how to create an issue from the API"], + exact: input.exact ?? EXACT, + repeat: input.repeat, + }); + if (!parsed.ok) throw new Error(parsed.error); + const { cells } = planCells(parsed.spec); + const created = createRecord({ spec: parsed.spec, agents: input.agents, cells }, dir); + let record = created; + for (const journey of input.journeys) { + record = appendJourney(created.id, { ...journey, createdAt: "2026-11-13T14:02:11.482Z" }, dir); + } + return record; +} + +const entry = (fields: Partial = {}): Omit => ({ + journeyId: FINAL.id, + goalIndex: 0, + level: "need", + repeat: 1, + runIds: [HAIKU_ROW.runId, SONNET_ROW.runId], + failed: [], + ...fields, +}); + +/** Delta: the fixture's journey with its rows replaced. */ +const detailWith = (rows: JourneyAgentRun[]): JourneyDetail => ({ ...FINAL, agents: rows }); + +/** Delta: one fixture row with fields changed. */ +const row = (base: JourneyAgentRun, fields: Partial): JourneyAgentRun => ({ + ...base, + ...fields, +}); + +function clientWith(read: (runId: string) => Promise = async () => trajectory) { + const trajectoryRead = vi.fn(read); + return { + client: { trajectory: trajectoryRead } as unknown as Parameters[0]["client"], + trajectoryRead, + }; +} + +async function collect(record: SetRecord, journeys: JourneyDetail[], client = clientWith()) { + return collectSet({ record, journeys, client: client.client }); +} + +describe("started runs", () => { + it("labels each row harness/model, takes bucket from the verdict and reads flags for graded runs", async () => { + const record = makeRecord({ agents: [HAIKU, SONNET], journeys: [entry()] }); + const client = clientWith(); + const out = await collect(record, [FINAL], client); + expect(out.runs).toEqual([ + { + goalIndex: 0, + level: "need", + repeat: 1, + agent: HAIKU_LABEL, + bucket: "satisfied", + flags: expect.objectContaining({ + never_visited: { flagged: true, reason: expect.any(String) }, + }), + }, + { + goalIndex: 0, + level: "need", + repeat: 1, + agent: SONNET_LABEL, + bucket: "unsatisfied", + flags: expect.any(Object), + }, + ]); + expect(out.trajectoryFailures).toBe(0); + expect(client.trajectoryRead.mock.calls.map(([id]) => id)).toEqual([ + HAIKU_ROW.runId, + SONNET_ROW.runId, + ]); + }); + + it("reads trajectories one at a time", async () => { + const record = makeRecord({ agents: [HAIKU, SONNET], journeys: [entry()] }); + let open = 0; + let peak = 0; + const client = clientWith(async () => { + open += 1; + peak = Math.max(peak, open); + await Promise.resolve(); + open -= 1; + return trajectory; + }); + await collect(record, [FINAL], client); + expect(peak).toBe(1); + }); + + it.each([ + ["succeeded", "satisfied", undefined], + ["partial", "partial", undefined], + ["failed", "unsatisfied", undefined], + ["not_measured", "not_graded_yet", undefined], + [null, "not_graded_yet", undefined], + ["mostly_ok", "not_gradable", "mostly_ok"], + ])("maps verdict %s to %s", async (verdict, bucket, rawVerdict) => { + const record = makeRecord({ + agents: [HAIKU], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const detail = detailWith([row(HAIKU_ROW, { verdict })]); + const { runs } = await collect(record, [detail]); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ bucket }); + expect(runs[0].rawVerdict).toBe(rawVerdict); + }); + + it.each(EXCLUDED_STATUSES)( + "excludes status %s with the raw status and reads no trajectory", + async (status) => { + const record = makeRecord({ + agents: [HAIKU], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const detail = detailWith([row(HAIKU_ROW, { status, verdict: null })]); + const client = clientWith(); + const { runs } = await collect(record, [detail], client); + expect(runs[0]).toMatchObject({ bucket: "excluded", rawStatus: status, flags: null }); + expect(client.trajectoryRead).not.toHaveBeenCalled(); + }, + ); + + it("keeps a run in flight with null flags and no trajectory read", async () => { + const record = makeRecord({ + agents: [HAIKU], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const detail = detailWith([row(HAIKU_ROW, { status: "running", verdict: null })]); + const client = clientWith(); + const { runs } = await collect(record, [detail], client); + expect(runs[0]).toMatchObject({ bucket: "in_flight", agent: HAIKU_LABEL, flags: null }); + expect(client.trajectoryRead).not.toHaveBeenCalled(); + }); +}); + +describe("run ids with no row", () => { + it("counts a run id the read did not list as in flight, in the column no other item claimed", async () => { + const record = makeRecord({ agents: [HAIKU, SONNET], journeys: [entry()] }); + const detail = detailWith([HAIKU_ROW]); + const { runs } = await collect(record, [detail]); + expect(runs.map((run) => [run.agent, run.bucket, run.flags])).toEqual([ + [HAIKU_LABEL, "satisfied", expect.any(Object)], + [SONNET_LABEL, "in_flight", null], + ]); + }); + + it("counts every run of a journey that was never read as in flight", async () => { + const record = makeRecord({ agents: [HAIKU, SONNET], journeys: [entry()] }); + const client = clientWith(); + const { runs, shape } = await collect(record, [], client); + expect(runs.map((run) => [run.agent, run.bucket, run.flags])).toEqual([ + [HAIKU_LABEL, "in_flight", null], + [SONNET_LABEL, "in_flight", null], + ]); + expect(shape.agents).toEqual([HAIKU_LABEL, SONNET_LABEL]); + expect(client.trajectoryRead).not.toHaveBeenCalled(); + }); + + it("falls back to the unknown column when no resolved agent is left to take the run", async () => { + const record = makeRecord({ agents: [HAIKU], journeys: [entry()] }); + const { runs, shape } = await collect(record, [detailWith([HAIKU_ROW])]); + expect(runs.map((run) => run.agent)).toEqual([HAIKU_LABEL, UNKNOWN_AGENT]); + expect(shape.agents).toEqual([HAIKU_LABEL, UNKNOWN_AGENT]); + }); +}); + +describe("runs that never started", () => { + it("counts a partial create's failed item as excluded with its reason, beside the run that started", async () => { + const record = makeRecord({ + agents: [HAIKU, CODEX], + journeys: [ + entry({ + runIds: [HAIKU_ROW.runId], + failed: [{ harness: "codex", model: "gpt-5", reason: FAILED_TO_START }], + }), + ], + }); + const { runs } = await collect(record, [detailWith([HAIKU_ROW])]); + expect(runs).toEqual([ + { + goalIndex: 0, + level: "need", + repeat: 1, + agent: "codex/gpt-5", + bucket: "excluded", + reason: FAILED_TO_START, + flags: null, + }, + expect.objectContaining({ agent: HAIKU_LABEL, bucket: "satisfied" }), + ]); + }); + + it("counts a Studio 502 entry with no journey and no run ids as one excluded run per failed agent", async () => { + const record = makeRecord({ + agents: [HAIKU, SONNET], + journeys: [ + entry({ + journeyId: null, + runIds: [], + failed: [ + { harness: "claude-code", model: HAIKU.model, reason: "agent runtime unavailable" }, + { harness: "claude-code", model: SONNET.model, reason: "agent runtime unavailable" }, + ], + }), + ], + }); + const client = clientWith(); + const { runs, trajectoryFailures } = await collect(record, [], client); + expect(runs.map((run) => [run.agent, run.bucket, run.reason])).toEqual([ + [HAIKU_LABEL, "excluded", "agent runtime unavailable"], + [SONNET_LABEL, "excluded", "agent runtime unavailable"], + ]); + expect(trajectoryFailures).toBe(0); + expect(client.trajectoryRead).not.toHaveBeenCalled(); + }); + + it("ignores a row whose run id is not in runIds, so the failed item is not counted twice", async () => { + const orphan = row(SONNET_ROW, { status: "failed", verdict: null }); + const record = makeRecord({ + agents: [HAIKU, SONNET], + journeys: [ + entry({ + runIds: [HAIKU_ROW.runId], + failed: [{ harness: "claude-code", model: SONNET.model, reason: FAILED_TO_START }], + }), + ], + }); + const { runs } = await collect(record, [detailWith([HAIKU_ROW, orphan])]); + expect(runs).toHaveLength(2); + expect(runs.filter((run) => run.agent === SONNET_LABEL)).toEqual([ + expect.objectContaining({ bucket: "excluded", reason: FAILED_TO_START }), + ]); + }); + + it("labels a failed item with no model by its bare harness when that harness is one resolved agent", async () => { + const record = makeRecord({ + agents: [HAIKU, CODEX], + journeys: [ + entry({ + runIds: [HAIKU_ROW.runId], + failed: [{ harness: "codex", reason: FAILED_TO_START }], + }), + ], + }); + const { runs, shape } = await collect(record, [detailWith([HAIKU_ROW])]); + expect(runs[0].agent).toBe("codex/gpt-5"); + expect(shape.agents).toEqual([HAIKU_LABEL, "codex/gpt-5"]); + }); +}); + +describe("trajectory reads", () => { + it("leaves flags null and counts the failure on a 404, and keeps collecting", async () => { + const record = makeRecord({ agents: [HAIKU, SONNET], journeys: [entry()] }); + const missing = new StudioApiError("run not found", { + status: 404, + payload: null, + code: "run_not_found", + missingScopes: [], + }); + const client = clientWith(async (runId) => { + if (runId === HAIKU_ROW.runId) throw missing; + return trajectory; + }); + const out = await collect(record, [FINAL], client); + expect(out.runs.map((run) => [run.agent, run.bucket, run.flags === null])).toEqual([ + [HAIKU_LABEL, "satisfied", true], + [SONNET_LABEL, "unsatisfied", false], + ]); + expect(out.trajectoryFailures).toBe(1); + expect(client.trajectoryRead).toHaveBeenCalledTimes(2); + }); + + it.each([ + ["a platform error", new PlatformApiError("bad gateway", { status: 502 })], + [ + "no connection", + new StudioConnectionError("https://platform.test", new TypeError("fetch failed")), + ], + ["an answer of the wrong shape", new StudioShapeError("GET /trajectory", "a steps list")], + ])("counts %s as a failed read", async (_name, error) => { + const record = makeRecord({ + agents: [HAIKU], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const client = clientWith(async () => { + throw error; + }); + const out = await collect(record, [detailWith([HAIKU_ROW])], client); + expect(out.runs[0].flags).toBeNull(); + expect(out.trajectoryFailures).toBe(1); + }); + + it("rethrows an error it did not expect", async () => { + const record = makeRecord({ + agents: [HAIKU], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const client = clientWith(async () => { + throw new RangeError("not a read failure"); + }); + await expect(collect(record, [detailWith([HAIKU_ROW])], client)).rejects.toThrow(RangeError); + }); + + it("flags blocked on a trajectory whose first step is a refusal (blocked-first-403.json)", async () => { + const site = blockedFirst403.domain; + const record = makeRecord({ + agents: [HAIKU], + site, + goals: ["Hire a freelancer for a logo"], + exact: `https://${site}/start`, + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const client = clientWith(async () => ({ steps: blockedFirst403.trajectory.steps })); + const { runs } = await collect(record, [detailWith([HAIKU_ROW])], client); + expect(runs[0].flags?.blocked.flagged).toBe(true); + expect(runs[0].flags?.blocked.reason).toEqual(expect.any(String)); + }); +}); + +describe("agent columns", () => { + const mismatched = row(HAIKU_ROW, { model: "claude-haiku-4-5" }); + + it("counts a row with a mismatched model id under its own column, and countSet does not throw", async () => { + const record = makeRecord({ + agents: [HAIKU, SONNET], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const { runs, shape } = await collect(record, [detailWith([mismatched])]); + expect(runs[0].agent).toBe("claude-code/claude-haiku-4-5"); + expect(shape.agents).toEqual([HAIKU_LABEL, SONNET_LABEL, "claude-code/claude-haiku-4-5"]); + const count = countSet(runs, shape); + const cell = count.cells.find( + (c) => c.agent === "claude-code/claude-haiku-4-5" && c.level === "need", + ); + expect(cell?.satisfied).toBe(1); + }); + + it("matches by harness alone when exactly one resolved agent has that harness", async () => { + const record = makeRecord({ + agents: [HAIKU], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const { runs, shape } = await collect(record, [detailWith([mismatched])]); + expect(runs[0].agent).toBe(HAIKU_LABEL); + expect(shape.agents).toEqual([HAIKU_LABEL]); + }); + + it("keeps a row with a null model under the bare harness when two resolved agents share it", async () => { + const record = makeRecord({ + agents: [HAIKU, SONNET], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const { runs, shape } = await collect(record, [detailWith([row(HAIKU_ROW, { model: null })])]); + expect(runs[0].agent).toBe("claude-code"); + expect(shape.agents.at(-1)).toBe("claude-code"); + }); +}); + +describe("shape", () => { + it("carries goals, each goal's levels in cell order, the repeat count and the resolved labels in order", async () => { + const record = makeRecord({ + agents: [SONNET, HAIKU], + goals: ["Find how to create an issue from the API", "Export your data"], + repeat: "3", + journeys: [], + }); + const { shape, runs } = await collect(record, []); + expect(runs).toEqual([]); + expect(shape).toEqual({ + goals: record.spec.goals, + levelsByGoal: [ + ["need", "brand", "exact"], + ["need", "brand", "exact"], + ], + agents: [SONNET_LABEL, HAIKU_LABEL], + plannedPerCell: 3, + }); + }); + + it("does not invent runs for cells that were never launched", async () => { + const record = makeRecord({ + agents: [HAIKU], + journeys: [entry({ runIds: [HAIKU_ROW.runId] })], + }); + const { runs, shape } = await collect(record, [detailWith([HAIKU_ROW])]); + expect(runs).toHaveLength(1); + const count = countSet(runs, shape); + expect(count.shortLevels.map((short) => short.level)).toEqual(["brand", "exact"]); + }); + + it("takes goal, level and repeat from the journey entry, not the row", async () => { + const record = makeRecord({ + agents: [HAIKU], + repeat: "2", + journeys: [entry({ level: "exact", repeat: 2, runIds: [HAIKU_ROW.runId] })], + }); + const { runs } = await collect(record, [detailWith([HAIKU_ROW])]); + expect(runs[0]).toMatchObject({ goalIndex: 0, level: "exact", repeat: 2 }); + }); +}); diff --git a/src/journey-set/collect.ts b/src/journey-set/collect.ts new file mode 100644 index 0000000..8f20e16 --- /dev/null +++ b/src/journey-set/collect.ts @@ -0,0 +1,181 @@ +// Turns the local record and the latest journey reads into one CountedRun per launched goal, +// level, repeat and agent, plus the SetShape countSet needs. This is the only producer of +// CountedRun and the only caller of runBucket. It prints nothing and decides no grade: Studio's +// status and verdict go through outcome.ts, and flags come from flags.ts over the stored steps. + +import { isStudioFailure } from "../api/studio"; +import type { JourneyAgentRun, JourneyDetail, StudioClient } from "../api/studio-types"; +import type { CountedRun, SetShape } from "./count"; +import { deriveSetFlags } from "./flags"; +import type { Level } from "./levels"; +import { type RunFacts, runBucket, runPhase } from "./outcome"; +import type { JourneyEntry, SetRecord } from "./record"; +import type { AgentChoice } from "./spec"; + +/** The column for a run that has no row to name its agent and no resolved agent left to take it. */ +export const UNKNOWN_AGENT = "unknown"; + +export interface CollectInput { + record: SetRecord; + /** Whatever the last read returned; a journey that was never read is simply absent. */ + journeys: readonly JourneyDetail[]; + client: Pick; +} + +export interface CollectResult { + runs: CountedRun[]; + shape: SetShape; + /** Graded runs whose trajectory read failed; their flags are null. */ + trajectoryFailures: number; +} + +/** The status given to a run no read has listed: outside both status sets, so it is in flight. */ +const UNSEEN_STATUS = "unseen"; + +/** A run before its flags are read. */ +interface Pending { + run: CountedRun; + /** Set only on a graded run: the one whose trajectory is read. */ + gradedRunId?: string; +} + +/** `harness/model`, or the bare harness when there is no model (Shared vocabulary 10). */ +function agentLabel(agent: { harness: string; model?: string | null }): string { + return agent.model ? `${agent.harness}/${agent.model}` : agent.harness; +} + +/** + * The column an item belongs to: its label when that is a resolved label; the one resolved agent + * with its harness when there is exactly one; otherwise its own label, which the shape then adds. + */ +function columnFor( + item: { harness: string; model?: string | null }, + resolved: readonly AgentChoice[], +): string { + const own = agentLabel(item); + if (resolved.some((agent) => agentLabel(agent) === own)) return own; + const sameHarness = resolved.filter((agent) => agent.harness === item.harness); + return sameHarness.length === 1 ? agentLabel(sameHarness[0]) : own; +} + +function cellOf(entry: JourneyEntry): { goalIndex: number; level: Level; repeat: number } { + return { goalIndex: entry.goalIndex, level: entry.level, repeat: entry.repeat }; +} + +function neverStarted(entry: JourneyEntry, resolved: readonly AgentChoice[]): Pending[] { + return entry.failed.map((item) => ({ + run: { + ...cellOf(entry), + agent: columnFor(item, resolved), + ...runBucket({ started: false, reason: item.reason }), + flags: null, + }, + })); +} + +function startedRun( + entry: JourneyEntry, + row: JourneyAgentRun, + resolved: readonly AgentChoice[], +): Pending { + const facts: RunFacts = { started: true, status: row.status, verdict: row.verdict }; + const run: CountedRun = { + ...cellOf(entry), + agent: columnFor(row, resolved), + ...runBucket(facts), + flags: null, + }; + return runPhase(row.status) === "graded" ? { run, gradedRunId: row.runId } : { run }; +} + +/** Run ids the journey reads did not list are in flight as far as ax knows. */ +function unseenRuns( + entry: JourneyEntry, + claimed: ReadonlySet, + resolved: readonly AgentChoice[], + count: number, +): Pending[] { + const free = resolved.map(agentLabel).filter((label) => !claimed.has(label)); + return Array.from({ length: count }, (_, index) => ({ + run: { + ...cellOf(entry), + agent: free[index] ?? UNKNOWN_AGENT, + ...runBucket({ started: true, status: UNSEEN_STATUS }), + flags: null, + }, + })); +} + +/** + * Every run id the entry started: its row when the journey read lists it, else in flight. + * Rows outside `runIds` are ignored: Studio writes a journey binding before it creates the run, + * so an agent that failed to start can leave a row that `failed[]` already counted. + */ +function startedRuns( + entry: JourneyEntry, + detail: JourneyDetail | undefined, + resolved: readonly AgentChoice[], +): Pending[] { + const rows = new Map((detail?.agents ?? []).map((row) => [row.runId, row])); + const found: Pending[] = []; + let unseen = 0; + for (const runId of new Set(entry.runIds)) { + const row = rows.get(runId); + if (row) found.push(startedRun(entry, row, resolved)); + else unseen += 1; + } + const claimed = new Set([ + ...entry.failed.map((item) => columnFor(item, resolved)), + ...found.map((pending) => pending.run.agent), + ]); + return [...found, ...unseenRuns(entry, claimed, resolved, unseen)]; +} + +/** One read per graded run, in sequence; a failed read leaves flags null and is counted. */ +async function readFlags( + pending: readonly Pending[], + site: string, + client: Pick, +): Promise { + let failures = 0; + for (const { run, gradedRunId } of pending) { + if (gradedRunId === undefined) continue; + try { + const { steps } = await client.trajectory(gradedRunId); + run.flags = deriveSetFlags(steps, site); + } catch (error) { + if (!isStudioFailure(error)) throw error; + failures += 1; + } + } + return failures; +} + +function buildShape(record: SetRecord, runs: readonly CountedRun[]): SetShape { + const agents = record.agents.map(agentLabel); + for (const run of runs) if (!agents.includes(run.agent)) agents.push(run.agent); + return { + goals: record.spec.goals, + levelsByGoal: record.spec.goals.map((_, goalIndex) => + record.cells.filter((cell) => cell.goalIndex === goalIndex).map((cell) => cell.level), + ), + agents, + plannedPerCell: record.spec.repeat, + }; +} + +export async function collectSet(input: CollectInput): Promise { + const { record, journeys, client } = input; + const details = new Map(journeys.map((journey) => [journey.id, journey])); + const pending: Pending[] = []; + for (const entry of record.journeys) { + const detail = entry.journeyId === null ? undefined : details.get(entry.journeyId); + pending.push( + ...neverStarted(entry, record.agents), + ...startedRuns(entry, detail, record.agents), + ); + } + const trajectoryFailures = await readFlags(pending, record.spec.site, client); + const runs = pending.map((item) => item.run); + return { runs, shape: buildShape(record, runs), trajectoryFailures }; +} diff --git a/src/journey-set/count.test.ts b/src/journey-set/count.test.ts new file mode 100644 index 0000000..2ae0346 --- /dev/null +++ b/src/journey-set/count.test.ts @@ -0,0 +1,514 @@ +import { describe, expect, it } from "vitest"; +import studioOraAi from "./__fixtures__/studio-trajectories/run-49ab0c88.json"; +import { + type CountedRun, + countSet, + GAP_MIN_DIFFERENCE, + LEVEL_LAYER, + RATE_FLOOR, + type SetCount, + type SetShape, +} from "./count"; +import { deriveSetFlags, type SetStep } from "./flags"; +import type { Level } from "./levels"; +import type { Bucket } from "./outcome"; +import blockedFirst403 from "./vendor/__fixtures__/run-flags/blocked-first-403.json"; + +// count.ts is arithmetic over runs, so every input here is a CountedRun array built in this file +// and named for what it isolates; there is no recorded payload to copy. Where a run needs a +// server value, it is Studio's own: FAILED_TO_START is the start error in oramono +// apps/studio-service/src/journeys/journeys-service.test.ts:629 (the value outcome.test.ts uses). +// Flags come from deriveSetFlags over flags.test.ts's fixtures, so their shape is the producer's. + +const AGENT_ONE = "agent-one/model-a"; +const AGENT_TWO = "agent-two/model-b"; +const FAILED_TO_START = "run gate unavailable"; + +const BLOCKED = deriveSetFlags( + blockedFirst403.trajectory.steps as SetStep[], + blockedFirst403.domain, +); +const NEVER_VISITED = deriveSetFlags(studioOraAi.steps, "linear.app"); +const CLEAN = deriveSetFlags(studioOraAi.steps, "ora.ai"); + +function run( + level: Level, + bucket: Bucket, + repeat: number, + fields: Partial = {}, +): CountedRun { + return { goalIndex: 0, level, agent: AGENT_ONE, repeat, bucket, flags: null, ...fields }; +} + +/** One run per bucket listed, repeats numbered from 1. */ +function runsOf(level: Level, buckets: Bucket[], fields: Partial = {}): CountedRun[] { + return buckets.map((bucket, index) => run(level, bucket, index + 1, fields)); +} + +function shape( + levelsByGoal: Level[][], + agents: string[] = [AGENT_ONE], + plannedPerCell = 1, +): SetShape { + return { + goals: levelsByGoal.map((_, index) => `Goal ${index + 1}`), + levelsByGoal, + agents, + plannedPerCell, + }; +} + +function cellOf(count: SetCount, level: Level, agent = AGENT_ONE, goalIndex = 0) { + const cell = count.cells.find( + (entry) => entry.goalIndex === goalIndex && entry.level === level && entry.agent === agent, + ); + if (!cell) throw new Error(`no cell for goalIndex ${goalIndex}, ${level}, ${agent}`); + return cell; +} + +describe("constants", () => { + it("pins the rate floor, the gap threshold and each upper level's layer", () => { + expect(RATE_FLOOR).toBe(5); + expect(GAP_MIN_DIFFERENCE).toBe(2); + expect(LEVEL_LAYER).toEqual({ + brand: "Discovery", + docs: "Discovery", + exact: "Access and Usability", + }); + }); +}); + +describe("cells", () => { + it("an empty set has no cells, gaps or short levels", () => { + expect(countSet([], shape([]))).toEqual({ cells: [], gaps: [], shortLevels: [] }); + }); + + it("plans one cell per goal, level and agent in ladder order, each with no data before runs", () => { + const LEVELS_GIVEN_OUT_OF_ORDER: Level[][] = [["exact", "brand"], ["need"]]; + const count = countSet([], shape(LEVELS_GIVEN_OUT_OF_ORDER, [AGENT_ONE, AGENT_TWO])); + expect(count.cells.map((cell) => [cell.goalIndex, cell.level, cell.agent])).toEqual([ + [0, "brand", AGENT_ONE], + [0, "brand", AGENT_TWO], + [0, "exact", AGENT_ONE], + [0, "exact", AGENT_TWO], + [1, "need", AGENT_ONE], + [1, "need", AGENT_TWO], + ]); + for (const cell of count.cells) { + expect(cell).toMatchObject({ runs: 0, gradable: 0, rate: null }); + } + }); + + it("a cell with 5 gradable runs shows a rate of satisfied over gradable", () => { + const FIVE_GRADABLE = runsOf("brand", [ + "satisfied", + "satisfied", + "satisfied", + "partial", + "unsatisfied", + ]); + expect( + cellOf(countSet(FIVE_GRADABLE, shape([["brand"]], [AGENT_ONE], 5)), "brand"), + ).toMatchObject({ runs: 5, satisfied: 3, partial: 1, unsatisfied: 1, gradable: 5, rate: 0.6 }); + }); + + it("a cell with 3 gradable runs shows counts only", () => { + const THREE_GRADABLE = runsOf("brand", ["satisfied", "satisfied", "unsatisfied"]); + expect( + cellOf(countSet(THREE_GRADABLE, shape([["brand"]], [AGENT_ONE], 3)), "brand"), + ).toMatchObject({ runs: 3, satisfied: 2, partial: 0, unsatisfied: 1, gradable: 3, rate: null }); + }); + + it("one gradable run under the floor still shows counts only", () => { + const FOUR_GRADABLE = runsOf("brand", ["satisfied", "satisfied", "satisfied", "satisfied"]); + expect(cellOf(countSet(FOUR_GRADABLE, shape([["brand"]], [AGENT_ONE], 4)), "brand").rate).toBe( + null, + ); + }); + + it("runs outside the three gradable buckets never reach the rate", () => { + const FIVE_GRADABLE_AND_ONE_OF_EACH_OTHER = [ + ...runsOf("brand", ["satisfied", "satisfied", "satisfied", "partial", "unsatisfied"]), + run("brand", "not_graded_yet", 6), + run("brand", "in_flight", 7), + run("brand", "excluded", 8, { rawStatus: "lost" }), + run("brand", "excluded", 9, { reason: FAILED_TO_START }), + run("brand", "not_gradable", 10, { rawVerdict: "abandoned" }), + ]; + const cell = cellOf( + countSet(FIVE_GRADABLE_AND_ONE_OF_EACH_OTHER, shape([["brand"]], [AGENT_ONE], 10)), + "brand", + ); + expect(cell).toMatchObject({ + runs: 10, + gradable: 5, + rate: 0.6, + notGradedYet: 1, + inFlight: 1, + }); + }); + + it("tallies excluded runs by raw status, never started runs by reason, and not gradable by verdict", () => { + const EXCLUDED_NEVER_STARTED_AND_UNGRADABLE: CountedRun[] = [ + run("brand", "excluded", 1, { rawStatus: "failed" }), + run("brand", "excluded", 2, { rawStatus: "lost" }), + run("brand", "excluded", 3, { rawStatus: "failed" }), + run("brand", "excluded", 4, { reason: FAILED_TO_START }), + run("brand", "excluded", 5, { reason: FAILED_TO_START }), + run("brand", "not_gradable", 6, { rawVerdict: "constructor" }), + run("brand", "not_gradable", 7, { rawVerdict: "abandoned" }), + ]; + const cell = cellOf( + countSet(EXCLUDED_NEVER_STARTED_AND_UNGRADABLE, shape([["brand"]], [AGENT_ONE], 7)), + "brand", + ); + expect(cell.excluded).toEqual([ + { value: "failed", count: 2 }, + { value: "lost", count: 1 }, + ]); + expect(cell.neverStarted).toEqual([{ value: FAILED_TO_START, count: 2 }]); + expect(cell.notGradable).toEqual([ + { value: "abandoned", count: 1 }, + { value: "constructor", count: 1 }, + ]); + expect(cell).toMatchObject({ runs: 7, gradable: 0, rate: null }); + }); + + it("counts the blocked and never visited flags per cell, and a run with no flags as neither", () => { + expect([BLOCKED.blocked.flagged, BLOCKED.never_visited.flagged]).toEqual([true, false]); + expect([NEVER_VISITED.blocked.flagged, NEVER_VISITED.never_visited.flagged]).toEqual([ + false, + true, + ]); + expect([CLEAN.blocked.flagged, CLEAN.never_visited.flagged]).toEqual([false, false]); + const FLAGGED_RUNS_ACROSS_TWO_AGENTS: CountedRun[] = [ + run("need", "unsatisfied", 1, { flags: BLOCKED }), + run("need", "unsatisfied", 2, { flags: NEVER_VISITED }), + run("need", "unsatisfied", 3, { flags: NEVER_VISITED }), + run("need", "satisfied", 4, { flags: CLEAN }), + run("need", "unsatisfied", 5, { flags: null }), + run("need", "unsatisfied", 1, { agent: AGENT_TWO, flags: BLOCKED }), + ]; + const count = countSet( + FLAGGED_RUNS_ACROSS_TWO_AGENTS, + shape([["need"]], [AGENT_ONE, AGENT_TWO], 5), + ); + expect(cellOf(count, "need").flags).toEqual({ blocked: 1, never_visited: 2 }); + expect(cellOf(count, "need", AGENT_TWO).flags).toEqual({ blocked: 1, never_visited: 0 }); + }); + + it("refuses a run that has no cell in the set instead of dropping it", () => { + const planned = shape([["brand"]]); + expect(() => countSet([run("brand", "satisfied", 1, { agent: AGENT_TWO })], planned)).toThrow( + RangeError, + ); + expect(() => countSet([run("docs", "satisfied", 1)], planned)).toThrow(RangeError); + expect(() => countSet([run("brand", "satisfied", 1, { goalIndex: 1 })], planned)).toThrow( + RangeError, + ); + }); +}); + +describe("gap lines", () => { + it("fires when adjacent levels differ by exactly 2 satisfied runs, naming the upper level's layer", () => { + const NEED_ONE_BRAND_THREE = [ + ...runsOf("need", ["satisfied", "unsatisfied", "unsatisfied"]), + ...runsOf("brand", ["satisfied", "satisfied", "satisfied"]), + ]; + expect(countSet(NEED_ONE_BRAND_THREE, shape([["need", "brand"]], [AGENT_ONE], 3)).gaps).toEqual( + [ + { + goalIndex: 0, + lower: { level: "need", satisfied: 1, gradable: 3 }, + upper: { level: "brand", satisfied: 3, gradable: 3 }, + difference: 2, + layer: "Discovery", + earlySignal: true, + }, + ], + ); + }); + + it("does not fire at a difference of 1", () => { + const NEED_ONE_BRAND_TWO = [ + ...runsOf("need", ["satisfied", "unsatisfied", "unsatisfied"]), + ...runsOf("brand", ["satisfied", "satisfied", "unsatisfied"]), + ]; + expect(countSet(NEED_ONE_BRAND_TWO, shape([["need", "brand"]], [AGENT_ONE], 3)).gaps).toEqual( + [], + ); + }); + + it("fires when more help satisfies fewer runs, with a negative difference", () => { + const DOCS_THREE_EXACT_ONE = [ + ...runsOf("docs", ["satisfied", "satisfied", "satisfied"]), + ...runsOf("exact", ["satisfied", "unsatisfied", "unsatisfied"]), + ]; + const [gap] = countSet(DOCS_THREE_EXACT_ONE, shape([["docs", "exact"]], [AGENT_ONE], 3)).gaps; + expect(gap).toMatchObject({ difference: -2, layer: "Access and Usability" }); + }); + + it("compares rates, not counts, when the levels have different gradable runs", () => { + const BRAND_FIVE_OF_FIVE_EXACT_TWO_OF_TWO = [ + ...runsOf("brand", ["satisfied", "satisfied", "satisfied", "satisfied", "satisfied"]), + ...runsOf("exact", ["satisfied", "satisfied", "excluded", "excluded", "excluded"]), + ]; + expect( + countSet(BRAND_FIVE_OF_FIVE_EXACT_TWO_OF_TWO, shape([["brand", "exact"]], [AGENT_ONE], 5)) + .gaps, + ).toEqual([]); + }); + + it("scales a rate gap to the smaller level's gradable runs", () => { + const NEED_ONE_OF_SEVEN_BRAND_NINE_OF_NINE = [ + ...runsOf("need", [ + "satisfied", + ...Array(6).fill("unsatisfied"), + ...Array(2).fill("excluded"), + ]), + ...runsOf("brand", Array(9).fill("satisfied")), + ]; + const [gap] = countSet( + NEED_ONE_OF_SEVEN_BRAND_NINE_OF_NINE, + shape([["need", "brand"]], [AGENT_ONE], 9), + ).gaps; + expect(gap).toMatchObject({ + lower: { level: "need", satisfied: 1, gradable: 7 }, + upper: { level: "brand", satisfied: 9, gradable: 9 }, + difference: 6, + }); + }); + + it("does not fire when the scaled gap is under 2, however far the counts differ", () => { + const DOCS_TWO_OF_FOUR_EXACT_SEVEN_OF_EIGHT = [ + ...runsOf("docs", [ + "satisfied", + "satisfied", + "unsatisfied", + "unsatisfied", + ...Array(4).fill("excluded"), + ]), + ...runsOf("exact", [...Array(7).fill("satisfied"), "unsatisfied"]), + ]; + expect( + countSet(DOCS_TWO_OF_FOUR_EXACT_SEVEN_OF_EIGHT, shape([["docs", "exact"]], [AGENT_ONE], 8)) + .gaps, + ).toEqual([]); + }); + + it("names the layer of the upper level at every rung of the ladder", () => { + const EACH_RUNG_TWO_MORE_SATISFIED = [ + ...runsOf("need", [ + "unsatisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + ]), + ...runsOf("brand", [ + "satisfied", + "satisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + ]), + ...runsOf("docs", [ + "satisfied", + "satisfied", + "satisfied", + "satisfied", + "unsatisfied", + "unsatisfied", + ]), + ...runsOf("exact", [ + "satisfied", + "satisfied", + "satisfied", + "satisfied", + "satisfied", + "satisfied", + ]), + ]; + const count = countSet( + EACH_RUNG_TWO_MORE_SATISFIED, + shape([["need", "brand", "docs", "exact"]], [AGENT_ONE], 6), + ); + expect(count.gaps.map((gap) => [gap.lower.level, gap.upper.level, gap.layer])).toEqual([ + ["need", "brand", "Discovery"], + ["brand", "docs", "Discovery"], + ["docs", "exact", "Access and Usability"], + ]); + }); + + it("compares brand to exact when the docs level was skipped", () => { + const BRAND_ONE_EXACT_THREE = [ + ...runsOf("brand", ["satisfied", "unsatisfied", "unsatisfied"]), + ...runsOf("exact", ["satisfied", "satisfied", "satisfied"]), + ]; + const [gap] = countSet(BRAND_ONE_EXACT_THREE, shape([["brand", "exact"]], [AGENT_ONE], 3)).gaps; + expect(gap).toMatchObject({ + lower: { level: "brand" }, + upper: { level: "exact" }, + layer: "Access and Usability", + }); + }); + + it("skips a planned level with no gradable run, comparing the levels on either side", () => { + const DOCS_ALL_EXCLUDED_OR_IN_FLIGHT = [ + ...runsOf("brand", ["satisfied", "unsatisfied", "unsatisfied"]), + run("docs", "excluded", 1, { rawStatus: "lost" }), + run("docs", "excluded", 2, { reason: FAILED_TO_START }), + run("docs", "in_flight", 3), + ...runsOf("exact", ["satisfied", "satisfied", "satisfied"]), + ]; + const count = countSet( + DOCS_ALL_EXCLUDED_OR_IN_FLIGHT, + shape([["brand", "docs", "exact"]], [AGENT_ONE], 3), + ); + expect(count.gaps.map((gap) => [gap.lower.level, gap.upper.level])).toEqual([ + ["brand", "exact"], + ]); + }); + + it("sums satisfied and gradable runs over the agents, so a gap never compares agents", () => { + const NO_AGENT_DIFFERS_BY_TWO_BUT_THE_TASK_DOES = [ + ...runsOf("need", ["satisfied", "unsatisfied", "unsatisfied"]), + ...runsOf("need", ["unsatisfied", "unsatisfied", "unsatisfied"], { agent: AGENT_TWO }), + ...runsOf("brand", ["satisfied", "satisfied", "unsatisfied"]), + ...runsOf("brand", ["satisfied", "unsatisfied", "unsatisfied"], { agent: AGENT_TWO }), + ]; + expect( + countSet( + NO_AGENT_DIFFERS_BY_TWO_BUT_THE_TASK_DOES, + shape([["need", "brand"]], [AGENT_ONE, AGENT_TWO], 3), + ).gaps, + ).toEqual([ + { + goalIndex: 0, + lower: { level: "need", satisfied: 1, gradable: 6 }, + upper: { level: "brand", satisfied: 3, gradable: 6 }, + difference: 2, + layer: "Discovery", + earlySignal: false, + }, + ]); + }); + + it("tags an early signal when either level has fewer than 5 gradable runs", () => { + const NEED_FOUR_GRADABLE_BRAND_FIVE = [ + ...runsOf("need", ["unsatisfied", "unsatisfied", "unsatisfied", "unsatisfied"]), + run("need", "excluded", 5, { rawStatus: "timeout" }), + ...runsOf("brand", ["satisfied", "satisfied", "satisfied", "unsatisfied", "unsatisfied"]), + ]; + const NEED_FIVE_GRADABLE_BRAND_FOUR = [ + ...runsOf("need", [ + "unsatisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + ]), + ...runsOf("brand", ["satisfied", "satisfied", "satisfied", "satisfied"]), + run("brand", "not_graded_yet", 5), + ]; + const BOTH_FIVE_GRADABLE = [ + ...runsOf("need", [ + "unsatisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + "unsatisfied", + ]), + ...runsOf("brand", ["satisfied", "satisfied", "satisfied", "unsatisfied", "unsatisfied"]), + ]; + const planned = shape([["need", "brand"]], [AGENT_ONE], 5); + expect(countSet(NEED_FOUR_GRADABLE_BRAND_FIVE, planned).gaps[0].earlySignal).toBe(true); + expect(countSet(NEED_FIVE_GRADABLE_BRAND_FOUR, planned).gaps[0].earlySignal).toBe(true); + expect(countSet(BOTH_FIVE_GRADABLE, planned).gaps[0].earlySignal).toBe(false); + }); + + it("keeps each goal's gaps to its own levels", () => { + const TASK_ONE_GAP_TASK_TWO_FLAT = [ + ...runsOf("need", ["unsatisfied", "unsatisfied"]), + ...runsOf("brand", ["satisfied", "satisfied"]), + ...runsOf("brand", ["satisfied", "unsatisfied"], { goalIndex: 1 }), + ...runsOf("exact", ["satisfied", "satisfied"], { goalIndex: 1 }), + ]; + const count = countSet( + TASK_ONE_GAP_TASK_TWO_FLAT, + shape( + [ + ["need", "brand"], + ["brand", "exact"], + ], + [AGENT_ONE], + 2, + ), + ); + expect(count.gaps.map((gap) => [gap.goalIndex, gap.lower.level, gap.upper.level])).toEqual([ + [0, "need", "brand"], + ]); + }); +}); + +describe("a set without levels", () => { + it("counts each goal's written cell and draws no gap, since one level has no neighbor", () => { + const runs = [ + ...runsOf("written", ["satisfied", "satisfied", "satisfied", "satisfied", "satisfied"]), + ...runsOf("written", ["unsatisfied", "unsatisfied", "unsatisfied", "partial", "partial"], { + goalIndex: 1, + }), + ]; + const count = countSet(runs, shape([["written"], ["written"]], [AGENT_ONE], 5)); + expect(count.cells.map((cell) => [cell.goalIndex, cell.level, cell.rate])).toEqual([ + [0, "written", 1], + [1, "written", 0], + ]); + expect(count.gaps).toEqual([]); + expect(count.shortLevels).toEqual([]); + }); +}); + +describe("short levels", () => { + it("marks a level a 402 stopped short, counting a run that failed to start as launched", () => { + // Repeat first over brand and exact with two agents and --repeat 3: repeats 1 and 2 launched + // both levels, repeat 3 launched brand and the 402 came at exact. + const STOPPED_BEFORE_EXACT_REPEAT_3: CountedRun[] = [1, 2, 3].flatMap((repeat) => [ + run("brand", "satisfied", repeat), + run("brand", "excluded", repeat, { agent: AGENT_TWO, reason: FAILED_TO_START }), + ...(repeat < 3 + ? [ + run("exact", "satisfied", repeat), + run("exact", "satisfied", repeat, { agent: AGENT_TWO }), + ] + : []), + ]); + const count = countSet( + STOPPED_BEFORE_EXACT_REPEAT_3, + shape([["brand", "exact"]], [AGENT_ONE, AGENT_TWO], 3), + ); + expect(count.shortLevels).toEqual([{ goalIndex: 0, level: "exact", launched: 4, planned: 6 }]); + }); + + it("marks every planned level short when the stop came before the first create", () => { + const count = countSet([], shape([["need", "brand"], ["brand"]], [AGENT_ONE], 2)); + expect(count.shortLevels).toEqual([ + { goalIndex: 0, level: "need", launched: 0, planned: 2 }, + { goalIndex: 0, level: "brand", launched: 0, planned: 2 }, + { goalIndex: 1, level: "brand", launched: 0, planned: 2 }, + ]); + expect(count.gaps).toEqual([]); + }); + + it("marks nothing short when every planned run launched", () => { + const EVERY_PLANNED_RUN = [ + ...runsOf("brand", ["satisfied", "in_flight"]), + ...runsOf("exact", ["not_graded_yet", "unsatisfied"]), + ]; + expect( + countSet(EVERY_PLANNED_RUN, shape([["brand", "exact"]], [AGENT_ONE], 2)).shortLevels, + ).toEqual([]); + }); +}); diff --git a/src/journey-set/count.ts b/src/journey-set/count.ts new file mode 100644 index 0000000..f5dbf03 --- /dev/null +++ b/src/journey-set/count.ts @@ -0,0 +1,276 @@ +// A set's counts: one cell per goal, level and agent, and the gap lines between adjacent levels. +// Pure arithmetic over runs whose bucket outcome.ts already derived; this decides no run's grade. + +import type { SetFlags } from "./flags"; +import { LEVELS, type Level } from "./levels"; +import type { Bucket } from "./outcome"; + +/** Gradable runs a cell needs to show a rate, and a level needs for its gap lines not to be early signals. */ +export const RATE_FLOOR = 5; + +/** Satisfied runs two adjacent levels must differ by before a gap line prints, at the smaller level's size. */ +export const GAP_MIN_DIFFERENCE = 2; + +/** + * The score layer that covers a gap, named by its upper level. Need is never the upper level, and + * written is the only level of its goal, so it has no neighbor to gap with. + */ +export const LEVEL_LAYER = { + brand: "Discovery", + docs: "Discovery", + exact: "Access and Usability", +} as const satisfies Readonly, string>>; + +export type Layer = (typeof LEVEL_LAYER)[keyof typeof LEVEL_LAYER]; + +/** One launched run, as collect.ts produces it from the record and the journey reads. */ +export type CountedRun = { + /** Index into `SetShape.goals`. */ + goalIndex: number; + level: Level; + agent: string; + repeat: number; + bucket: Bucket; + rawStatus?: string; + rawVerdict?: string; + reason?: string; + flags: SetFlags | null; +}; + +/** What the set planned, so cells with no runs and levels stopped short still show. */ +export interface SetShape { + /** The set's goals in order; a run's `goalIndex` indexes this list. */ + goals: readonly string[]; + /** The levels each goal runs (skipped levels absent), indexed like `goals`. */ + levelsByGoal: readonly (readonly Level[])[]; + /** One report column per agent, in order; a run's `agent` is one of these. */ + agents: readonly string[]; + /** Runs each goal x level x agent cell was planned to launch: the repeat count. */ + plannedPerCell: number; +} + +/** Server strings with how many runs carried each, most first, then by value. */ +export type Tally = { value: string; count: number }[]; + +/** Runs in the cell that carry each flag. A run whose flags were not read counts as neither. */ +export type FlagCounts = Record; + +export interface CellCount { + goalIndex: number; + level: Level; + agent: string; + /** Every run counted here, whatever its bucket; 0 means no data. */ + runs: number; + satisfied: number; + partial: number; + unsatisfied: number; + /** satisfied + partial + unsatisfied. */ + gradable: number; + /** satisfied / gradable, 0 to 1, from RATE_FLOOR gradable runs; null below it. */ + rate: number | null; + notGradedYet: number; + inFlight: number; + /** Runs that started and ended in an excluded status, by raw status. */ + excluded: Tally; + /** Runs that never started, by reason. Excluded too: never counted against the site. */ + neverStarted: Tally; + /** Graded runs whose verdict maps to no bucket, by raw verdict. */ + notGradable: Tally; + flags: FlagCounts; +} + +/** One level of one goal, summed over the set's agents. */ +export interface LevelTotal { + level: Level; + satisfied: number; + gradable: number; +} + +export interface GapLine { + goalIndex: number; + lower: LevelTotal; + upper: LevelTotal; + /** + * The rate gap in satisfied runs at the smaller level's size (upper.satisfied - lower.satisfied + * when both have the same gradable runs): positive when more help satisfied more runs. + */ + difference: number; + layer: Layer; + /** Either level has fewer than RATE_FLOOR gradable runs. */ + earlySignal: boolean; +} + +/** A level that launched fewer runs than planned, because launching stopped (a 402, for one). */ +export interface ShortLevel { + goalIndex: number; + level: Level; + launched: number; + planned: number; +} + +export interface SetCount { + cells: CellCount[]; + gaps: GapLine[]; + shortLevels: ShortLevel[]; +} + +/** A goal's levels in report order (LEVELS), whatever order the shape lists them in. */ +function plannedLevels(shape: SetShape, goalIndex: number): Level[] { + const planned = shape.levelsByGoal[goalIndex] ?? []; + return LEVELS.filter((level) => planned.includes(level)); +} + +function hasCell(shape: SetShape, run: CountedRun): boolean { + return ( + Number.isInteger(run.goalIndex) && + run.goalIndex >= 0 && + run.goalIndex < shape.goals.length && + plannedLevels(shape, run.goalIndex).includes(run.level) && + shape.agents.includes(run.agent) + ); +} + +/** A run outside the plan is a producer bug; dropping it silently would miscount the set. */ +function assertEveryRunHasACell(runs: readonly CountedRun[], shape: SetShape): void { + const stray = runs.find((run) => !hasCell(shape, run)); + if (stray) { + throw new RangeError( + `a run of goal ${stray.goalIndex}, level ${stray.level}, agent ${stray.agent} has no cell in the set`, + ); + } +} + +// A Map, not an object: the values are server strings, and "constructor" must not resolve. +function tally(values: readonly string[]): Tally { + const counts = new Map(); + for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1); + return [...counts] + .map(([value, count]) => ({ value, count })) + .sort((a, b) => b.count - a.count || (a.value < b.value ? -1 : 1)); +} + +function countBucket(runs: readonly CountedRun[], bucket: Bucket): number { + return runs.filter((run) => run.bucket === bucket).length; +} + +function flagCounts(runs: readonly CountedRun[]): FlagCounts { + return { + blocked: runs.filter((run) => run.flags?.blocked.flagged).length, + never_visited: runs.filter((run) => run.flags?.never_visited.flagged).length, + }; +} + +/** A started run that was excluded carries its raw status; one that never started, its reason. */ +function excludedTallies( + runs: readonly CountedRun[], +): Pick { + const excluded = runs.filter((run) => run.bucket === "excluded"); + return { + excluded: tally(excluded.flatMap((run) => (run.rawStatus ? [run.rawStatus] : []))), + neverStarted: tally(excluded.flatMap((run) => (run.rawStatus ? [] : [run.reason ?? ""]))), + }; +} + +function countCell( + runs: readonly CountedRun[], + goalIndex: number, + level: Level, + agent: string, +): CellCount { + const inCell = runs.filter( + (run) => run.goalIndex === goalIndex && run.level === level && run.agent === agent, + ); + const satisfied = countBucket(inCell, "satisfied"); + const partial = countBucket(inCell, "partial"); + const unsatisfied = countBucket(inCell, "unsatisfied"); + const gradable = satisfied + partial + unsatisfied; + const notGradable = inCell.filter((run) => run.bucket === "not_gradable"); + return { + goalIndex, + level, + agent, + runs: inCell.length, + satisfied, + partial, + unsatisfied, + gradable, + rate: gradable >= RATE_FLOOR ? satisfied / gradable : null, + notGradedYet: countBucket(inCell, "not_graded_yet"), + inFlight: countBucket(inCell, "in_flight"), + ...excludedTallies(inCell), + notGradable: tally(notGradable.map((run) => run.rawVerdict ?? "")), + flags: flagCounts(inCell), + }; +} + +function countCells(runs: readonly CountedRun[], shape: SetShape): CellCount[] { + return shape.goals.flatMap((_, goalIndex) => + plannedLevels(shape, goalIndex).flatMap((level) => + shape.agents.map((agent) => countCell(runs, goalIndex, level, agent)), + ), + ); +} + +function levelTotal(cells: readonly CellCount[], goalIndex: number, level: Level): LevelTotal { + const atLevel = cells.filter((cell) => cell.goalIndex === goalIndex && cell.level === level); + return { + level, + satisfied: atLevel.reduce((sum, cell) => sum + cell.satisfied, 0), + gradable: atLevel.reduce((sum, cell) => sum + cell.gradable, 0), + }; +} + +/** + * The rate gap between two levels, in satisfied runs at the smaller level's size: the plain + * difference in satisfied runs when both levels have the same gradable runs. The threshold is + * compared in whole numbers, so no rounding decides whether a line prints. + */ +function gapBetween(goalIndex: number, lower: LevelTotal, upper: LevelTotal): GapLine | null { + const crossed = upper.satisfied * lower.gradable - lower.satisfied * upper.gradable; + const larger = Math.max(lower.gradable, upper.gradable); + if (upper.level === "need" || upper.level === "written") return null; + if (Math.abs(crossed) < GAP_MIN_DIFFERENCE * larger) return null; + return { + goalIndex, + lower, + upper, + difference: crossed / larger, + layer: LEVEL_LAYER[upper.level], + earlySignal: lower.gradable < RATE_FLOOR || upper.gradable < RATE_FLOOR, + }; +} + +/** + * Gaps between adjacent levels among those that ran for the goal, summed over agents, so a gap + * line is about the site and never compares agents. A level with no gradable run has no satisfied + * count to compare (skipped, stopped before launch, or every run excluded or in flight). + */ +function goalGaps(cells: readonly CellCount[], shape: SetShape, goalIndex: number): GapLine[] { + const ran = plannedLevels(shape, goalIndex) + .map((level) => levelTotal(cells, goalIndex, level)) + .filter((total) => total.gradable > 0); + return ran.slice(1).flatMap((upper, index) => gapBetween(goalIndex, ran[index], upper) ?? []); +} + +function shortLevels(runs: readonly CountedRun[], shape: SetShape): ShortLevel[] { + const planned = shape.plannedPerCell * shape.agents.length; + return shape.goals.flatMap((_, goalIndex) => + plannedLevels(shape, goalIndex).flatMap((level) => { + const launched = runs.filter( + (run) => run.goalIndex === goalIndex && run.level === level, + ).length; + return launched < planned ? [{ goalIndex, level, launched, planned }] : []; + }), + ); +} + +/** Every cell of the plan (goal, level, agent), the gap lines per goal, and the short levels. */ +export function countSet(runs: readonly CountedRun[], shape: SetShape): SetCount { + assertEveryRunHasACell(runs, shape); + const cells = countCells(runs, shape); + return { + cells, + gaps: shape.goals.flatMap((_, goalIndex) => goalGaps(cells, shape, goalIndex)), + shortLevels: shortLevels(runs, shape), + }; +} diff --git a/src/journey-set/flags.test.ts b/src/journey-set/flags.test.ts new file mode 100644 index 0000000..6bd963f --- /dev/null +++ b/src/journey-set/flags.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import openclawOraAi from "./__fixtures__/studio-trajectories/openclaw-f6f23ac8.json"; +import studioOraAi from "./__fixtures__/studio-trajectories/run-49ab0c88.json"; +import { deriveSetFlags } from "./flags"; +import blockedFirst403 from "./vendor/__fixtures__/run-flags/blocked-first-403.json"; +import cleanOnsite from "./vendor/__fixtures__/run-flags/clean-onsite.json"; +import memoryAnswer from "./vendor/__fixtures__/run-flags/memory-answer.json"; +import offsiteOnly from "./vendor/__fixtures__/run-flags/offsite-only.json"; + +// ora's own cases for the blocked rule run in ./vendor/run-flags.test.ts. This +// file covers what ax adds: the set's two flags from Studio's step list. +// +// Fixture provenance. +// vendor/__fixtures__/run-flags/*.json are ora's (see the vendored spec's header). +// studio-trajectories/*.json are recorded Studio trajectories from oramono at +// deb82e279, trimmed by script to each step's id, type, url_host and status: +// - run-49ab0c88: apps/studio-ui/src/lib/trajectory/__fixtures__/run-49ab0c88.trajectory.json +// (ora.ai, every on-site step without a status). +// - openclaw-f6f23ac8: apps/e2e/src/trace-validation/__fixtures__/openclaw-f6f23ac8.trajectory.json +// (ora.ai with 200s, a 404 on api.ora.ai, and a third party docs host). +// Variants below pass a different site, never edited steps. + +const NOT_FLAGGED = { flagged: false }; +const stepsOf = (fixture: { trajectory: { steps: unknown[] } }) => + fixture.trajectory.steps as Array<{ url_host?: string; status?: number }>; + +describe("blocked", () => { + it("passes ora's flag through from the steps alone", () => { + expect(deriveSetFlags(stepsOf(blockedFirst403), blockedFirst403.domain).blocked).toEqual({ + flagged: true, + reason: "The site refused the agent's first request with HTTP 403", + }); + expect(deriveSetFlags(stepsOf(memoryAnswer), memoryAnswer.domain).blocked).toEqual(NOT_FLAGGED); + }); + + it("treats a Studio step with a null status as no evidence", () => { + expect(deriveSetFlags(studioOraAi.steps, "ora.ai").blocked).toEqual(NOT_FLAGGED); + }); + + it("does not flag a Studio run whose only refusal-like status is a 404 on a subdomain", () => { + expect(deriveSetFlags(openclawOraAi.steps, "ora.ai").blocked).toEqual(NOT_FLAGGED); + }); +}); + +describe("never visited the site", () => { + it("is raised when no step reached the site, naming the site", () => { + expect(deriveSetFlags(stepsOf(offsiteOnly), offsiteOnly.domain).never_visited).toEqual({ + flagged: true, + reason: "The agent never opened a page on linear.com.", + }); + }); + + it("is not raised when any step reached the site or a subdomain", () => { + expect(deriveSetFlags(stepsOf(cleanOnsite), cleanOnsite.domain).never_visited).toEqual( + NOT_FLAGGED, + ); + expect(deriveSetFlags(stepsOf(blockedFirst403), "fiverr.com").never_visited).toEqual( + NOT_FLAGGED, + ); + expect(deriveSetFlags(studioOraAi.steps, "ora.ai").never_visited).toEqual(NOT_FLAGGED); + expect(deriveSetFlags(openclawOraAi.steps, "ora.ai").never_visited).toEqual(NOT_FLAGGED); + }); + + it("is raised on a real Studio run judged against a site it did not visit", () => { + // The ora.ai run, asked about stripe.com instead. + expect(deriveSetFlags(studioOraAi.steps, "stripe.com").never_visited).toEqual({ + flagged: true, + reason: "The agent never opened a page on stripe.com.", + }); + }); + + it("is raised for a run with no steps, which is never blocked", () => { + expect(deriveSetFlags([], "linear.app")).toEqual({ + blocked: NOT_FLAGGED, + never_visited: { flagged: true, reason: "The agent never opened a page on linear.app." }, + }); + }); +}); diff --git a/src/journey-set/flags.ts b/src/journey-set/flags.ts new file mode 100644 index 0000000..7956eca --- /dev/null +++ b/src/journey-set/flags.ts @@ -0,0 +1,24 @@ +import { deriveRunFlags, type RunFlag, reachedSite } from "./vendor/run-flags"; + +export { isSiteHost, type RunFlag } from "./vendor/run-flags"; + +/** A trajectory step as Studio returns it; only the host and status are read. */ +export type SetStep = { url_host?: string | null; status?: number | null }; + +export type SetFlags = { blocked: RunFlag; never_visited: RunFlag }; + +/** + * The two notes a set prints beside a run's verdict; they never change it. + * "Blocked" is ora's own rule (vendored), so it means what the free journey's + * flag means. ora's memory flag has no Studio source, so a set raises + * "never visited" in its place. + */ +export function deriveSetFlags(steps: readonly SetStep[], site: string): SetFlags { + const result = { trajectory: { steps } }; + return { + blocked: deriveRunFlags(result, site).blocked, + never_visited: reachedSite(result, site) + ? { flagged: false } + : { flagged: true, reason: `The agent never opened a page on ${site}.` }, + }; +} diff --git a/src/journey-set/follow.test.ts b/src/journey-set/follow.test.ts new file mode 100644 index 0000000..3bcd8eb --- /dev/null +++ b/src/journey-set/follow.test.ts @@ -0,0 +1,404 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import detailFinal from "../api/__fixtures__/studio/journey-detail-final.json"; +import detailRunning from "../api/__fixtures__/studio/journey-detail-running.json"; +import { StudioApiError } from "../api/studio"; +import type { JourneyAgentRun, JourneyDetail } from "../api/studio-types"; +import { + abortableSleep, + type FollowInput, + type FollowProgress, + followJourneys, + POLL_INTERVAL_MS, + READ_FAILURE_LIMIT, + SETTLE_WINDOW_MS, +} from "./follow"; +import { DEFAULT_WAIT_MINUTES } from "./spec"; + +// Journey reads are M2.2's recorded fixtures (src/api/__fixtures__/studio/journey-detail-*.json, +// provenance in src/api/studio.test.ts), unwrapped from their { success, data } envelope as that +// test does. Each case changes only the fields it needs, through the named deltas below. The +// insight fake answers false where Studio answers its insight_not_ready 404; that body also says +// terminal: true once the run has ended (runs-api run-detail-proxy-routes.ts:80), and studio.ts +// maps it to false all the same, so every not-ready case here is the terminal one. + +/** Run 0 running (started), run 1 queued (no startedAt). */ +const RUNNING: JourneyDetail = detailRunning.data; +/** Both runs succeeded, with verdicts succeeded and failed. */ +const FINAL: JourneyDetail = detailFinal.data; + +function withRun(detail: JourneyDetail, index: number, delta: Partial) { + const agents = detail.agents.map((run, i) => (i === index ? { ...run, ...delta } : run)); + return { ...detail, agents }; +} + +/** Another journey of the same set: the same read under new journey and run ids. */ +function asJourney(detail: JourneyDetail, n: number): JourneyDetail { + const agents = detail.agents.map((run) => ({ ...run, runId: `${run.runId}-${n}` })); + return { ...detail, id: `${detail.id}-${n}`, agents }; +} + +/** Graded, and the judge has not written its verdict yet. */ +const AWAITING = withRun(FINAL, 0, { verdict: "not_measured" }); +/** Run 0 graded and awaiting its verdict, run 1 still queued. */ +const MIXED = withRun(RUNNING, 0, { + status: "succeeded", + verdict: "not_measured", + finishedAt: FINAL.agents[0].finishedAt, +}); +/** Run 0 finished, run 1 started and running. */ +const SECOND_RUNNING = withRun(FINAL, 1, { + status: "running", + verdict: null, + finishedAt: null, +}); +/** Run 0 canceled; Studio writes not_measured on a cancel (see outcome.test.ts). */ +const CANCELED = withRun(FINAL, 0, { status: "canceled", verdict: "not_measured" }); +/** A create where no agent started. */ +const NOTHING_STARTED: JourneyDetail = { ...FINAL, agents: [] }; + +const ID = FINAL.id; +const RUN_0 = FINAL.agents[0].runId; + +type Step = T | Error; + +/** Scripted reads per journey id and per run id; the last step repeats once a script runs out. */ +function fakeStudio( + journeys: Record[]>, + insights: Record[]> = {}, +) { + const counts = { journey: new Map(), insight: new Map() }; + const next = (script: Step[] | undefined, count: Map, id: string): T => { + if (!script) throw new Error(`unscripted read of ${id}`); + const n = count.get(id) ?? 0; + count.set(id, n + 1); + const step = script[Math.min(n, script.length - 1)]; + if (step instanceof Error) throw step; + return step; + }; + return { + client: { + journey: async (id: string) => next(journeys[id], counts.journey, id), + insightLanded: async (runId: string) => next(insights[runId], counts.insight, runId), + }, + reads: (id: string) => counts.journey.get(id) ?? 0, + insightReads: (runId: string) => counts.insight.get(runId) ?? 0, + }; +} + +/** A clock that only moves when follow sleeps or a test advances it. */ +function fakeClock() { + let t = 0; + const sleeps: number[] = []; + return { + now: () => t, + advance: (ms: number) => { + t += ms; + }, + sleep: async (ms: number) => { + sleeps.push(ms); + t += ms; + }, + sleeps, + }; +} + +function follow( + client: FollowInput["client"], + journeyIds: readonly string[], + options: Partial & { clock?: ReturnType } = {}, +) { + const { clock = fakeClock(), ...rest } = options; + const progress: FollowProgress[] = []; + const done = followJourneys({ + client, + journeyIds, + waitMinutes: DEFAULT_WAIT_MINUTES, + now: clock.now, + sleep: clock.sleep, + onProgress: (p) => progress.push(p), + ...rest, + }); + return { done, clock, progress }; +} + +describe("followJourneys", () => { + it("pins its named constants", () => { + expect([POLL_INTERVAL_MS, SETTLE_WINDOW_MS, READ_FAILURE_LIMIT]).toEqual([10_000, 600_000, 3]); + }); + + it("settles after one round when every run is final and graded, without sleeping", async () => { + const studio = fakeStudio({ [ID]: [FINAL] }); + const { done, clock } = follow(studio.client, [ID]); + const result = await done; + expect(result.ended).toBe("settled"); + expect(result.journeys).toEqual(new Map([[ID, FINAL]])); + expect(result.pendingGrades).toBe(0); + expect(result).not.toHaveProperty("lastError"); + expect(clock.sleeps).toEqual([]); + expect(studio.insightReads(RUN_0)).toBe(0); + }); + + it("settles a journey with no agents on its first read, and no journeys at once", async () => { + const studio = fakeStudio({ [ID]: [NOTHING_STARTED] }); + expect((await follow(studio.client, [ID]).done).ended).toBe("settled"); + expect(studio.reads(ID)).toBe(1); + expect((await follow(studio.client, []).done).ended).toBe("settled"); + }); + + it("keeps polling a not_measured graded run while its insight is not ready", async () => { + const studio = fakeStudio( + { [ID]: [AWAITING, AWAITING, AWAITING, FINAL] }, + { [RUN_0]: [false] }, + ); + const { done, clock } = follow(studio.client, [ID]); + const result = await done; + expect(result.ended).toBe("settled"); + expect(result.journeys.get(ID)).toEqual(FINAL); + expect(studio.reads(ID)).toBe(4); + expect(studio.insightReads(RUN_0)).toBe(3); + expect(clock.sleeps).toEqual([POLL_INTERVAL_MS, POLL_INTERVAL_MS, POLL_INTERVAL_MS]); + }); + + it("reads the journey once more after the insight lands, then settles on not_measured", async () => { + const studio = fakeStudio({ [ID]: [AWAITING] }, { [RUN_0]: [false, true] }); + const result = await follow(studio.client, [ID]).done; + expect(result.ended).toBe("settled"); + expect(result.journeys.get(ID)?.agents[0].verdict).toBe("not_measured"); + expect(result.pendingGrades).toBe(0); + expect(studio.reads(ID)).toBe(3); + expect(studio.insightReads(RUN_0)).toBe(2); + }); + + it("asks for no insight until every run in the journey is final, and none after it lands", async () => { + const studio = fakeStudio({ [ID]: [MIXED, MIXED, AWAITING] }, { [RUN_0]: [true] }); + const result = await follow(studio.client, [ID]).done; + expect(result.ended).toBe("settled"); + expect(studio.reads(ID)).toBe(4); + expect(studio.insightReads(RUN_0)).toBe(1); + }); + + it("never asks for the insight of an excluded run, whatever its verdict", async () => { + const studio = fakeStudio({ [ID]: [CANCELED] }); + const result = await follow(studio.client, [ID]).done; + expect(result.ended).toBe("settled"); + expect(studio.insightReads(RUN_0)).toBe(0); + }); + + it("does not read a settled journey again", async () => { + const other = asJourney(RUNNING, 2); + const studio = fakeStudio({ [ID]: [FINAL], [other.id]: [other, other, asJourney(FINAL, 2)] }); + const result = await follow(studio.client, [ID, other.id]).done; + expect(result.ended).toBe("settled"); + expect(studio.reads(ID)).toBe(1); + expect(studio.reads(other.id)).toBe(3); + }); + + it("ends settle_limit when a grade is pending SETTLE_WINDOW_MS after the last run became final", async () => { + const studio = fakeStudio({ [ID]: [RUNNING, AWAITING] }, { [RUN_0]: [false] }); + const { done, clock } = follow(studio.client, [ID]); + const result = await done; + expect(result.ended).toBe("settle_limit"); + expect(result.pendingGrades).toBe(1); + expect(result.journeys.get(ID)).toEqual(AWAITING); + expect(clock.now()).toBe(POLL_INTERVAL_MS + SETTLE_WINDOW_MS); + }); + + it("ends wait_limit with runs in flight, never sleeping past the wait deadline", async () => { + const studio = fakeStudio({ [ID]: [RUNNING] }); + const clock = fakeClock(); + const slowRead = async (id: string) => { + clock.advance(3_000); + return studio.client.journey(id); + }; + const client = { ...studio.client, journey: slowRead }; + const result = await follow(client, [ID], { clock, waitMinutes: 1 }).done; + expect(result.ended).toBe("wait_limit"); + expect(result.pendingGrades).toBe(0); + expect(clock.sleeps).toEqual([10_000, 10_000, 10_000, 10_000, 5_000]); + expect(clock.now()).toBe(63_000); + }); + + it("ends settle_limit, not wait_limit, when the wait runs out with every run final", async () => { + const studio = fakeStudio( + { [ID]: [RUNNING, RUNNING, RUNNING, AWAITING] }, + { [RUN_0]: [false] }, + ); + const { done, clock } = follow(studio.client, [ID], { waitMinutes: 1 }); + const result = await done; + expect(result.ended).toBe("settle_limit"); + expect(result.pendingGrades).toBe(1); + expect(clock.now()).toBe(60_000); + }); +}); + +describe("followJourneys detaching", () => { + it("detaches when the signal aborts mid sleep, keeping the last reads", async () => { + const controller = new AbortController(); + const clock = fakeClock(); + const sleep = async (ms: number) => { + await clock.sleep(ms); + if (clock.sleeps.length === 2) controller.abort(); + }; + const studio = fakeStudio({ [ID]: [RUNNING, MIXED] }); + const options = { clock, sleep, signal: controller.signal }; + const result = await follow(studio.client, [ID], options).done; + expect(result.ended).toBe("detached"); + expect(result.journeys.get(ID)).toEqual(MIXED); + expect(result.pendingGrades).toBe(1); + expect(studio.reads(ID)).toBe(2); + }); + + it("detaches when an injected sleep rejects on abort, and rethrows any other rejection", async () => { + const controller = new AbortController(); + const studio = fakeStudio({ [ID]: [RUNNING] }); + const abortingSleep = async () => { + controller.abort(); + throw new Error("sleep aborted"); + }; + const options = { sleep: abortingSleep, signal: controller.signal }; + expect((await follow(studio.client, [ID], options).done).ended).toBe("detached"); + + const broken = new Error("timer broke"); + const brokenSleep = async () => { + throw broken; + }; + await expect(follow(studio.client, [ID], { sleep: brokenSleep }).done).rejects.toBe(broken); + }); + + it("detaches between reads without reading the next journey or reporting progress", async () => { + const controller = new AbortController(); + const other = asJourney(RUNNING, 2); + const studio = fakeStudio({ [ID]: [RUNNING], [other.id]: [other] }); + const abortingRead = async (id: string) => { + controller.abort(); + return studio.client.journey(id); + }; + const client = { ...studio.client, journey: abortingRead }; + const { done, progress } = follow(client, [ID, other.id], { signal: controller.signal }); + const result = await done; + expect(result.ended).toBe("detached"); + expect([...result.journeys.keys()]).toEqual([ID]); + expect(studio.reads(other.id)).toBe(0); + expect(progress).toEqual([]); + }); + + it("detaches before any read when the signal has already aborted", async () => { + const studio = fakeStudio({ [ID]: [RUNNING] }); + const result = await follow(studio.client, [ID], { signal: AbortSignal.abort() }).done; + expect(result.ended).toBe("detached"); + expect(result.journeys.size).toBe(0); + expect(studio.reads(ID)).toBe(0); + }); +}); + +describe("followJourneys read failures", () => { + const unavailable = new StudioApiError("ora request failed (503)", { + status: 503, + payload: null, + code: null, + missingScopes: [], + }); + + it("retries a failed round, keeping its successful reads, past two failures in a row", async () => { + const other = asJourney(RUNNING, 2); + const studio = fakeStudio({ + [ID]: [unavailable, unavailable, FINAL], + [other.id]: [other, asJourney(FINAL, 2)], + }); + const result = await follow(studio.client, [ID, other.id]).done; + expect(result.ended).toBe("settled"); + expect(studio.reads(ID)).toBe(3); + expect(studio.reads(other.id)).toBe(2); + expect([...result.journeys.keys()]).toEqual([ID, other.id]); + }); + + it("ends read_failed after READ_FAILURE_LIMIT failed rounds, with the error and the last good reads", async () => { + const studio = fakeStudio({ [ID]: [RUNNING, unavailable] }); + const result = await follow(studio.client, [ID]).done; + expect(result.ended).toBe("read_failed"); + expect(result.lastError).toBe(unavailable); + expect(result.journeys.get(ID)).toEqual(RUNNING); + expect(studio.reads(ID)).toBe(1 + READ_FAILURE_LIMIT); + }); + + it("resets the count after a round with no failure", async () => { + const studio = fakeStudio({ + [ID]: [unavailable, unavailable, RUNNING, unavailable, unavailable, FINAL], + }); + const result = await follow(studio.client, [ID]).done; + expect(result.ended).toBe("settled"); + expect(studio.reads(ID)).toBe(6); + }); + + it("lets an error that is not the platform's propagate instead of retrying it", async () => { + const bug = new RangeError("a bug in ax, not a failed read"); + const studio = fakeStudio({ [ID]: [bug] }); + await expect(follow(studio.client, [ID]).done).rejects.toBe(bug); + expect(studio.reads(ID)).toBe(1); + }); + + it("counts a failed insight read as a failed round", async () => { + const notFound = new StudioApiError("ora request failed (404 run_not_found)", { + status: 404, + payload: { error: "run_not_found" }, + code: "run_not_found", + missingScopes: [], + }); + const studio = fakeStudio({ [ID]: [AWAITING] }, { [RUN_0]: [notFound] }); + const result = await follow(studio.client, [ID]).done; + expect(result.ended).toBe("read_failed"); + expect(result.lastError).toBe(notFound); + expect(result.journeys.get(ID)).toEqual(AWAITING); + expect(result.pendingGrades).toBe(1); + expect(studio.insightReads(RUN_0)).toBe(READ_FAILURE_LIMIT); + }); +}); + +describe("followJourneys progress", () => { + it("counts queued, running and finished runs after each round, settled journeys included", async () => { + const other = asJourney(FINAL, 2); + const studio = fakeStudio({ [ID]: [RUNNING, SECOND_RUNNING, FINAL], [other.id]: [other] }); + const { done, progress } = follow(studio.client, [ID, other.id]); + expect((await done).ended).toBe("settled"); + expect(progress).toEqual([ + { queued: 1, running: 1, finished: 2, waitingForSlot: true }, + { queued: 0, running: 1, finished: 3, waitingForSlot: false }, + { queued: 0, running: 0, finished: 4, waitingForSlot: false }, + ]); + expect(studio.reads(other.id)).toBe(1); + }); +}); + +describe("abortableSleep", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("resolves after the given time", async () => { + vi.useFakeTimers(); + let woke = false; + const sleeping = abortableSleep(POLL_INTERVAL_MS).then(() => { + woke = true; + }); + await vi.advanceTimersByTimeAsync(POLL_INTERVAL_MS - 1); + expect(woke).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await sleeping; + expect(woke).toBe(true); + }); + + it("resolves as soon as the signal aborts and clears its timer", async () => { + vi.useFakeTimers(); + const controller = new AbortController(); + const sleeping = abortableSleep(POLL_INTERVAL_MS, controller.signal); + controller.abort(); + await sleeping; + expect(vi.getTimerCount()).toBe(0); + }); + + it("resolves at once on a signal that has already aborted", async () => { + vi.useFakeTimers(); + await abortableSleep(POLL_INTERVAL_MS, AbortSignal.abort()); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/src/journey-set/follow.ts b/src/journey-set/follow.ts new file mode 100644 index 0000000..0d997de --- /dev/null +++ b/src/journey-set/follow.ts @@ -0,0 +1,245 @@ +import { isStudioFailure } from "../api/studio"; +import type { JourneyAgentRun, JourneyDetail, StudioClient } from "../api/studio-types"; +import { awaitsGrade, runPhase } from "./outcome"; + +// Follows a set's journeys on Studio until every run is final and its grade has settled, or a +// limit is reached. It prints nothing: progress goes to a callback and the ending is returned. +// It never exchanges keys; the client refreshes its own bearer. + +/** Time between two rounds of reads. */ +export const POLL_INTERVAL_MS = 10_000; + +/** + * How long grades may stay pending after the last run became final. It is the only bound on + * waiting for an insight: the not-ready 404 says `terminal: true` as soon as the run has ended + * (runs-api run-detail-proxy-routes.ts:80), and grading happens after that, so the flag never + * means the insight will not come. + */ +export const SETTLE_WINDOW_MS = 600_000; + +/** Failed rounds in a row that end following. */ +export const READ_FAILURE_LIMIT = 3; + +const MINUTE_MS = 60_000; + +export type FollowEnd = "settled" | "settle_limit" | "wait_limit" | "detached" | "read_failed"; + +export interface FollowProgress { + /** In flight and not started yet. */ + queued: number; + /** In flight and started. */ + running: number; + /** Final: graded or excluded. */ + finished: number; + /** A run is queued, waiting for the plan's run slot. */ + waitingForSlot: boolean; +} + +export interface FollowInput { + client: Pick; + /** The record's journeys that have an id. */ + journeyIds: readonly string[]; + /** Counted from the start of following. */ + waitMinutes: number; + /** Aborting detaches; the runs keep going on Studio. */ + signal?: AbortSignal; + now: () => number; + /** May resolve early or reject when the signal aborts; follow reads the signal afterwards. */ + sleep: (ms: number, signal?: AbortSignal) => Promise; + /** Called after every round of reads. */ + onProgress?: (progress: FollowProgress) => void; +} + +export interface FollowResult { + /** + * The latest successful read of each journey, keyed by journey id in the order of + * `journeyIds`. A journey that never read successfully is absent. + */ + journeys: Map; + ended: FollowEnd; + /** Runs still awaiting a grade when following ended. */ + pendingGrades: number; + /** The error of the last failed read; set only when `ended` is `read_failed`. */ + lastError?: unknown; +} + +interface FollowState { + details: Map; + settled: Set; + /** Runs whose insight has landed, and whether their journey has been read since. */ + landed: Map; +} + +interface Round { + aborted: boolean; + failed: boolean; + error?: unknown; +} + +export async function followJourneys(input: FollowInput): Promise { + const state: FollowState = { details: new Map(), settled: new Set(), landed: new Map() }; + const waitDeadline = input.now() + input.waitMinutes * MINUTE_MS; + let failedRounds = 0; + let allFinalAt: number | undefined; + for (;;) { + const round = await readRound(input, state); + if (round.aborted) return result(input, state, "detached"); + input.onProgress?.(progressOf(state)); + const now = input.now(); + allFinalAt = everyRunFinal(input.journeyIds, state) ? (allFinalAt ?? now) : undefined; + failedRounds = round.failed ? failedRounds + 1 : 0; + const settleDeadline = + allFinalAt === undefined ? Number.POSITIVE_INFINITY : allFinalAt + SETTLE_WINDOW_MS; + // Settled wins over an abort that arrived during this round's reads: nothing is left running. + if (input.journeyIds.every((id) => state.settled.has(id))) { + return result(input, state, "settled"); + } + if (input.signal?.aborted) return result(input, state, "detached"); + if (failedRounds >= READ_FAILURE_LIMIT) { + return result(input, state, "read_failed", round.error); + } + if (now >= Math.min(settleDeadline, waitDeadline)) { + return result(input, state, allFinalAt === undefined ? "wait_limit" : "settle_limit"); + } + const ms = Math.min(POLL_INTERVAL_MS, settleDeadline - now, waitDeadline - now); + if (!(await pause(input, ms))) return result(input, state, "detached"); + } +} + +/** A real sleep for `followJourneys`: resolves after `ms`, or as soon as the signal aborts. */ +export function abortableSleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) { + resolve(); + return; + } + const finish = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", finish); + resolve(); + }; + const timer = setTimeout(finish, ms); + signal?.addEventListener("abort", finish, { once: true }); + }); +} + +/** One read of every journey not settled yet, in order, stopping at an abort. */ +async function readRound(input: FollowInput, state: FollowState): Promise { + const round: Round = { aborted: false, failed: false }; + for (const id of input.journeyIds) { + if (state.settled.has(id)) continue; + await readJourney(input, state, id, round); + if (round.aborted) break; + } + return round; +} + +async function readJourney( + input: FollowInput, + state: FollowState, + id: string, + round: Round, +): Promise { + if (input.signal?.aborted) { + round.aborted = true; + return; + } + let detail: JourneyDetail; + try { + detail = await input.client.journey(id); + } catch (error) { + // The platform's failures fail the round and are retried; READ_FAILURE_LIMIT bounds them. + if (!isStudioFailure(error)) throw error; + failRound(round, error); + return; + } + state.details.set(id, detail); + for (const run of detail.agents) { + if (state.landed.has(run.runId)) state.landed.set(run.runId, true); + } + if (!detail.agents.every(isFinal)) return; + for (const run of detail.agents) { + if (!awaitsGrade(run.status, run.verdict) || state.landed.has(run.runId)) continue; + if (input.signal?.aborted) { + round.aborted = true; + return; + } + await readInsight(input, state, run.runId, round); + } + if (detail.agents.every((run) => gradeSettled(run, state))) state.settled.add(id); +} + +/** A landed insight still needs one more read of its journey: the verdict is written after it. */ +async function readInsight( + input: FollowInput, + state: FollowState, + runId: string, + round: Round, +): Promise { + try { + if (await input.client.insightLanded(runId)) state.landed.set(runId, false); + } catch (error) { + // Same rule as a journey read: the round fails and is retried. + if (!isStudioFailure(error)) throw error; + failRound(round, error); + } +} + +/** Sleeps between rounds; false when the signal aborted during the sleep. */ +async function pause(input: FollowInput, ms: number): Promise { + try { + await input.sleep(ms, input.signal); + } catch (error) { + // An injected sleep may reject on abort; any other rejection is a real failure. + if (!input.signal?.aborted) throw error; + } + return !input.signal?.aborted; +} + +function failRound(round: Round, error: unknown): void { + round.failed = true; + round.error = error; +} + +function isFinal(run: JourneyAgentRun): boolean { + return runPhase(run.status) !== "in_flight"; +} + +/** No grade to wait for, or the insight landed and the journey was read again after it. */ +function gradeSettled(run: JourneyAgentRun, state: FollowState): boolean { + return !awaitsGrade(run.status, run.verdict) || state.landed.get(run.runId) === true; +} + +function everyRunFinal(journeyIds: readonly string[], state: FollowState): boolean { + return journeyIds.every((id) => state.details.get(id)?.agents.every(isFinal) === true); +} + +function allRuns(state: FollowState): JourneyAgentRun[] { + return [...state.details.values()].flatMap((detail) => detail.agents); +} + +function progressOf(state: FollowState): FollowProgress { + const progress = { queued: 0, running: 0, finished: 0 }; + for (const run of allRuns(state)) { + if (isFinal(run)) progress.finished++; + else if (run.startedAt) progress.running++; + else progress.queued++; + } + return { ...progress, waitingForSlot: progress.queued > 0 }; +} + +function result( + input: FollowInput, + state: FollowState, + ended: FollowEnd, + lastError?: unknown, +): FollowResult { + const journeys = new Map(); + for (const id of input.journeyIds) { + const detail = state.details.get(id); + if (detail) journeys.set(id, detail); + } + const pendingGrades = allRuns(state).filter((run) => !gradeSettled(run, state)).length; + const base = { journeys, ended, pendingGrades }; + return ended === "read_failed" ? { ...base, lastError } : base; +} diff --git a/src/journey-set/launch.test.ts b/src/journey-set/launch.test.ts new file mode 100644 index 0000000..73c1699 --- /dev/null +++ b/src/journey-set/launch.test.ts @@ -0,0 +1,384 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import quota402 from "../api/__fixtures__/studio/error-402-quota.json"; +import created from "../api/__fixtures__/studio/journey-create.json"; +import createdPartial from "../api/__fixtures__/studio/journey-create-partial.json"; +import { PlatformApiError } from "../api/platform"; +import { createStudioClient, StudioApiError, StudioConnectionError } from "../api/studio"; +import type { StudioAgentPair } from "../api/studio-types"; +import { type LaunchProgress, type LaunchResult, launchSet } from "./launch"; +import { createRecord, readRecord, type SetRecord } from "./record"; +import { type PlannedCell, parseSetSpec, planCells } from "./spec"; + +// Every case drives a real createStudioClient through a mocked fetch, so each error has the shape +// the client really throws. Fixtures: src/api/__fixtures__/studio/ (provenance in studio.test.ts). + +const BASE = "https://platform.test"; +const SITE = "linear.app"; +const AGENTS: StudioAgentPair[] = [ + { harness: "claude-code", model: "claude-haiku-4-5-20251001" }, + { harness: "claude-code", model: "claude-sonnet-5-5" }, +]; + +const asJson = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + +// Studio's own 502 on create: nothing started, the journey id kept, every agent in failed[]. +// Delta on journey-create-partial.json, as studio.test.ts builds it. +const NONE_STARTED_502 = { + ...createdPartial, + success: false, + data: { + ...createdPartial.data, + runIds: [], + streamUrls: [], + failed: [ + { ...createdPartial.data.failed[0], model: AGENTS[0].model }, + createdPartial.data.failed[0], + ], + }, +}; + +type Reply = () => Response | Promise; + +const ok: Reply = () => asJson(created, 201); +const partial: Reply = () => asJson(createdPartial, 201); +const studio502: Reply = () => asJson(NONE_STARTED_502, 502); +const gateway502: Reply = () => new Response("bad gateway", { status: 502 }); +const quota: Reply = () => asJson(quota402, 402); +const serverError: Reply = () => asJson({ error: "boom", code: "internal" }, 500); +const networkError: Reply = () => { + throw new TypeError("fetch failed"); +}; + +/** Answers the key exchange, then each create from `replies` in turn; a create past the script fails loudly. */ +function studioMock(replies: Reply[], exchange: Reply = () => asJson({ token: "bearer" })) { + const bodies: Array> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string | URL, init: RequestInit = {}) => { + const at = String(url); + if (at.endsWith("/auth/token")) return exchange(); + if (init.method !== "POST" || at !== `${BASE}/studio/v1/journeys`) { + throw new Error(`unrouted ${init.method} ${at}`); + } + bodies.push(JSON.parse(String(init.body))); + const reply = replies[bodies.length - 1]; + if (!reply) throw new Error(`create ${bodies.length} is past the script`); + return reply(); + }), + ); + return { bodies }; +} + +const client = () => createStudioClient({ apiKey: "ora_sk_local", baseUrl: BASE }); + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ax-set-launch-")); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + rmSync(dir, { recursive: true, force: true }); +}); + +/** Two goals that name the site, so the need level is skipped: two levels each (brand, exact). */ +function plan(repeat: number): { cells: PlannedCell[]; record: SetRecord } { + const parsed = parseSetSpec({ + site: SITE, + levels: true, + goals: ["Create an issue on linear.app", "Plan a sprint on linear.app"], + exact: "https://linear.app/developers", + repeat: String(repeat), + }); + if (!parsed.ok) throw new Error(parsed.error); + const { cells } = planCells(parsed.spec); + const record = createRecord({ spec: parsed.spec, agents: AGENTS, cells }, dir); + return { cells, record }; +} + +function launch( + record: SetRecord, + cells: PlannedCell[], + extra: { signal?: AbortSignal; onProgress?: (p: LaunchProgress) => void } = {}, +): Promise { + return launchSet({ + client: client(), + recordId: record.id, + site: SITE, + cells, + agents: AGENTS, + repeat: record.spec.repeat, + dir, + ...extra, + }); +} + +/** The record on disk says what the launch returned. */ +function expectRecordMatches(record: SetRecord, result: LaunchResult): SetRecord { + const onDisk = readRecord(record.id, dir); + if (!onDisk) throw new Error("record vanished"); + const started = onDisk.journeys.reduce((sum, journey) => sum + journey.runIds.length, 0); + expect(started).toBe(result.launchedRuns); + expect(onDisk.stoppedReason).toBe(result.stoppedReason); + expect(onDisk.notLaunched).toBe(result.stoppedReason ? result.notLaunched : undefined); + return onDisk; +} + +describe("launchSet", () => { + it("launches repeat first, then goal, then level, with every agent and only targetDomain", async () => { + const { cells, record } = plan(2); + expect(cells.map(({ goalIndex, level }) => `${goalIndex}:${level}`)).toEqual([ + "0:brand", + "0:exact", + "1:brand", + "1:exact", + ]); + const { bodies } = studioMock(Array(8).fill(ok)); + const progress: LaunchProgress[] = []; + const result = await launch(record, cells, { onProgress: (p) => progress.push(p) }); + + expect(result).toEqual({ launchedRuns: 16, notLaunched: 0 }); + const onDisk = expectRecordMatches(record, result); + expect( + onDisk.journeys.map(({ goalIndex, level, repeat }) => `${repeat}:${goalIndex}:${level}`), + ).toEqual([1, 2].flatMap((r) => cells.map((cell) => `${r}:${cell.goalIndex}:${cell.level}`))); + expect(bodies.map((body) => body.goal)).toEqual( + [1, 2].flatMap(() => cells.map((cell) => cell.prompt)), + ); + for (const body of bodies) { + expect(Object.keys(body).sort()).toEqual(["agents", "goal", "targetDomain"]); + expect(body.targetDomain).toBe(SITE); + expect(body.agents).toEqual(AGENTS); + expect(body).not.toHaveProperty("domains"); + } + expect(progress).toEqual(Array.from({ length: 8 }, (_, i) => ({ created: i + 1, planned: 8 }))); + }); + + it("stamps each entry with the clock when its create returned", async () => { + const { cells, record } = plan(1); + studioMock(Array(4).fill(ok)); + const times = [1, 2, 3, 4].map((n) => new Date(Date.UTC(2026, 10, 13, 12, 0, n))); + const result = await launchSet({ + client: client(), + recordId: record.id, + site: SITE, + cells, + agents: AGENTS, + repeat: 1, + dir, + now: () => times.shift() ?? new Date(0), + }); + const onDisk = expectRecordMatches(record, result); + expect(onDisk.journeys.map((journey) => journey.createdAt)).toEqual([ + "2026-11-13T12:00:01.000Z", + "2026-11-13T12:00:02.000Z", + "2026-11-13T12:00:03.000Z", + "2026-11-13T12:00:04.000Z", + ]); + }); + + it("records a partial create's failed agents raw and keeps launching", async () => { + const { cells, record } = plan(1); + studioMock([partial, ok, ok, ok]); + const result = await launch(record, cells); + + expect(result).toEqual({ launchedRuns: 7, notLaunched: 0 }); + const onDisk = expectRecordMatches(record, result); + expect(onDisk.journeys).toHaveLength(4); + expect(onDisk.journeys[0]).toMatchObject({ + journeyId: createdPartial.data.journeyId, + runIds: createdPartial.data.runIds, + failed: [ + { + harness: "claude-code", + model: "claude-sonnet-5-5", + reason: createdPartial.data.failed[0].reason, + }, + ], + }); + expect(onDisk.journeys[0].failed[0]).not.toHaveProperty("site"); + expect(onDisk.journeys[1].failed).toEqual([]); + }); + + it("records Studio's 502 with its journey id and carries on, resetting the count on success", async () => { + const { cells, record } = plan(2); + // Two 502s, a success, two more 502s, then successes: never three in a row. + studioMock([studio502, studio502, ok, studio502, studio502, ok, ok, ok]); + const result = await launch(record, cells); + + expect(result).toEqual({ launchedRuns: 8, notLaunched: 0 }); + const onDisk = expectRecordMatches(record, result); + expect(onDisk.journeys).toHaveLength(8); + expect(onDisk.journeys[0]).toMatchObject({ + journeyId: NONE_STARTED_502.data.journeyId, + runIds: [], + failed: NONE_STARTED_502.data.failed, + goalIndex: 0, + level: "brand", + repeat: 1, + }); + expect(onDisk.journeys.map((journey) => journey.runIds.length)).toEqual([ + 0, 0, 2, 0, 0, 2, 2, 2, + ]); + }); + + it("stops with platform_down after three Studio 502s in a row", async () => { + const { cells, record } = plan(2); + const { bodies } = studioMock([ok, studio502, studio502, studio502, ok]); + const result = await launch(record, cells); + + expect(bodies).toHaveLength(4); + // The third 502 is recorded with its failed agents, so it is not also counted as not launched. + expect(result).toMatchObject({ + launchedRuns: 2, + notLaunched: 4 * 2, + stoppedReason: "platform_down", + }); + expect(result.lastError).toBeInstanceOf(StudioApiError); + expect(expectRecordMatches(record, result).journeys).toHaveLength(4); + }); + + it("stops at once on a gateway 502 and records nothing for it", async () => { + const { cells, record } = plan(2); + const { bodies } = studioMock([ok, gateway502, ok]); + const result = await launch(record, cells); + + expect(bodies).toHaveLength(2); + expect(result).toMatchObject({ + launchedRuns: 2, + notLaunched: 7 * 2, + stoppedReason: "platform_down", + }); + expect(result.lastError).toMatchObject({ status: 502, gateway: true }); + expect(expectRecordMatches(record, result).journeys).toHaveLength(1); + }); + + it("stops with quota on a 402 at the third create and counts the runs not launched", async () => { + const { cells, record } = plan(2); + const { bodies } = studioMock([ok, ok, quota, ok]); + const result = await launch(record, cells); + + expect(bodies).toHaveLength(3); + expect(result).toMatchObject({ launchedRuns: 4, notLaunched: 6 * 2, stoppedReason: "quota" }); + expect(result.lastError).toMatchObject({ status: 402 }); + expect(expectRecordMatches(record, result).journeys).toHaveLength(2); + }); + + it("stops with api_error on any other Studio error", async () => { + const { cells, record } = plan(1); + studioMock([ok, serverError]); + const result = await launch(record, cells); + + expect(result).toMatchObject({ + launchedRuns: 2, + notLaunched: 3 * 2, + stoppedReason: "api_error", + }); + expect(result.lastError).toMatchObject({ status: 500 }); + expectRecordMatches(record, result); + }); + + it("stops with api_error when the key exchange fails mid launch", async () => { + const { cells, record } = plan(1); + studioMock([ok], () => asJson({ error: "revoked" }, 401)); + const result = await launch(record, cells); + + expect(result).toMatchObject({ + launchedRuns: 0, + notLaunched: 4 * 2, + stoppedReason: "api_error", + }); + expect(result.lastError).toBeInstanceOf(PlatformApiError); + expect(expectRecordMatches(record, result).journeys).toEqual([]); + }); + + it("stops with platform_down on a network failure and keeps the error", async () => { + const { cells, record } = plan(1); + studioMock([ok, networkError]); + const result = await launch(record, cells); + + expect(result).toMatchObject({ + launchedRuns: 2, + notLaunched: 3 * 2, + stoppedReason: "platform_down", + }); + expect(result.lastError).toBeInstanceOf(StudioConnectionError); + expectRecordMatches(record, result); + }); + + it("stops with interrupted before the next create when the signal is set, never aborting one in flight", async () => { + const { cells, record } = plan(2); + const controller = new AbortController(); + const second: Reply = () => { + controller.abort(); + return ok(); + }; + const { bodies } = studioMock([ok, second, ok]); + const result = await launch(record, cells, { signal: controller.signal }); + + // The create that was in flight when the signal fired is recorded in full. + expect(bodies).toHaveLength(2); + expect(result).toEqual({ + launchedRuns: 4, + notLaunched: 6 * 2, + stoppedReason: "interrupted", + }); + expect(expectRecordMatches(record, result).journeys).toHaveLength(2); + }); + + it("stops before the first create when the signal is already set", async () => { + const { cells, record } = plan(1); + const { bodies } = studioMock([ok]); + const result = await launch(record, cells, { signal: AbortSignal.abort() }); + + expect(bodies).toHaveLength(0); + expect(result).toEqual({ launchedRuns: 0, notLaunched: 4 * 2, stoppedReason: "interrupted" }); + expectRecordMatches(record, result); + }); + + it("propagates an error that is not the platform's instead of reading it as a stop", async () => { + const { cells, record } = plan(1); + const bug = new RangeError("a bug in ax, not a platform failure"); + await expect( + launchSet({ + client: { + createJourney: async () => { + throw bug; + }, + }, + recordId: record.id, + site: SITE, + cells, + agents: AGENTS, + repeat: 1, + dir, + }), + ).rejects.toBe(bug); + expect(readRecord(record.id, dir)?.stoppedReason).toBeUndefined(); + }); + + it("propagates a failure to write the record instead of reading it as a stop", async () => { + const { cells, record } = plan(1); + studioMock([ok]); + await expect( + launchSet({ + client: client(), + recordId: "j-0000000000", + site: SITE, + cells, + agents: AGENTS, + repeat: 1, + dir, + }), + ).rejects.toThrow(/no set record/); + expect(readRecord(record.id, dir)?.journeys).toEqual([]); + }); +}); diff --git a/src/journey-set/launch.ts b/src/journey-set/launch.ts new file mode 100644 index 0000000..82de795 --- /dev/null +++ b/src/journey-set/launch.ts @@ -0,0 +1,158 @@ +import { PlatformApiError } from "../api/platform"; +import { isStudioFailure, notStartedJourney, StudioApiError } from "../api/studio"; +import type { + CreatedJourney, + CreateFailure, + StudioAgentPair, + StudioClient, +} from "../api/studio-types"; +import { + appendJourney, + type FailedAgent, + type JourneyEntry, + markStopped, + type StoppedReason, +} from "./record"; +import type { PlannedCell } from "./spec"; + +// Launches a set's journeys against Studio, one create per cell and repeat, and records each one +// the moment it returns, so a crash or Ctrl-C leaves a record that `ax journey view` and +// `ax journey cancel` can pick up. It prints nothing: the progress callback reports state and a +// later module renders. +// Order is repeat first (every cell once, then every cell again), so a set cut short has the +// same cells at every level of help rather than only the first goals. There is no sleep between +// creates; Studio queues runs beyond the plan's slot count itself. + +/** Studio's own no-agent-started 502s in a row that stop launching: the platform is not starting runs. */ +export const STUDIO_502_LIMIT = 3; + +export interface LaunchInput { + client: Pick; + /** The record created before the first create; every entry and the stop are written to it. */ + recordId: string; + /** Sent as `targetDomain` on every create. */ + site: string; + /** The planned cells in goal, then ladder order (`planCells`). */ + cells: readonly PlannedCell[]; + /** The resolved agents; every create sends all of them. */ + agents: readonly StudioAgentPair[]; + repeat: number; + /** Checked before each create; a create in flight is never aborted. */ + signal?: AbortSignal; + /** Called after each journey is recorded; `planned` counts journeys (cells times repeats). */ + onProgress?: (progress: LaunchProgress) => void; + now?: () => Date; + /** The record directory; tests pass a temp one. */ + dir?: string; +} + +export interface LaunchProgress { + created: number; + planned: number; +} + +export interface LaunchResult { + /** Runs that started, summed over the recorded journeys. */ + launchedRuns: number; + /** Runs of the creates that were never recorded, `agents.length` each. */ + notLaunched: number; + stoppedReason?: StoppedReason; + /** The error that stopped launching; unset for a clean launch and for `interrupted`. */ + lastError?: unknown; +} + +/** A create's failed agents as the record keeps them: harness, model and the server's reason, raw. */ +function failedAgents(failed: readonly CreateFailure[]): FailedAgent[] { + return failed.map(({ harness, model, reason }) => ({ + harness, + ...(model !== undefined && { model }), + reason, + })); +} + +/** + * Why a create that failed on the platform stops launching. Studio's own 502 is handled before + * this (it is recorded, not a stop). A gateway 502, no connection, or an answer of the wrong shape + * leave it unknown whether the create landed, so they read as the platform being down; a 402 is + * the plan's quota; every other HTTP failure, a bearer refresh failing mid launch included, is an + * API error. + */ +function stopReasonOf(error: unknown): StoppedReason { + if (error instanceof StudioApiError && error.gateway) return "platform_down"; + if (error instanceof StudioApiError && error.status === 402) return "quota"; + if (error instanceof PlatformApiError) return "api_error"; + return "platform_down"; +} + +/** + * Run every create. Stops on the first of: an abort, a quota, an API error, a gateway or network + * failure, or three Studio 502s in a row. A stop is written to the record with the runs that never + * started; a clean launch writes nothing beyond its entries. A failure to write the record + * itself propagates: launching blind would leave journeys no one can find. + */ +export async function launchSet(input: LaunchInput): Promise { + const { client, recordId, site, cells, agents, repeat, signal, onProgress, dir } = input; + const now = input.now ?? (() => new Date()); + const planned = cells.length * repeat; + let recorded = 0; + let launchedRuns = 0; + let studio502sInARow = 0; + + const record = (entry: Omit): void => { + appendJourney(recordId, { ...entry, createdAt: now().toISOString() }, dir); + recorded++; + launchedRuns += entry.runIds.length; + onProgress?.({ created: recorded, planned }); + }; + + const stop = (stoppedReason: StoppedReason, lastError?: unknown): LaunchResult => { + const notLaunched = (planned - recorded) * agents.length; + markStopped(recordId, stoppedReason, notLaunched, dir); + return { + launchedRuns, + notLaunched, + stoppedReason, + ...(lastError !== undefined && { lastError }), + }; + }; + + for (let n = 0; n < planned; n++) { + if (signal?.aborted) return stop("interrupted"); + const cell = cells[n % cells.length]; + const place = { + goalIndex: cell.goalIndex, + level: cell.level, + repeat: Math.floor(n / cells.length) + 1, + }; + // Only the create sits in the try: a failure writing the record must propagate, not read as a stop. + let created: CreatedJourney; + try { + created = await client.createJourney({ + goal: cell.prompt, + agents: [...agents], + targetDomain: site, + }); + } catch (error) { + if (!isStudioFailure(error)) throw error; + const notStarted = notStartedJourney(error); + if (!notStarted) return stop(stopReasonOf(error), error); + record({ + journeyId: notStarted.journeyId, + ...place, + runIds: [], + failed: failedAgents(notStarted.failed), + }); + studio502sInARow++; + if (studio502sInARow >= STUDIO_502_LIMIT) return stop("platform_down", error); + continue; + } + studio502sInARow = 0; + record({ + journeyId: created.journeyId, + ...place, + runIds: created.runIds, + failed: failedAgents(created.failed), + }); + } + return { launchedRuns, notLaunched: 0 }; +} diff --git a/src/journey-set/levels.test.ts b/src/journey-set/levels.test.ts new file mode 100644 index 0000000..4f9ad50 --- /dev/null +++ b/src/journey-set/levels.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, it } from "vitest"; +import { + brandPrompt, + DOCS_SKIP_NOTE, + defaultBrand, + docsPrompt, + EXACT_SKIP_NOTE, + exactPrompt, + LADDER, + LEVEL_LABELS, + LEVELS, + namesSite, + needPrompt, + needSkipNote, + siteTermIn, + writtenPrompt, +} from "./levels"; + +describe("the ladder", () => { + it("runs from the least help to the most", () => { + expect(LADDER).toEqual(["need", "brand", "docs", "exact"]); + }); + + it("puts the written level, which runs without the ladder, before it in report order", () => { + expect(LEVELS).toEqual(["written", "need", "brand", "docs", "exact"]); + }); + + it("labels every level", () => { + expect(LEVELS.map((level) => LEVEL_LABELS[level])).toEqual([ + "As written", + "Need only", + "Brand named", + "Docs linked", + "Exact target", + ]); + }); +}); + +describe("defaultBrand", () => { + it("is the domain's name label with its first letter uppercased", () => { + expect(defaultBrand("linear.app")).toBe("Linear"); + expect(defaultBrand("stripe.com")).toBe("Stripe"); + expect(defaultBrand("x.ai")).toBe("X"); + }); + + it("skips a leading www and any subdomain", () => { + expect(defaultBrand("www.linear.app")).toBe("Linear"); + expect(defaultBrand("docs.stripe.com")).toBe("Stripe"); + expect(defaultBrand("api.eu.example.com")).toBe("Example"); + }); + + it("treats a generic second level label before a two letter country code as part of the suffix", () => { + expect(defaultBrand("bbc.co.uk")).toBe("Bbc"); + expect(defaultBrand("www.bbc.co.uk")).toBe("Bbc"); + expect(defaultBrand("shop.example.com.au")).toBe("Example"); + expect(defaultBrand("example.io")).toBe("Example"); + }); + + it("keeps a short brand under a two letter ending as the brand, not a suffix", () => { + expect(defaultBrand("docs.ora.ai")).toBe("Ora"); + expect(defaultBrand("api.x.ai")).toBe("X"); + expect(defaultBrand("app.ora.ai")).toBe("Ora"); + }); +}); + +describe("prompt templates", () => { + const GOAL = "Find how to create an issue from the API"; + + it("asks the need level with the goal as written", () => { + expect(needPrompt(GOAL)).toBe(GOAL); + expect(needPrompt(` ${GOAL} `)).toBe(GOAL); + }); + + it("sends the written level the way the need level is sent", () => { + expect(writtenPrompt(GOAL)).toBe(GOAL); + expect(writtenPrompt(` ${GOAL} `)).toBe(needPrompt(` ${GOAL} `)); + expect(writtenPrompt("Find linear.app's pricing?")).toBe("Find linear.app's pricing?"); + }); + + it("names the brand after the goal", () => { + expect(brandPrompt(GOAL, "Linear")).toBe( + "Find how to create an issue from the API. Use Linear.", + ); + }); + + it("names the brand and its documentation after the goal", () => { + expect(docsPrompt(GOAL, "Linear", "https://linear.app/docs")).toBe( + "Find how to create an issue from the API. Use Linear. Its documentation is at https://linear.app/docs.", + ); + }); + + it("gives the exact starting point after the goal", () => { + expect(exactPrompt(GOAL, "https://linear.app/developers")).toBe( + "Find how to create an issue from the API. Start at https://linear.app/developers.", + ); + }); + + it("adds a period only when the goal does not already end a sentence", () => { + expect(brandPrompt("Find the pricing page.", "Linear")).toBe( + "Find the pricing page. Use Linear.", + ); + expect(brandPrompt("Where is the pricing page?", "Linear")).toBe( + "Where is the pricing page? Use Linear.", + ); + expect(exactPrompt("Book a demo!", "https://linear.app/contact")).toBe( + "Book a demo! Start at https://linear.app/contact.", + ); + expect(docsPrompt("Find the rate limits", "Linear", "https://linear.app/docs")).toBe( + "Find the rate limits. Use Linear. Its documentation is at https://linear.app/docs.", + ); + }); + + it("trims the goal before deciding on the period", () => { + expect(brandPrompt(" Find the pricing page. ", "Linear")).toBe( + "Find the pricing page. Use Linear.", + ); + expect(brandPrompt("Find the pricing page ", "Linear")).toBe( + "Find the pricing page. Use Linear.", + ); + }); +}); + +describe("namesSite (the need-level leak check)", () => { + it("is true when the goal contains the site's domain", () => { + expect(namesSite("Open linear.app and find the pricing", "linear.app", "Acme")).toBe(true); + expect(namesSite("See https://linear.app/docs for the API", "linear.app", "Acme")).toBe(true); + }); + + it("is true when the goal names the brand as a whole word", () => { + expect(namesSite("Find the Linear API docs", "linear.app", "Linear")).toBe(true); + expect(namesSite("Linear's pricing for teams", "linear.app", "Linear")).toBe(true); + expect(namesSite("Compare plans (Linear)", "linear.app", "Linear")).toBe(true); + }); + + it("is false when the brand only appears inside another word", () => { + expect(namesSite("Explain nonlinear regression", "linear.app", "Linear")).toBe(false); + expect(namesSite("Grow linearly with the team", "linear.app", "Linear")).toBe(false); + expect(namesSite("Use a linear_model in sklearn", "linear.app", "Linear")).toBe(false); + }); + + it("ignores case on both the domain and the brand", () => { + expect(namesSite("find the LINEAR api", "linear.app", "Linear")).toBe(true); + expect(namesSite("Open LINEAR.APP", "linear.app", "Acme")).toBe(true); + expect(namesSite("find the Linear api", "linear.app", "linear")).toBe(true); + }); + + it("matches a fullwidth spelling of the brand", () => { + expect(namesSite("Use ＬＩＮＥＡＲ for this", "linear.app", "Linear")).toBe(true); + }); + + it("matches a brand whose ends are not letters", () => { + expect(namesSite("Write C++ bindings for the API", "isocpp.org", "C++")).toBe(true); + expect(namesSite("Write C bindings for the API", "isocpp.org", "C++")).toBe(false); + }); + + it("is false when the goal names neither", () => { + expect(namesSite("Find how to create an issue from the API", "linear.app", "Linear")).toBe( + false, + ); + }); +}); + +describe("siteTermIn", () => { + it("names the brand when the brand matched, even when the domain did too", () => { + expect(siteTermIn("Find the Linear API docs", "linear.app", "Linear")).toBe("Linear"); + expect(siteTermIn("Open linear.app", "linear.app", "Linear")).toBe("Linear"); + }); + + it("names the domain when only the domain matched", () => { + expect(siteTermIn("Open linear.app", "linear.app", "Acme")).toBe("linear.app"); + }); + + it("keeps the brand as configured, not as the goal spelled it", () => { + expect(siteTermIn("find the LINEAR api", "linear.app", "Linear")).toBe("Linear"); + }); + + it("is null when nothing in the goal names the site", () => { + expect(siteTermIn("Find a project tracker", "linear.app", "Linear")).toBeNull(); + }); +}); + +describe("skip notes", () => { + it("says why the need level is skipped, naming what matched", () => { + expect(needSkipNote("Linear")).toBe( + "The goal names Linear, so the need level is skipped. Write the goal without it to include that level.", + ); + expect(needSkipNote("linear.app")).toBe( + "The goal names linear.app, so the need level is skipped. Write the goal without it to include that level.", + ); + }); + + it("says how to include the docs level", () => { + expect(DOCS_SKIP_NOTE).toBe( + "No --docs URL, so the docs level is skipped. Add --docs to include it.", + ); + }); + + it("says how to include the exact level", () => { + expect(EXACT_SKIP_NOTE).toBe( + "No --exact URL, so the exact level is skipped. Add --exact to include it.", + ); + }); +}); diff --git a/src/journey-set/levels.ts b/src/journey-set/levels.ts new file mode 100644 index 0000000..eda8636 --- /dev/null +++ b/src/journey-set/levels.ts @@ -0,0 +1,129 @@ +// The levels a set of journeys runs a goal at: how much help the prompt gives the agent. With +// levels on, each goal climbs the ladder; with levels off, a set runs each goal once, as written. +// Every prompt comes from a fixed template here, never from a model, so whether the need level +// leaks the site is decided in code (namesSite). + +/** The ladder that levels climb. */ +export type LadderLevel = "need" | "brand" | "docs" | "exact"; + +/** A ladder level, or `written`: the goal as typed, the only level of a set without levels. */ +export type Level = "written" | LadderLevel; + +/** Least help first. Prompts, launch order, report rows and gap adjacency all follow this order. */ +export const LADDER: readonly LadderLevel[] = ["need", "brand", "docs", "exact"]; + +/** Every level in report order. `written` never shares a set with a ladder level. */ +export const LEVELS: readonly Level[] = ["written", ...LADDER]; + +export const LEVEL_LABELS: Readonly> = { + written: "As written", + need: "Need only", + brand: "Brand named", + docs: "Docs linked", + exact: "Exact target", +}; + +/** Second level labels that country registries sell under (`co.uk`, `com.au`, `ne.jp`). */ +const GENERIC_SECOND_LEVEL: ReadonlySet = new Set([ + "ac", + "co", + "com", + "edu", + "go", + "gob", + "gov", + "ltd", + "mil", + "ne", + "net", + "or", + "org", + "plc", +]); + +/** + * The domain's name label with its first letter uppercased: `linear.app`, `www.linear.app` and + * `docs.linear.app` all give `Linear`. A generic second level label before a two letter country + * code counts as part of the suffix (`bbc.co.uk` gives `Bbc`), while a brand there stays the + * brand (`docs.ora.ai` gives `Ora`). A guess, not a lookup: the preview names the brand it used + * and `--brand` overrides it. + */ +export function defaultBrand(site: string): string { + const labels = site.split(".").filter(Boolean); + const last = labels.length - 1; + const countrySuffix = + labels.length >= 3 && labels[last].length === 2 && GENERIC_SECOND_LEVEL.has(labels[last - 1]); + const label = labels[Math.max(0, last - (countrySuffix ? 2 : 1))] ?? ""; + return label.charAt(0).toUpperCase() + label.slice(1); +} + +/** `{goal.}`: the trimmed goal, with a period unless it already ends in `.`, `?` or `!`. */ +function sentence(goal: string): string { + const trimmed = goal.trim(); + return /[.?!]$/.test(trimmed) ? trimmed : `${trimmed}.`; +} + +export function needPrompt(goal: string): string { + return goal.trim(); +} + +/** A set without levels sends the goal the way the need level does: trimmed, nothing added. */ +export function writtenPrompt(goal: string): string { + return needPrompt(goal); +} + +export function brandPrompt(goal: string, brand: string): string { + return `${sentence(goal)} Use ${brand}.`; +} + +export function docsPrompt(goal: string, brand: string, docs: string): string { + return `${sentence(goal)} Use ${brand}. Its documentation is at ${docs}.`; +} + +export function exactPrompt(goal: string, exact: string): string { + return `${sentence(goal)} Start at ${exact}.`; +} + +/** Compared form: NFKC and lowercase, so a fullwidth or mixed-case spelling still matches. */ +function fold(value: string): string { + return value.normalize("NFKC").toLowerCase(); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** True when `word` appears in `text` with no letter, digit or underscore touching either end. */ +function hasWholeWord(text: string, word: string): boolean { + if (!word) return false; + const edge = "[\\p{L}\\p{N}_]"; + return new RegExp(`(? to include it."; + +export const EXACT_SKIP_NOTE = + "No --exact URL, so the exact level is skipped. Add --exact to include it."; diff --git a/src/journey-set/outcome.test.ts b/src/journey-set/outcome.test.ts new file mode 100644 index 0000000..7bdee9d --- /dev/null +++ b/src/journey-set/outcome.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from "vitest"; +import { awaitsGrade, EXCLUDED_STATUSES, GRADED_STATUSES, runBucket, runPhase } from "./outcome"; + +// Provenance, oramono develop deb82e279 (2026-11-13): the two status sets are Studio's own +// terminal sets, RUN_SUCCESS_STATUSES and RUN_FAILURE_STATUSES in +// packages/node/journey-stages/src/lifecycle-bucket.ts:7-8, and the verdicts are its RUN_VERDICTS +// (packages/node/journey-stages/src/run-verdict.ts:27). Studio's verdict on an excluded status is +// `failed`, or `not_measured` for a cancel (run-verdict.ts:45-49), so each excluded case below +// carries that verdict to show the status decides first. A create's failed[] reason is the start +// error's message (apps/studio-service/src/journeys/journey-fanout.ts:183); FAILED_TO_START is the +// one in apps/studio-service/src/journeys/journeys-service.test.ts:629. + +const FAILED_TO_START = "run gate unavailable"; + +const STUDIO_VERDICT_ON_EXCLUDED: Array<[string, string]> = [ + ["failed", "failed"], + ["timeout", "failed"], + ["timed_out", "failed"], + ["error", "failed"], + ["canceled", "not_measured"], + ["cancelled", "not_measured"], + ["lost", "failed"], +]; + +const IN_FLIGHT = ["queued", "pending", "running", "warming_up"]; + +describe("status sets", () => { + it("pins Studio's graded and excluded statuses, with no status in both", () => { + expect(GRADED_STATUSES).toEqual(["succeeded", "completed"]); + expect(EXCLUDED_STATUSES).toEqual([ + "failed", + "timeout", + "timed_out", + "error", + "canceled", + "cancelled", + "lost", + ]); + expect(GRADED_STATUSES.filter((s) => EXCLUDED_STATUSES.includes(s))).toEqual([]); + }); +}); + +describe("runPhase", () => { + it.each(["succeeded", "completed"])("%s is graded", (status) => { + expect(runPhase(status)).toBe("graded"); + }); + + it.each(STUDIO_VERDICT_ON_EXCLUDED.map(([status]) => status))("%s is excluded", (status) => { + expect(runPhase(status)).toBe("excluded"); + }); + + it.each(IN_FLIGHT)("%s is in flight", (status) => { + expect(runPhase(status)).toBe("in_flight"); + }); +}); + +describe("awaitsGrade", () => { + it.each([ + ["not_measured", "not_measured"], + ["null", null], + ["absent", undefined], + ])("a graded status with a %s verdict is waiting for its grade", (_, verdict) => { + expect(awaitsGrade("succeeded", verdict)).toBe(true); + expect(awaitsGrade("completed", verdict)).toBe(true); + }); + + it.each(["succeeded", "partial", "failed", "abandoned"])( + "a graded status with a %s verdict is not waiting", + (verdict) => { + expect(awaitsGrade("succeeded", verdict)).toBe(false); + }, + ); + + it("an excluded or in flight run is never waiting for a grade", () => { + expect(awaitsGrade("canceled", "not_measured")).toBe(false); + expect(awaitsGrade("lost", null)).toBe(false); + expect(awaitsGrade("running", null)).toBe(false); + }); +}); + +describe("runBucket", () => { + it("a run that never started is excluded with its reason", () => { + expect(runBucket({ started: false, reason: FAILED_TO_START })).toEqual({ + bucket: "excluded", + reason: FAILED_TO_START, + }); + }); + + it.each(STUDIO_VERDICT_ON_EXCLUDED)( + "a %s run is excluded with its raw status, whatever its verdict (%s)", + (status, verdict) => { + expect(runBucket({ started: true, status, verdict })).toEqual({ + bucket: "excluded", + rawStatus: status, + }); + }, + ); + + it.each([ + ["succeeded", "satisfied"], + ["partial", "partial"], + ["failed", "unsatisfied"], + ["not_measured", "not_graded_yet"], + ])("on a graded status, a %s verdict is %s", (verdict, bucket) => { + expect(runBucket({ started: true, status: "succeeded", verdict })).toEqual({ bucket }); + expect(runBucket({ started: true, status: "completed", verdict })).toEqual({ bucket }); + }); + + it("a null or absent verdict on a graded status is not graded yet", () => { + expect(runBucket({ started: true, status: "succeeded", verdict: null })).toEqual({ + bucket: "not_graded_yet", + }); + expect(runBucket({ started: true, status: "completed" })).toEqual({ + bucket: "not_graded_yet", + }); + }); + + it.each(["abandoned", "", "constructor"])( + "an unknown verdict %j on a graded status is not gradable with its raw value", + (verdict) => { + expect(runBucket({ started: true, status: "succeeded", verdict })).toEqual({ + bucket: "not_gradable", + rawVerdict: verdict, + }); + }, + ); + + it.each(IN_FLIGHT)("a %s run is in flight", (status) => { + expect(runBucket({ started: true, status, verdict: null })).toEqual({ bucket: "in_flight" }); + }); +}); diff --git a/src/journey-set/outcome.ts b/src/journey-set/outcome.ts new file mode 100644 index 0000000..e682926 --- /dev/null +++ b/src/journey-set/outcome.ts @@ -0,0 +1,79 @@ +// How one run of a set counts. Studio owns every grade: this maps the status it reports and the +// verdict it wrote into a bucket, status first, and judges nothing itself. It is the only place a +// run's bucket is derived. + +/** Final statuses whose verdict counts. */ +export const GRADED_STATUSES: readonly string[] = ["succeeded", "completed"]; + +/** + * Final statuses that never count against the site: the platform's fault (a harness crash is + * stored as failed, the stuck run reaper writes lost) or the caller's choice (a cancel). Studio's + * own verdict calls most of them failed; a set excludes them instead. + */ +export const EXCLUDED_STATUSES: readonly string[] = [ + "failed", + "timeout", + "timed_out", + "error", + "canceled", + "cancelled", + "lost", +]; + +export type RunPhase = "in_flight" | "graded" | "excluded"; + +export type Bucket = + | "satisfied" + | "partial" + | "unsatisfied" + | "not_graded_yet" + | "not_gradable" + | "excluded" + | "in_flight"; + +/** A run that never started (a create's failed[] entry or a 502), or its latest status and verdict. */ +export type RunFacts = + | { started: false; reason: string } + | { started: true; status: string; verdict?: string | null }; + +export interface RunOutcome { + bucket: Bucket; + /** The final status of an excluded run that started. */ + rawStatus?: string; + /** The verdict as Studio wrote it, on a not gradable run. */ + rawVerdict?: string; + /** Why an excluded run never started. */ + reason?: string; +} + +const NOT_MEASURED = "not_measured"; + +// A Map, not an object literal: the verdict is a server string, and "constructor" must not resolve. +const VERDICT_BUCKETS = new Map([ + ["succeeded", "satisfied"], + ["partial", "partial"], + ["failed", "unsatisfied"], + [NOT_MEASURED, "not_graded_yet"], +]); + +/** Any status outside both sets is still in flight. */ +export function runPhase(status: string): RunPhase { + if (GRADED_STATUSES.includes(status)) return "graded"; + if (EXCLUDED_STATUSES.includes(status)) return "excluded"; + return "in_flight"; +} + +/** A graded run whose verdict the judge has not written yet. */ +export function awaitsGrade(status: string, verdict: string | null | undefined): boolean { + return runPhase(status) === "graded" && (verdict == null || verdict === NOT_MEASURED); +} + +export function runBucket(run: RunFacts): RunOutcome { + if (!run.started) return { bucket: "excluded", reason: run.reason }; + const phase = runPhase(run.status); + if (phase === "excluded") return { bucket: "excluded", rawStatus: run.status }; + if (phase === "in_flight") return { bucket: "in_flight" }; + if (run.verdict == null) return { bucket: "not_graded_yet" }; + const bucket = VERDICT_BUCKETS.get(run.verdict); + return bucket ? { bucket } : { bucket: "not_gradable", rawVerdict: run.verdict }; +} diff --git a/src/journey-set/preview.test.ts b/src/journey-set/preview.test.ts new file mode 100644 index 0000000..c6d1851 --- /dev/null +++ b/src/journey-set/preview.test.ts @@ -0,0 +1,381 @@ +import { describe, expect, it } from "vitest"; +import agentsFixture from "../api/__fixtures__/studio/agents.json"; +import usageFixture from "../api/__fixtures__/studio/usage.json"; +import usageUnlimitedFixture from "../api/__fixtures__/studio/usage-unlimited.json"; +import { STUDIO_SCOPES, type StudioAgent, type StudioUsage } from "../api/studio-types"; +import { type PreviewInput, type PreviewResult, previewSet, type Refusal } from "./preview"; +import { MAX_RUNS, parseSetSpec, planCells, type SetSpecInput } from "./spec"; + +// The recorded Studio bodies, unwrapped from their `{ success, data }` envelope. +const catalog: StudioAgent[] = agentsFixture.data.agents; +const usage: StudioUsage = usageFixture.data; +const usageUnlimited: StudioUsage = usageUnlimitedFixture.data; + +// Named deltas from the recorded catalog and usage. +const claudeCodeUnavailable = catalog.map((entry) => + entry.harness === "claude-code" + ? { ...entry, available: false, reason: "needs the Pro plan" } + : entry, +); +const haikuUnavailable = catalog.map((entry) => + entry.model === "claude-haiku-4-5-20251001" + ? { ...entry, available: false, reason: "needs the Standard plan" } + : entry, +); +const everyAgentAvailable = catalog.map((entry) => ({ ...entry, available: true, reason: null })); +const nothingAvailable = catalog.map((entry) => ({ + ...entry, + available: false, + reason: "needs the Standard plan", +})); +const withRemaining = (remaining: number): StudioUsage => ({ + ...usage, + runs: { ...usage.runs, remaining }, +}); + +const ALL_SCOPES = [...STUDIO_SCOPES]; +const HAIKU = "claude-code/claude-haiku-4-5-20251001"; +const SONNET = "claude-code/claude-sonnet-5-5"; + +const GOAL = "Find how to create an issue from the API"; + +/** A spec and its cells from the real parser and planner. */ +function planned(overrides: Partial = {}, extraTasks = 0) { + const parsed = parseSetSpec({ + site: "linear.app", + levels: true, + goals: [GOAL, ...Array.from({ length: extraTasks }, (_, i) => `Goal number ${i + 2}`)], + ...overrides, + }); + if (!parsed.ok) throw new Error(parsed.error); + return { spec: parsed.spec, cells: planCells(parsed.spec).cells }; +} + +function preview( + overrides: Partial = {}, + rest: Partial = {}, + extraTasks = 0, +): PreviewResult { + const { spec, cells } = planned(overrides, extraTasks); + return previewSet({ spec, cells, scopes: ALL_SCOPES, agents: catalog, usage, ...rest }); +} + +function refusalsOf(result: PreviewResult): Refusal[] { + if (result.ok) throw new Error("expected a refusal"); + return result.refusals; +} + +function kindsOf(result: PreviewResult): string[] { + return refusalsOf(result).map((refusal) => refusal.kind); +} + +describe("previewSet go", () => { + it("uses the plan's first available Claude Code agent when --agents was omitted", () => { + const result = preview(); + expect(result).toEqual({ + ok: true, + agents: [ + { + harness: "claude-code", + model: "claude-haiku-4-5-20251001", + label: HAIKU, + harnessName: "Claude Code", + modelName: "Claude Haiku 4.5", + }, + ], + defaultedAgent: true, + totals: { runs: 2, journeys: 2, runsPerRepeat: 2 }, + allowance: { + kind: "limited", + limit: 50, + remaining: 38, + resetsAt: "2026-12-01T00:00:00.000Z", + }, + }); + }); + + it("prefers the Claude Code agent over an available agent listed before it", () => { + const offered = catalog.map((agent) => ({ ...agent, available: true })); + expect(offered[0].harness).toBe("codex"); + const result = preview({}, { agents: offered }); + if (!result.ok) throw new Error("expected go"); + expect(result.defaultedAgent).toBe(true); + expect(result.agents.map((agent) => agent.label)).toEqual([HAIKU]); + }); + + it("falls back to the first available agent when the plan offers no Claude Code agent", () => { + const offered = catalog + .filter((agent) => agent.harness !== "claude-code") + .map((agent) => ({ ...agent, available: true })); + const result = preview({}, { agents: offered }); + if (!result.ok) throw new Error("expected go"); + expect(result.defaultedAgent).toBe(true); + expect(result.agents.map((agent) => agent.harness)).toEqual(["codex"]); + }); + + it("resolves explicit pairs to the catalog entries and labels them harness/model", () => { + const result = preview({ agents: `${HAIKU},${SONNET}` }); + if (!result.ok) throw new Error("expected go"); + expect(result.defaultedAgent).toBe(false); + expect(result.agents.map((agent) => agent.label)).toEqual([HAIKU, SONNET]); + expect(result.agents[1]).toMatchObject({ + harness: "claude-code", + model: "claude-sonnet-5-5", + modelName: "Claude Sonnet 5.5", + }); + expect(result.totals.runsPerRepeat).toBe(4); + }); + + it("resolves a bare harness to its first available entry in catalog order", () => { + const result = preview({ agents: "claude-code" }); + if (!result.ok) throw new Error("expected go"); + expect(result.agents.map((agent) => agent.label)).toEqual([HAIKU]); + }); + + it("skips an unavailable entry when resolving a bare harness", () => { + const result = preview({ agents: "claude-code" }, { agents: haikuUnavailable }); + if (!result.ok) throw new Error("expected go"); + expect(result.agents.map((agent) => agent.label)).toEqual([SONNET]); + }); + + it("totals runs, journeys and runs per repeat with the resolved agent count", () => { + const result = preview({ + agents: `${HAIKU},${SONNET}`, + repeat: "3", + docs: "https://linear.app/docs", + }); + if (!result.ok) throw new Error("expected go"); + // 3 cells (need, brand, docs) x 2 agents x 3 repeats. + expect(result.totals).toEqual({ runs: 18, journeys: 9, runsPerRepeat: 6 }); + }); + + it("counts the default agent once however many repeats", () => { + const result = preview({ repeat: "4" }); + if (!result.ok) throw new Error("expected go"); + expect(result.totals).toEqual({ runs: 8, journeys: 8, runsPerRepeat: 2 }); + }); +}); + +describe("previewSet allowance", () => { + it("returns a no-limit marker with its reset time when the plan sets no limit", () => { + const result = preview({}, { usage: usageUnlimited }); + if (!result.ok) throw new Error("expected go"); + expect(result.allowance).toEqual({ kind: "unlimited", resetsAt: null }); + }); + + it("skips the allowance check when remaining is null, while the cap still applies", () => { + const allowed = preview({ repeat: "10" }, { usage: usageUnlimited }, 4); + expect(allowed.ok).toBe(true); + const over = preview( + { + repeat: "10", + agents: `${HAIKU},${SONNET}`, + docs: "https://a.dev", + exact: "https://a.dev/x", + }, + { usage: usageUnlimited }, + 4, + ); + expect(kindsOf(over)).toEqual(["over_cap"]); + }); + + it("refuses with what fits when a smaller repeat count fits", () => { + // 4 cells x 5 goals = 20 cells, 1 agent, 3 repeats = 60 runs; 38 remain. + const result = preview( + { repeat: "3", docs: "https://linear.app/docs", exact: "https://linear.app/developers" }, + {}, + 4, + ); + expect(refusalsOf(result)).toEqual([ + { + kind: "over_allowance", + runs: 60, + remaining: 38, + resetsAt: "2026-12-01T00:00:00.000Z", + fits: { repeat: 1, runs: 20 }, + // Levels are on, so --repeat 1 is still a set. + oneJourney: false, + }, + ]); + }); + + it("picks the largest repeat count that fits", () => { + // 2 cells x 1 agent = 2 per repeat; 10 repeats = 20 runs; 7 remain, so 3 repeats = 6 runs. + const result = preview({ repeat: "10" }, { usage: withRemaining(7) }); + expect(refusalsOf(result)).toMatchObject([ + { kind: "over_allowance", runs: 20, remaining: 7, fits: { repeat: 3, runs: 6 } }, + ]); + }); + + it("says not even one repeat fits with a null fit", () => { + const result = preview({ repeat: "2" }, { usage: withRemaining(1) }); + expect(refusalsOf(result)).toMatchObject([ + { kind: "over_allowance", runs: 4, remaining: 1, fits: null }, + ]); + }); + + it("allows a set that uses exactly what remains", () => { + expect(preview({ repeat: "2" }, { usage: withRemaining(4) }).ok).toBe(true); + }); +}); + +describe("previewSet run cap", () => { + // 5 goals x 4 levels = 20 cells. + const full = { docs: "https://linear.app/docs", exact: "https://linear.app/developers" }; + + it("allows exactly MAX_RUNS", () => { + const result = preview({ ...full, repeat: "10" }, { usage: usageUnlimited }, 4); + if (!result.ok) throw new Error("expected go"); + expect(result.totals.runs).toBe(MAX_RUNS); + }); + + it("refuses one past MAX_RUNS even with no allowance limit", () => { + // 17 cells x 3 agents x 4 repeats = 204. + const { spec, cells } = planned( + { ...full, repeat: "4", agents: `${HAIKU},${SONNET},codex/gpt-5.6-luna` }, + 4, + ); + const result = previewSet({ + spec, + cells: cells.slice(0, 17), + scopes: ALL_SCOPES, + agents: everyAgentAvailable, + usage: usageUnlimited, + }); + expect(refusalsOf(result)).toEqual([{ kind: "over_cap", runs: 204, maxRuns: MAX_RUNS }]); + }); + + it("reports the cap and the allowance together", () => { + const { spec, cells } = planned( + { ...full, repeat: "4", agents: `${HAIKU},${SONNET},codex/gpt-5.6-luna` }, + 4, + ); + const result = previewSet({ + spec, + cells: cells.slice(0, 17), + scopes: ALL_SCOPES, + agents: everyAgentAvailable, + usage, + }); + expect(kindsOf(result)).toEqual(["over_cap", "over_allowance"]); + }); + + it("never suggests a repeat count that would itself exceed the cap", () => { + // 20 cells x 3 agents = 60 per repeat; 10 repeats = 600 runs; 550 remain. 9 repeats (540) + // fit the allowance but not the cap, so the largest suggestion is 3 repeats (180). + const result = previewSet({ + ...planned({ ...full, repeat: "10", agents: `${HAIKU},${SONNET},codex/gpt-5.6-luna` }, 4), + scopes: ALL_SCOPES, + agents: everyAgentAvailable, + usage: withRemaining(550), + }); + expect(refusalsOf(result)).toMatchObject([ + { kind: "over_cap", runs: 600 }, + { kind: "over_allowance", fits: { repeat: 3, runs: 180 } }, + ]); + }); +}); + +describe("previewSet scopes", () => { + it("names every missing scope in STUDIO_SCOPES order", () => { + const { spec, cells } = planned(); + const result = previewSet({ spec, cells, scopes: ["runs:write", "journeys:read"] }); + expect(refusalsOf(result)).toEqual([ + { kind: "missing_scopes", missing: ["journeys:write", "runs:read", "intents:read"] }, + ]); + }); + + it("still checks agents and totals with the data it has", () => { + const { spec, cells } = planned({ agents: "nope" }); + const result = previewSet({ + spec, + cells, + scopes: ["journeys:write"], + agents: catalog, + usage, + }); + expect(kindsOf(result)).toEqual(["missing_scopes", "unknown_agent"]); + }); + + it("refuses a clean set that lacks a scope", () => { + const { spec, cells } = planned(); + const scopes = ALL_SCOPES.filter((scope) => scope !== "runs:write"); + const result = previewSet({ spec, cells, scopes, agents: catalog, usage }); + expect(refusalsOf(result)).toEqual([{ kind: "missing_scopes", missing: ["runs:write"] }]); + }); + + it("throws when the data is absent although no scope is missing", () => { + const { spec, cells } = planned(); + expect(() => previewSet({ spec, cells, scopes: ALL_SCOPES })).toThrow(/agent catalog/); + }); +}); + +describe("previewSet agent refusals", () => { + it("refuses an unknown harness", () => { + expect(refusalsOf(preview({ agents: "gemini-cli" }))).toEqual([ + { kind: "unknown_agent", requested: "gemini-cli" }, + ]); + }); + + it("refuses a pair whose model the catalog does not list", () => { + expect(refusalsOf(preview({ agents: "claude-code/claude-opus-9" }))).toEqual([ + { kind: "unknown_agent", requested: "claude-code/claude-opus-9" }, + ]); + }); + + it("refuses an unavailable pair with the catalog's reason", () => { + expect(refusalsOf(preview({ agents: "codex/gpt-5.6-luna" }))).toEqual([ + { + kind: "unavailable_agent", + requested: "codex/gpt-5.6-luna", + reason: "needs the Standard plan", + }, + ]); + }); + + it("refuses a bare harness with no available entry, carrying the first entry's reason", () => { + const result = preview({ agents: "claude-code" }, { agents: claudeCodeUnavailable }); + expect(refusalsOf(result)).toEqual([ + { kind: "unavailable_agent", requested: "claude-code", reason: "needs the Pro plan" }, + ]); + }); + + it("refuses a default when the catalog offers no available agent", () => { + const result = preview({}, { agents: nothingAvailable }); + expect(refusalsOf(result)).toEqual([{ kind: "no_agent_offered" }]); + }); + + it("refuses an empty catalog as no agent offered", () => { + expect(refusalsOf(preview({}, { agents: [] }))).toEqual([{ kind: "no_agent_offered" }]); + }); + + it("refuses a bare harness and the pair it resolves to as a duplicate", () => { + expect(refusalsOf(preview({ agents: `claude-code,${HAIKU}` }))).toEqual([ + { kind: "duplicate_agent", requested: ["claude-code", HAIKU], label: HAIKU }, + ]); + }); + + it("does not call two different models of one harness a duplicate", () => { + expect(preview({ agents: `${HAIKU},${SONNET}` }).ok).toBe(true); + }); + + it("reports every agent refusal in the order requested", () => { + const result = preview({ agents: "gemini-cli,codex/gpt-5.6-luna" }); + expect(kindsOf(result)).toEqual(["unknown_agent", "unavailable_agent"]); + }); + + it("hides the cap and allowance checks behind an agent refusal", () => { + // 20 cells x 10 repeats would break the cap and the allowance, but the agent is unknown. + const result = preview( + { + agents: "gemini-cli", + repeat: "10", + docs: "https://linear.app/docs", + exact: "https://linear.app/developers", + }, + { usage: withRemaining(0) }, + 4, + ); + expect(kindsOf(result)).toEqual(["unknown_agent"]); + }); +}); diff --git a/src/journey-set/preview.ts b/src/journey-set/preview.ts new file mode 100644 index 0000000..1299b62 --- /dev/null +++ b/src/journey-set/preview.ts @@ -0,0 +1,242 @@ +// The go or refuse decision before a set launches: a pure function over data the command +// fetched (the key's scopes, the agent catalog, this month's usage) and the planned cells. +// It resolves each requested agent to a catalog pair, totals the runs, and lists every reason +// the set cannot start. It prints nothing; the preview renderer words the result. + +import { + STUDIO_SCOPES, + type StudioAgent, + type StudioScope, + type StudioUsage, +} from "../api/studio-types"; +import { + type AgentChoice, + DEFAULT_HARNESS, + exceedsRunCap, + MAX_RUNS, + type PlannedCell, + plannedRuns, + type SetSpec, +} from "./spec"; + +export interface PreviewInput { + spec: SetSpec; + cells: readonly PlannedCell[]; + /** The scopes the key's bearer carries. */ + scopes: readonly string[]; + /** The catalog in the order Studio lists it. Required unless a scope refusal blocks the fetch. */ + agents?: readonly StudioAgent[]; + /** This month's usage. Required unless a scope refusal blocks the fetch. */ + usage?: StudioUsage; +} + +/** An agent resolved to the catalog's pair; launch sends `{ harness, model }`. */ +export interface ResolvedAgent { + harness: string; + model: string; + /** `harness/model`, the form --agents accepts and every report column uses. */ + label: string; + harnessName: string; + modelName: string; +} + +export interface PreviewTotals { + /** Runs the set launches: cells x resolved agents x repeats. */ + runs: number; + /** One per goal, level and repeat. */ + journeys: number; + runsPerRepeat: number; +} + +/** What is left of the plan's monthly runs, for the renderer. */ +export type Allowance = + | { kind: "limited"; limit: number | null; remaining: number; resetsAt: string | null } + | { kind: "unlimited"; resetsAt: string | null }; + +export type Refusal = + | { kind: "missing_scopes"; missing: StudioScope[] } + /** `requested` is the text form of the choice: `harness` or `harness/model`. */ + | { kind: "unknown_agent"; requested: string } + | { kind: "unavailable_agent"; requested: string; reason: string | null } + | { kind: "duplicate_agent"; requested: string[]; label: string } + | { kind: "no_agent_offered" } + | { kind: "over_cap"; runs: number; maxRuns: number } + | { + kind: "over_allowance"; + runs: number; + remaining: number; + resetsAt: string | null; + /** The largest repeat count that fits, with its runs; null when not even one repeat fits. */ + fits: { repeat: number; runs: number } | null; + /** + * True when only --repeat made this a set and the repeat that fits is 1: the call that fits + * is one journey, so the hint says so instead of suggesting --repeat 1. + */ + oneJourney: boolean; + }; + +export type PreviewResult = + | { + ok: true; + agents: ResolvedAgent[]; + /** True when --agents was omitted and the default rule picked the agent (resolveDefault). */ + defaultedAgent: boolean; + totals: PreviewTotals; + allowance: Allowance; + } + | { ok: false; refusals: Refusal[] }; + +function requestText(choice: AgentChoice): string { + return choice.model ? `${choice.harness}/${choice.model}` : choice.harness; +} + +function resolved(entry: StudioAgent): ResolvedAgent { + return { + harness: entry.harness, + model: entry.model, + label: `${entry.harness}/${entry.model}`, + harnessName: entry.harnessName, + modelName: entry.modelName, + }; +} + +/** The catalog entry a choice names, or the refusal for why none can run. */ +function lookup( + choice: AgentChoice, + catalog: readonly StudioAgent[], +): { entry: StudioAgent } | { refusal: Refusal } { + const requested = requestText(choice); + const sameHarness = catalog.filter((entry) => entry.harness === choice.harness); + const candidates = choice.model + ? sameHarness.filter((entry) => entry.model === choice.model) + : sameHarness; + if (candidates.length === 0) return { refusal: { kind: "unknown_agent", requested } }; + const entry = candidates.find((candidate) => candidate.available); + if (entry) return { entry }; + return { refusal: { kind: "unavailable_agent", requested, reason: candidates[0].reason } }; +} + +interface Resolution { + agents: ResolvedAgent[]; + defaultedAgent: boolean; + refusals: Refusal[]; +} + +/** The plan's first Claude Code agent, the harness a single journey defaults to, else its first. */ +function resolveDefault(catalog: readonly StudioAgent[]): Resolution { + const offered = catalog.filter((entry) => entry.available); + const chosen = offered.find((entry) => entry.harness === DEFAULT_HARNESS) ?? offered[0]; + if (!chosen) { + return { agents: [], defaultedAgent: false, refusals: [{ kind: "no_agent_offered" }] }; + } + return { agents: [resolved(chosen)], defaultedAgent: true, refusals: [] }; +} + +/** Every requested agent against the catalog; two requests landing on one pair are refused. */ +function resolveAgents(spec: SetSpec, catalog: readonly StudioAgent[]): Resolution { + if (spec.agents.length === 0) return resolveDefault(catalog); + const agents: ResolvedAgent[] = []; + const refusals: Refusal[] = []; + const requestsByLabel = new Map(); + for (const choice of spec.agents) { + const found = lookup(choice, catalog); + if ("refusal" in found) { + refusals.push(found.refusal); + continue; + } + const agent = resolved(found.entry); + const earlier = requestsByLabel.get(agent.label); + if (earlier) { + earlier.push(requestText(choice)); + continue; + } + requestsByLabel.set(agent.label, [requestText(choice)]); + agents.push(agent); + } + for (const [label, requested] of requestsByLabel) { + if (requested.length > 1) refusals.push({ kind: "duplicate_agent", requested, label }); + } + return { agents, defaultedAgent: false, refusals }; +} + +/** The largest repeat count below the requested one that fits the allowance and the run cap. */ +function largestFittingRepeat( + perRepeat: number, + remaining: number, + requestedRepeat: number, +): { repeat: number; runs: number } | null { + for (let repeat = requestedRepeat - 1; repeat >= 1; repeat--) { + const runs = repeat * perRepeat; + if (runs <= remaining && !exceedsRunCap(runs)) return { repeat, runs }; + } + return null; +} + +function allowanceOf(usage: StudioUsage): Allowance { + const { limit, remaining } = usage.runs; + if (remaining === null) return { kind: "unlimited", resetsAt: usage.resetsAt }; + return { kind: "limited", limit, remaining, resetsAt: usage.resetsAt }; +} + +function limitRefusals( + totals: PreviewTotals, + spec: Pick, + allowance: Allowance, +): Refusal[] { + const refusals: Refusal[] = []; + if (exceedsRunCap(totals.runs)) { + refusals.push({ kind: "over_cap", runs: totals.runs, maxRuns: MAX_RUNS }); + } + if (allowance.kind === "limited" && allowance.remaining < totals.runs) { + const fits = largestFittingRepeat(totals.runsPerRepeat, allowance.remaining, spec.repeat); + refusals.push({ + kind: "over_allowance", + runs: totals.runs, + remaining: allowance.remaining, + resetsAt: allowance.resetsAt, + fits, + oneJourney: fits?.repeat === 1 && !spec.levels && totals.runsPerRepeat === 1, + }); + } + return refusals; +} + +/** + * Decide go or refuse. Every refusal that the data allows is returned together. An agent refusal + * hides the cap and allowance checks, because their totals depend on an agent count not yet + * known. A missing scope blocks the fetch of the agents and usage, so a caller that could not + * fetch them omits them; omitting them while no scope is missing is a caller bug and throws. + */ +export function previewSet(input: PreviewInput): PreviewResult { + const { spec, cells, scopes, agents: catalog, usage } = input; + const refusals: Refusal[] = []; + + const missing = STUDIO_SCOPES.filter((scope) => !scopes.includes(scope)); + if (missing.length > 0) refusals.push({ kind: "missing_scopes", missing }); + else if (!catalog || !usage) { + throw new Error("previewSet needs the agent catalog and usage once no scope is missing"); + } + + const resolution = catalog ? resolveAgents(spec, catalog) : undefined; + if (resolution) refusals.push(...resolution.refusals); + if (!resolution || resolution.refusals.length > 0) return { ok: false, refusals }; + + const perRepeat = cells.length * resolution.agents.length; + const totals: PreviewTotals = { + runs: plannedRuns(cells, resolution.agents.length, spec.repeat), + journeys: cells.length * spec.repeat, + runsPerRepeat: perRepeat, + }; + const allowance = usage ? allowanceOf(usage) : undefined; + // Without usage the allowance check is skipped, but the cap still applies. + refusals.push(...limitRefusals(totals, spec, allowance ?? { kind: "unlimited", resetsAt: null })); + if (refusals.length > 0 || !allowance) return { ok: false, refusals }; + + return { + ok: true, + agents: resolution.agents, + defaultedAgent: resolution.defaultedAgent, + totals, + allowance, + }; +} diff --git a/src/journey-set/record.test.ts b/src/journey-set/record.test.ts new file mode 100644 index 0000000..7149e7e --- /dev/null +++ b/src/journey-set/record.test.ts @@ -0,0 +1,308 @@ +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + appendJourney, + createRecord, + DEFAULT_SETS_DIR, + isSetId, + type JourneyEntry, + listRecords, + markStopped, + readRecord, + SET_ID_PATTERN, + type SetRecord, +} from "./record"; +import { parseSetSpec, planCells, type SetSpec } from "./spec"; + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ax-set-record-")); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +function specFor(site = "linear.app"): SetSpec { + const parsed = parseSetSpec({ + site, + levels: true, + goals: ["Track a bug", "Plan a sprint"], + agents: "claude-code/sonnet,codex/gpt-5", + exact: `https://${site}/docs`, + repeat: "2", + }); + if (!parsed.ok) throw new Error(parsed.error); + return parsed.spec; +} + +function newRecord(site = "linear.app"): SetRecord { + const spec = specFor(site); + return createRecord( + { + spec, + agents: [ + { harness: "claude-code", model: "sonnet" }, + { harness: "codex", model: "gpt-5" }, + ], + cells: planCells(spec).cells, + }, + dir, + ); +} + +function entry(n: number, overrides: Partial = {}): JourneyEntry { + return { + journeyId: `j-${n}`, + goalIndex: 0, + level: "brand", + repeat: 1, + runIds: [`r-${n}a`, `r-${n}b`], + failed: [], + createdAt: `2026-11-13T10:00:0${n}.000Z`, + ...overrides, + }; +} + +describe("the id", () => { + it("is t- plus ten lowercase characters", () => { + const record = newRecord(); + expect(record.id).toMatch(SET_ID_PATTERN); + expect(SET_ID_PATTERN.source).toBe("^j-[a-z0-9]{10}$"); + }); + + it("differs between records", () => { + expect(newRecord().id).not.toBe(newRecord().id); + }); + + it("accepts a well formed id and nothing else", () => { + expect(isSetId("j-abc123def0")).toBe(true); + for (const bad of [ + "", + "j-short", + "j-ABC123DEF0", + "j-abc123def01", + "x-abc123def0", + "j-../../", + ]) { + expect(isSetId(bad)).toBe(false); + } + }); +}); + +describe("createRecord", () => { + it("writes the record before any journey exists", () => { + const record = newRecord(); + expect(record.journeys).toEqual([]); + expect(record.stoppedReason).toBeUndefined(); + expect(readdirSync(dir)).toEqual([`${record.id}.json`]); + }); + + it("round trips the spec, the resolved agents and the cells", () => { + const record = newRecord(); + const spec = specFor(); + expect(readRecord(record.id, dir)).toEqual({ + id: record.id, + createdAt: record.createdAt, + spec, + agents: [ + { harness: "claude-code", model: "sonnet" }, + { harness: "codex", model: "gpt-5" }, + ], + cells: planCells(spec).cells, + journeys: [], + }); + expect(Number.isNaN(Date.parse(record.createdAt))).toBe(false); + }); + + it("creates the directory when it is missing", () => { + const nested = join(dir, "a", "b"); + const spec = specFor(); + const record = createRecord({ spec, agents: [], cells: [] }, nested); + expect(readRecord(record.id, nested)?.id).toBe(record.id); + }); +}); + +describe("appendJourney", () => { + it("keeps entries in the order they were appended", () => { + const { id } = newRecord(); + appendJourney(id, entry(1), dir); + appendJourney(id, entry(2, { level: "exact" }), dir); + appendJourney(id, entry(3, { repeat: 2 }), dir); + const record = readRecord(id, dir); + expect(record?.journeys.map((journey) => journey.journeyId)).toEqual(["j-1", "j-2", "j-3"]); + expect(record?.journeys[1].level).toBe("exact"); + }); + + it("stores a null journey id and per agent failures", () => { + const { id } = newRecord(); + const failed = [{ harness: "codex", model: "gpt-5", reason: "gateway timeout" }]; + appendJourney(id, entry(1, { journeyId: null, runIds: [], failed }), dir); + const stored = readRecord(id, dir)?.journeys[0]; + expect(stored?.journeyId).toBeNull(); + expect(stored?.failed).toEqual(failed); + }); + + it("returns the updated record", () => { + const { id } = newRecord(); + expect(appendJourney(id, entry(1), dir).journeys).toHaveLength(1); + }); + + it("throws for a record that does not exist", () => { + expect(() => appendJourney("j-aaaaaaaaaa", entry(1), dir)).toThrow(/no set record/); + }); + + it("throws for an invalid id without touching the disk", () => { + expect(() => appendJourney("../x", entry(1), dir)).toThrow(/invalid set id/); + expect(readdirSync(dir)).toEqual([]); + }); +}); + +describe("markStopped", () => { + it("records why launching stopped and how many runs never started", () => { + const { id } = newRecord(); + appendJourney(id, entry(1), dir); + const stopped = markStopped(id, "quota", 6, dir); + expect(stopped.stoppedReason).toBe("quota"); + expect(stopped.notLaunched).toBe(6); + const stored = readRecord(id, dir); + expect(stored?.stoppedReason).toBe("quota"); + expect(stored?.notLaunched).toBe(6); + expect(stored?.journeys).toHaveLength(1); + }); + + it("accepts every stop reason", () => { + for (const reason of ["quota", "platform_down", "api_error", "interrupted"] as const) { + const { id } = newRecord(); + expect(markStopped(id, reason, 0, dir).stoppedReason).toBe(reason); + } + }); + + it("throws for an invalid id", () => { + expect(() => markStopped("j-../../", "interrupted", 1, dir)).toThrow(/invalid set id/); + }); +}); + +describe("atomic writes", () => { + it("leaves no temp file behind after create, append or stop", () => { + const { id } = newRecord(); + appendJourney(id, entry(1), dir); + markStopped(id, "interrupted", 3, dir); + expect(readdirSync(dir)).toEqual([`${id}.json`]); + }); + + it("leaves the old record intact and no temp file when a write fails", () => { + const { id } = newRecord(); + const before = readFileSync(join(dir, `${id}.json`), "utf8"); + const circular: JourneyEntry = entry(1); + (circular as unknown as { self: unknown }).self = circular; + expect(() => appendJourney(id, circular, dir)).toThrow(); + expect(readFileSync(join(dir, `${id}.json`), "utf8")).toBe(before); + expect(readdirSync(dir)).toEqual([`${id}.json`]); + }); +}); + +describe("readRecord", () => { + it("returns null for an id that has no record", () => { + expect(readRecord("j-aaaaaaaaaa", dir)).toBeNull(); + }); + + it("returns null for a missing directory", () => { + expect(readRecord("j-aaaaaaaaaa", join(dir, "nope"))).toBeNull(); + }); + + it("rejects traversal ids before building a path", () => { + const outside = join(dir, "x.json"); + writeFileSync(outside, JSON.stringify({ id: "x" })); + const inner = join(dir, "inner"); + mkdirSync(inner); + expect(readRecord("../x", inner)).toBeNull(); + expect(readRecord("j-../../", inner)).toBeNull(); + expect(readRecord("../x.json", inner)).toBeNull(); + }); + + it("throws a named error for a file that is not a set record", () => { + writeFileSync(join(dir, "j-aaaaaaaaaa.json"), "{ not json"); + expect(() => readRecord("j-aaaaaaaaaa", dir)).toThrow(/j-aaaaaaaaaa/); + writeFileSync(join(dir, "j-bbbbbbbbbb.json"), JSON.stringify({ id: "j-bbbbbbbbbb" })); + expect(() => readRecord("j-bbbbbbbbbb", dir)).toThrow(/j-bbbbbbbbbb/); + }); +}); + +describe("listRecords", () => { + it("returns one summary per file, newest first", () => { + const first = newRecord("linear.app"); + appendJourney(first.id, entry(1), dir); + appendJourney(first.id, entry(2, { runIds: ["r-x"] }), dir); + const second = newRecord("stripe.com"); + const stored = readRecord(second.id, dir) as SetRecord; + writeFileSync( + join(dir, `${second.id}.json`), + JSON.stringify({ ...stored, createdAt: "2099-01-01T00:00:00.000Z" }), + ); + const { records, skipped } = listRecords(dir); + expect(skipped).toEqual([]); + expect(records).toEqual([ + { + id: second.id, + createdAt: "2099-01-01T00:00:00.000Z", + site: "stripe.com", + journeys: 0, + runs: 0, + }, + { id: first.id, createdAt: first.createdAt, site: "linear.app", journeys: 2, runs: 3 }, + ]); + }); + + it("skips a file that does not parse and names it", () => { + const good = newRecord(); + writeFileSync(join(dir, "j-corruptaaa.json"), "{ not json"); + writeFileSync(join(dir, "j-wrongshape.json"), JSON.stringify({ hello: "world" })); + const { records, skipped } = listRecords(dir); + expect(records.map((summary) => summary.id)).toEqual([good.id]); + expect(skipped).toEqual(["j-corruptaaa.json", "j-wrongshape.json"]); + }); + + it("ignores files that are not set records", () => { + const good = newRecord(); + writeFileSync(join(dir, "notes.txt"), "hi"); + writeFileSync(join(dir, `.${good.id}.1234.tmp`), "{}"); + const { records, skipped } = listRecords(dir); + expect(records).toHaveLength(1); + expect(skipped).toEqual([]); + }); + + it("is empty when the directory does not exist", () => { + expect(listRecords(join(dir, "nope"))).toEqual({ records: [], skipped: [] }); + }); +}); + +describe("secrets", () => { + function keysOf(value: unknown): string[] { + if (Array.isArray(value)) return value.flatMap(keysOf); + if (value && typeof value === "object") { + return Object.entries(value).flatMap(([key, inner]) => [key, ...keysOf(inner)]); + } + return []; + } + + it("never serializes a key or token field", () => { + const { id } = newRecord(); + appendJourney(id, entry(1), dir); + markStopped(id, "api_error", 2, dir); + const text = readFileSync(join(dir, `${id}.json`), "utf8"); + const keys = keysOf(JSON.parse(text)).map((key) => key.toLowerCase()); + expect(keys).not.toContain("ora_api_key"); + expect(keys.filter((key) => key.includes("token") || key.includes("bearer"))).toEqual([]); + expect(text).not.toMatch(/ORA_API_KEY|Bearer/); + }); +}); + +describe("the default directory", () => { + it("is ~/.ax/journeys", () => { + expect(DEFAULT_SETS_DIR).toMatch(/[\\/]\.ax[\\/]journeys$/); + }); +}); diff --git a/src/journey-set/record.ts b/src/journey-set/record.ts new file mode 100644 index 0000000..c83f12b --- /dev/null +++ b/src/journey-set/record.ts @@ -0,0 +1,261 @@ +// The local record of a set of journeys: one JSON file per set under ~/.ax/journeys/, written +// before the first journey is created and extended the moment each create returns, so +// `ax journey view`, `list` and `cancel` can pick a set up after the terminal is gone. The record +// holds the parsed spec, the resolved agents, the planned cells and one entry per create. It has no +// field for a key or a token and never will: the secret lives in the environment only (AGENTS.md +// invariant 7). Everything here is synchronous, so a read-modify-write cannot interleave with +// another in the same process. + +import { randomBytes } from "node:crypto"; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { Level } from "./levels"; +import type { AgentChoice, PlannedCell, SetSpec } from "./spec"; + +export const DEFAULT_SETS_DIR = join(homedir(), ".ax", "journeys"); + +export const SET_ID_PATTERN = /^j-[a-z0-9]{10}$/; + +const RECORD_FILE = /^(j-[a-z0-9]{10})\.json$/; + +export type StoppedReason = "quota" | "platform_down" | "api_error" | "interrupted"; + +/** An agent whose run did not start in a create, with the reason the server gave. */ +export interface FailedAgent { + harness: string; + model?: string; + reason: string; +} + +/** + * One create. `goalIndex` is the index into `spec.goals` and `repeat` counts from 1. + * `journeyId` is null when a 502 carried no journey. `runIds` are the runs that started. + */ +export interface JourneyEntry { + journeyId: string | null; + goalIndex: number; + level: Level; + repeat: number; + runIds: string[]; + failed: FailedAgent[]; + createdAt: string; +} + +export interface SetRecord { + id: string; + createdAt: string; + spec: SetSpec; + /** The agents as the catalog resolved them (the spec's own list is empty when --agents was omitted). */ + agents: AgentChoice[]; + cells: PlannedCell[]; + journeys: JourneyEntry[]; + stoppedReason?: StoppedReason; + /** Runs that never started because launching stopped. */ + notLaunched?: number; +} + +export interface NewRecord { + spec: SetSpec; + agents: AgentChoice[]; + cells: PlannedCell[]; +} + +export interface RecordSummary { + id: string; + createdAt: string; + site: string; + journeys: number; + /** Runs that started, summed over the journeys. */ + runs: number; +} + +export interface RecordList { + records: RecordSummary[]; + /** File names that did not parse as a set record. */ + skipped: string[]; +} + +export function isSetId(id: string): boolean { + return SET_ID_PATTERN.test(id); +} + +function newId(dir: string): string { + for (;;) { + const id = `j-${randomBytes(5).toString("hex")}`; + if (!existsSync(recordPath(id, dir))) return id; + } +} + +/** The only place a path is built from an id; the id is checked first. */ +function recordPath(id: string, dir: string): string { + if (!isSetId(id)) throw new Error(`invalid set id ${JSON.stringify(id)}`); + return join(dir, `${id}.json`); +} + +/** A file that exists but is not a set record; `listRecords` skips these, `readRecord` surfaces them. */ +export class CorruptRecordError extends Error { + constructor(id: string, why: string) { + super(`set record ${id} ${why}`); + } +} + +function isNotFound(error: unknown): boolean { + return error instanceof Error && "code" in error && error.code === "ENOENT"; +} + +/** Write a temp file beside the target, then rename it over: a reader sees the old or the new file. */ +function writeAtomic(path: string, record: SetRecord, dir: string): void { + const text = `${JSON.stringify(record, null, "\t")}\n`; + mkdirSync(dir, { recursive: true, mode: 0o700 }); + const temp = join(dir, `.${record.id}.${randomBytes(4).toString("hex")}.tmp`); + try { + writeFileSync(temp, text, { mode: 0o600 }); + renameSync(temp, path); + } catch (error) { + rmSync(temp, { force: true }); + throw error; + } +} + +function looksLikeRecord(value: unknown): value is SetRecord { + if (!value || typeof value !== "object") return false; + const record = value as Partial; + return ( + typeof record.id === "string" && + typeof record.createdAt === "string" && + typeof record.spec?.site === "string" && + Array.isArray(record.journeys) + ); +} + +function parseRecord(text: string, id: string): SetRecord { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch (error) { + // Only a malformed file is expected here; anything else propagates. + if (!(error instanceof SyntaxError)) throw error; + throw new CorruptRecordError(id, "is not valid JSON"); + } + if (!looksLikeRecord(parsed) || parsed.id !== id) { + throw new CorruptRecordError(id, "is not a set record"); + } + return parsed; +} + +/** Create the record before the first journey: the spec, the resolved agents, the cells, no journeys. */ +export function createRecord(input: NewRecord, dir = DEFAULT_SETS_DIR): SetRecord { + const id = newId(dir); + const record: SetRecord = { + id, + createdAt: new Date().toISOString(), + spec: input.spec, + agents: input.agents, + cells: input.cells, + journeys: [], + }; + writeAtomic(recordPath(id, dir), record, dir); + return record; +} + +/** The record for `id`, or null when the id is not well formed or no such record exists. */ +export function readRecord(id: string, dir = DEFAULT_SETS_DIR): SetRecord | null { + if (!isSetId(id)) return null; + let text: string; + try { + text = readFileSync(recordPath(id, dir), "utf8"); + } catch (error) { + // A missing file or directory is how an unknown id looks; other errors propagate. + if (isNotFound(error)) return null; + throw error; + } + return parseRecord(text, id); +} + +function readExisting(id: string, dir: string): SetRecord { + recordPath(id, dir); // throws on an invalid id; readRecord alone would answer null + const record = readRecord(id, dir); + if (!record) throw new Error(`no set record ${id} in ${dir}`); + return record; +} + +function update(id: string, dir: string, change: (record: SetRecord) => void): SetRecord { + const record = readExisting(id, dir); + change(record); + writeAtomic(recordPath(id, dir), record, dir); + return record; +} + +/** Append one create's outcome, in the order the creates returned. */ +export function appendJourney( + id: string, + journey: JourneyEntry, + dir = DEFAULT_SETS_DIR, +): SetRecord { + return update(id, dir, (record) => { + record.journeys.push(journey); + }); +} + +/** Record that launching stopped early and how many runs never started. */ +export function markStopped( + id: string, + stoppedReason: StoppedReason, + notLaunched: number, + dir = DEFAULT_SETS_DIR, +): SetRecord { + return update(id, dir, (record) => { + record.stoppedReason = stoppedReason; + record.notLaunched = notLaunched; + }); +} + +function summarize(record: SetRecord): RecordSummary { + const runs = record.journeys.reduce((total, journey) => total + journey.runIds.length, 0); + return { + id: record.id, + createdAt: record.createdAt, + site: record.spec.site, + journeys: record.journeys.length, + runs, + }; +} + +function newestFirst(a: RecordSummary, b: RecordSummary): number { + return b.createdAt.localeCompare(a.createdAt) || a.id.localeCompare(b.id); +} + +/** One summary per record file, newest first. Local only: no server is asked. */ +export function listRecords(dir = DEFAULT_SETS_DIR): RecordList { + let names: string[]; + try { + names = readdirSync(dir); + } catch (error) { + // No directory yet means no set has run on this machine. + if (isNotFound(error)) return { records: [], skipped: [] }; + throw error; + } + const records: RecordSummary[] = []; + const skipped: string[] = []; + for (const name of names.sort()) { + const match = RECORD_FILE.exec(name); + if (!match) continue; + try { + records.push(summarize(parseRecord(readFileSync(join(dir, name), "utf8"), match[1]))); + } catch (error) { + // A corrupt file is reported by name; a failed read (permissions, I/O) is not ours to hide. + if (!(error instanceof CorruptRecordError)) throw error; + skipped.push(name); + } + } + return { records: records.sort(newestFirst), skipped }; +} diff --git a/src/journey-set/render-preview.test.ts b/src/journey-set/render-preview.test.ts new file mode 100644 index 0000000..a0b8772 --- /dev/null +++ b/src/journey-set/render-preview.test.ts @@ -0,0 +1,527 @@ +import { describe, expect, it } from "vitest"; +import agentsFixture from "../api/__fixtures__/studio/agents.json"; +import usageFixture from "../api/__fixtures__/studio/usage.json"; +import usageUnlimitedFixture from "../api/__fixtures__/studio/usage-unlimited.json"; +import { StudioConnectionError, StudioShapeError } from "../api/studio"; +import { studioError } from "../api/studio-errors"; +import { STUDIO_SCOPES, type StudioAgent, type StudioUsage } from "../api/studio-types"; +import type { LaunchResult } from "./launch"; +import { type PreviewInput, previewSet, type Refusal } from "./preview"; +import { + closeTerminalLine, + confirmPrompt, + GET_A_KEY, + isProductNotGranted, + PRODUCT_NOT_GRANTED_MESSAGE, + type PreviewRender, + progressLine, + readFailureMessage, + renderPreview, + renderRefusal, + renderRefusals, + stopMessage, +} from "./render-preview"; +import { parseSetSpec, planCells, type SetSpecInput } from "./spec"; + +// The recorded Studio bodies, unwrapped from their `{ success, data }` envelope, run through the +// real previewSet, parseSetSpec and planCells. +const catalog: StudioAgent[] = agentsFixture.data.agents; +const usage: StudioUsage = usageFixture.data; +const usageUnlimited: StudioUsage = usageUnlimitedFixture.data; + +const GOAL = "Find how to create an issue from the API"; +const ESC = String.fromCharCode(27); +// biome-ignore lint/suspicious/noControlCharactersInRegex: the test looks for exactly these +const CONTROL = /[\u0000-\u001f\u007f-\u009f]/; + +function inputs(overrides: Partial = {}, goals: string[] = [GOAL]) { + const parsed = parseSetSpec({ site: "linear.app", goals, levels: true, ...overrides }); + if (!parsed.ok) throw new Error(parsed.error); + const plan = planCells(parsed.spec); + return { spec: parsed.spec, cells: plan.cells, skipped: plan.skipped }; +} + +function go( + overrides: Partial = {}, + rest: Partial = {}, + goals: string[] = [GOAL], +): PreviewRender { + const { spec, cells, skipped } = inputs(overrides, goals); + const preview = previewSet({ + spec, + cells, + scopes: [...STUDIO_SCOPES], + agents: catalog, + usage, + ...rest, + }); + if (!preview.ok) throw new Error(`expected a go: ${JSON.stringify(preview.refusals)}`); + return { spec, cells, skipped, preview }; +} + +function refuse( + overrides: Partial, + rest: Partial = {}, + goals: string[] = [GOAL], +): Refusal[] { + const { spec, cells } = inputs(overrides, goals); + const result = previewSet({ + spec, + cells, + scopes: [...STUDIO_SCOPES], + agents: catalog, + usage, + ...rest, + }); + if (result.ok) throw new Error("expected a refusal"); + return result.refusals; +} + +const withUsage = (resetsAt: string | null, remaining = 38): StudioUsage => ({ + ...usage, + runs: { ...usage.runs, remaining }, + resetsAt, +}); + +describe("GET_A_KEY", () => { + it("is the pinned copy, word for word", () => { + expect(GET_A_KEY).toBe( + "`ax journey` runs on your Front account. Sign up at app.ora.ai with a work email (personal addresses are not accepted), then create an API key in your account settings with these scopes: `journeys:write`, `journeys:read`, `runs:read`, `runs:write`, `intents:read`. Set it as `ORA_API_KEY`. The Free plan includes a monthly number of runs; before a set of journeys launches, its preview shows what is left.", + ); + }); + + it("names every scope the client asks for", () => { + for (const scope of STUDIO_SCOPES) expect(GET_A_KEY).toContain(`\`${scope}\``); + }); +}); + +describe("renderPreview", () => { + it("words the totals, the allowance and the tracked sites for one goal at three levels", () => { + const lines = renderPreview(go({ docs: "https://linear.app/docs", repeat: "2" })); + expect(lines).toContain("This set uses 6 runs (1 goal x 3 levels x 1 agent x 2 repeats)."); + expect(lines).toContain("Your plan has 38 runs left this month (resets Dec 1)."); + expect(lines).toContain( + "It adds linear.app to your Front account's tracked sites if it is not there yet (with a daily ora score scan, which does not count against your plan) and 6 journeys to your journeys list.", + ); + }); + + it("pluralises every count at 1 and at more than 1", () => { + const many = renderPreview(go({}, {}, ["Create an issue", "Find the pricing page"])); + expect(many[0]).toBe("This set uses 4 runs (2 goals x 2 levels x 1 agent x 1 repeat)."); + expect(many.at(-1)).toContain("and 4 journeys to your journeys list."); + + const one = renderPreview(oneRun()); + expect(one[0]).toBe("This set uses 1 run (1 goal x 1 level x 1 agent x 1 repeat)."); + expect(one).toContain("Your plan has 1 run left this month."); + expect(one.at(-1)).toContain("and 1 journey to your journeys list."); + }); + + it("uses the goal and level pairs form when goals run different numbers of levels", () => { + const lines = renderPreview( + go( + { + agents: "claude-code/claude-haiku-4-5-20251001,claude-code/claude-sonnet-5-5", + repeat: "3", + }, + {}, + ["Create an issue", "Use Linear to create an issue"], + ), + ); + expect(lines[0]).toBe("This set uses 18 runs (3 goal and level pairs x 2 agents x 3 repeats)."); + }); + + it("names the default agent only when --agents was omitted", () => { + const defaulted = renderPreview(go()); + expect(defaulted).toContain( + "Agent: Claude Code with Claude Haiku 4.5, the Claude Code agent your plan offers. Choose others with --agents.", + ); + const codex = catalog + .filter((agent) => agent.harness === "codex") + .map((agent) => ({ ...agent, available: true })); + expect(renderPreview(go({}, { agents: codex }))).toContain( + "Agent: Codex with GPT-5.6 Luna, the first one your plan offers, since it offers no Claude Code agent. Choose others with --agents.", + ); + const chosen = renderPreview(go({ agents: "claude-code/claude-sonnet-5-5" })); + expect(chosen.some((line) => line.startsWith("Agent: "))).toBe(false); + }); + + it("names the brand it used", () => { + expect(renderPreview(go())).toContain("Brand in the prompts: Linear. Change it with --brand."); + expect(renderPreview(go({ brand: "Acme" }))).toContain( + "Brand in the prompts: Acme. Change it with --brand.", + ); + }); + + it("leaves the brand out without levels, since no prompt carries it", () => { + const lines = renderPreview(go({ levels: false })); + expect(lines[0]).toBe("This set uses 1 run (1 goal x 1 level x 1 agent x 1 repeat)."); + expect(lines.some((line) => line.startsWith("Brand in the prompts"))).toBe(false); + }); + + it("prints every skip note verbatim", () => { + const render = go({}, {}, ["Use Linear to create an issue", GOAL]); + expect(render.skipped).toHaveLength(5); + const lines = renderPreview(render); + for (const skip of render.skipped) expect(lines).toContain(skip.note); + expect(lines).toContain( + "No --docs URL, so the docs level is skipped. Add --docs to include it.", + ); + expect(lines).toContain( + "No --exact URL, so the exact level is skipped. Add --exact to include it.", + ); + expect(lines).toContain( + "The goal names Linear, so the need level is skipped. Write the goal without it to include that level.", + ); + }); + + it("says there is no monthly limit for an unlimited plan", () => { + const lines = renderPreview(go({}, { usage: usageUnlimited })); + expect(lines).toContain("Your plan has no monthly run limit."); + expect(lines.some((line) => line.startsWith("Your plan has") && line.includes("left"))).toBe( + false, + ); + }); + + it("drops the parenthesis when the reset date is null", () => { + const lines = renderPreview(go({}, { usage: withUsage(null) })); + expect(lines).toContain("Your plan has 38 runs left this month."); + }); + + it("drops the parenthesis when the reset date does not parse", () => { + const lines = renderPreview(go({}, { usage: withUsage("soon") })); + expect(lines).toContain("Your plan has 38 runs left this month."); + }); + + it("reads the reset date in UTC on any machine", () => { + const earlier = process.env.TZ; + try { + for (const zone of ["Pacific/Honolulu", "Pacific/Auckland", "UTC"]) { + process.env.TZ = zone; + const midnight = renderPreview(go({}, { usage: withUsage("2026-12-01T00:00:00Z") })); + expect(midnight).toContain("Your plan has 38 runs left this month (resets Dec 1)."); + const lateEvening = renderPreview(go({}, { usage: withUsage("2026-12-01T23:30:00Z") })); + expect(lateEvening).toContain("Your plan has 38 runs left this month (resets Dec 1)."); + } + } finally { + if (earlier === undefined) delete process.env.TZ; + else process.env.TZ = earlier; + } + }); + + it("strips an escape sequence from a server string", () => { + const hostile = catalog.map((entry) => ({ + ...entry, + harnessName: `${ESC}[2JCode`, + modelName: `Model${ESC}]0;title\u0007`, + })); + const lines = renderPreview(go({}, { agents: hostile })); + const agentLine = lines.find((line) => line.startsWith("Agent: ")); + expect(agentLine).toBeDefined(); + expect(agentLine).not.toMatch(CONTROL); + expect(agentLine).toContain("Code"); + }); +}); + +/** One goal at the brand level only: one run, one journey, one run left on the plan. */ +function oneRun(): PreviewRender { + const render = go({}, {}, ["Create an issue"]); + const cells = render.cells.filter((cell) => cell.level === "brand"); + const preview = previewSet({ + spec: render.spec, + cells, + scopes: [...STUDIO_SCOPES], + agents: catalog, + usage: withUsage(null, 1), + }); + if (!preview.ok) throw new Error("expected a go"); + return { ...render, cells, skipped: [], preview }; +} + +describe("the line printed once the record exists", () => { + it("carries the real id", () => { + expect(closeTerminalLine("j-abc123def4")).toBe( + "You can close the terminal; `ax journey view j-abc123def4` picks it up.", + ); + }); +}); + +describe("confirmPrompt", () => { + it("asks a question about the runs, ready for an answer", () => { + expect(confirmPrompt(6)).toBe("Launch 6 runs? [y/N] "); + expect(confirmPrompt(1)).toBe("Launch 1 run? [y/N] "); + }); +}); + +describe("renderRefusal", () => { + it("lists each missing scope and ends with GET_A_KEY", () => { + const refusals = refuse( + {}, + { scopes: ["journeys:read", "runs:read"], agents: undefined, usage: undefined }, + ); + const lines = renderRefusals(refusals); + expect(lines[0]).toBe( + "Your API key is missing these scopes: journeys:write, runs:write, intents:read.", + ); + expect(lines.at(-1)).toBe(GET_A_KEY); + }); + + it("names an unknown agent", () => { + const [refusal] = refuse({ agents: "nope" }); + expect(renderRefusal(refusal)).toEqual([ + "No agent is named nope. --agents takes harness or harness/model.", + ]); + }); + + it("quotes the catalog's reason for an unavailable agent", () => { + const [refusal] = refuse({ agents: "codex" }); + expect(renderRefusal(refusal)).toEqual([ + "The agent codex is not available to your account: needs the Standard plan.", + ]); + }); + + it("handles an unavailable agent with no reason and a reason that ends in a period", () => { + expect(renderRefusal({ kind: "unavailable_agent", requested: "codex", reason: null })).toEqual([ + "The agent codex is not available to your account.", + ]); + expect( + renderRefusal({ kind: "unavailable_agent", requested: "codex", reason: "needs Pro." }), + ).toEqual(["The agent codex is not available to your account: needs Pro."]); + }); + + it("strips an escape sequence from a catalog reason", () => { + const [line] = renderRefusal({ + kind: "unavailable_agent", + requested: "codex", + reason: `${ESC}[31mneeds Pro${ESC}[0m`, + }); + expect(line).not.toMatch(CONTROL); + expect(line).toContain("needs Pro"); + }); + + it("names two requests that land on one agent", () => { + const [refusal] = refuse({ agents: "claude-code,claude-code/claude-haiku-4-5-20251001" }); + expect(renderRefusal(refusal)).toEqual([ + "claude-code and claude-code/claude-haiku-4-5-20251001 name the same agent, claude-code/claude-haiku-4-5-20251001. Name it once.", + ]); + }); + + it("says no agent is offered", () => { + const none = catalog.map((entry) => ({ ...entry, available: false })); + const [refusal] = refuse({}, { agents: none }); + expect(renderRefusal(refusal)).toEqual([ + "No agent is available to your account right now. Check your plan at app.ora.ai.", + ]); + }); + + it("states the run cap", () => { + const everyAgent = catalog.map((entry) => ({ ...entry, available: true, reason: null })); + const [refusal] = refuse( + { + agents: "codex,claude-code/claude-haiku-4-5-20251001,claude-code/claude-sonnet-5-5", + repeat: "10", + }, + { agents: everyAgent }, + ["a", "b", "c", "d", "e"], + ); + expect(renderRefusal(refusal)).toEqual([ + "This set would use 300 runs, and a set is capped at 200. Cut goals, agents or repeats.", + ]); + }); + + it("says which repeat count fits, pluralised", () => { + // Two cells per repeat with 6 runs left: 3 repeats do not fit, 2 do. + const two = refuse({ repeat: "4" }, { usage: withUsage("2026-12-01T00:00:00Z", 6) }, [ + "Create an issue", + ]); + expect(renderRefusal(two[0])).toEqual([ + "This set needs 8 runs and your plan has 6 runs left this month (resets Dec 1).", + "3 repeats fit: 6 runs. Try --repeat 3.", + ]); + const one = refuse({ repeat: "3" }, { usage: withUsage(null, 2) }, ["Create an issue"]); + expect(renderRefusal(one[0])).toEqual([ + "This set needs 6 runs and your plan has 2 runs left this month.", + "1 repeat fits: 2 runs. Try --repeat 1.", + ]); + const exactTwo = refuse({ repeat: "3" }, { usage: withUsage(null, 5) }, ["Create an issue"]); + expect(renderRefusal(exactTwo[0])[1]).toBe("2 repeats fit: 4 runs. Try --repeat 2."); + }); + + it("says when not even one repeat fits", () => { + const [refusal] = refuse({}, { usage: withUsage("2026-12-01T00:00:00Z", 1) }, [ + "Create an issue", + ]); + expect(renderRefusal(refusal)).toEqual([ + "This set needs 2 runs and your plan has 1 run left this month (resets Dec 1).", + "Not even one repeat fits in what is left. Cut goals or agents, or wait for the reset.", + ]); + }); + + it("prints every refusal when there are several", () => { + const lines = renderRefusals([ + { kind: "unknown_agent", requested: "a" }, + { kind: "unknown_agent", requested: "b" }, + ]); + expect(lines).toHaveLength(2); + }); +}); + +describe("the product grant message", () => { + it("says the key is for another product and ends with GET_A_KEY", () => { + expect(PRODUCT_NOT_GRANTED_MESSAGE).toContain("made for another product"); + expect(PRODUCT_NOT_GRANTED_MESSAGE).toContain("Create a key in your Front account settings."); + expect(PRODUCT_NOT_GRANTED_MESSAGE.endsWith(GET_A_KEY)).toBe(true); + }); + + it("recognises a 403 with that code and nothing else", () => { + const grant = studioError({ status: 403, payload: { code: "PRODUCT_NOT_GRANTED" } }); + expect(isProductNotGranted(grant)).toBe(true); + expect(isProductNotGranted(studioError({ status: 403, payload: { code: "OTHER" } }))).toBe( + false, + ); + expect( + isProductNotGranted(studioError({ status: 401, payload: { code: "PRODUCT_NOT_GRANTED" } })), + ).toBe(false); + expect(isProductNotGranted(new Error("PRODUCT_NOT_GRANTED"))).toBe(false); + }); +}); + +describe("progressLine", () => { + it("counts queued, running and finished", () => { + expect(progressLine({ queued: 2, running: 1, finished: 3, waitingForSlot: false })).toBe( + "2 queued, 1 running, 3 finished", + ); + }); + + it("adds the slot clause when a run waits for one", () => { + expect(progressLine({ queued: 2, running: 1, finished: 0, waitingForSlot: true })).toBe( + "2 queued, 1 running, 0 finished; waiting for your plan's run slot", + ); + }); +}); + +describe("stopMessage", () => { + const stop = (over: Partial): LaunchResult => ({ + launchedRuns: 4, + notLaunched: 6, + ...over, + }); + const MAY_HAVE_STARTED = + "The last journey may have started its runs anyway; they will show in your account's journeys list."; + const DOWN = "The platform is likely down, so launching stopped. Runs that started keep running."; + + it("is empty for a launch that ran to the end", () => { + expect(stopMessage(stop({ notLaunched: 0 }))).toEqual([]); + }); + + it("says how many runs the quota left unlaunched, and that what started is followed", () => { + expect(stopMessage(stop({ stoppedReason: "quota" }))).toEqual([ + "Your plan's run limit was reached, so launching stopped. 6 runs were not launched.", + "Following the 4 runs that started.", + ]); + expect(stopMessage(stop({ stoppedReason: "quota", notLaunched: 1, launchedRuns: 1 }))).toEqual([ + "Your plan's run limit was reached, so launching stopped. 1 run was not launched.", + "Following the 1 run that started.", + ]); + }); + + it("says no run started when the quota stopped the first journey", () => { + expect(stopMessage(stop({ stoppedReason: "quota", launchedRuns: 0 }))[1]).toBe( + "No run started.", + ); + }); + + it("adds the may have started line for a gateway 502", () => { + const lastError = studioError({ status: 502, payload: null }); + expect(lastError.gateway).toBe(true); + expect(stopMessage(stop({ stoppedReason: "platform_down", lastError }))).toEqual([ + DOWN, + MAY_HAVE_STARTED, + ]); + }); + + it("adds the may have started line for a connection error", () => { + const lastError = new StudioConnectionError("https://api.example", new Error("boom")); + expect(stopMessage(stop({ stoppedReason: "platform_down", lastError }))).toContain( + MAY_HAVE_STARTED, + ); + }); + + it("adds the may have started line for an answer of the wrong shape", () => { + const lastError = new StudioShapeError("a create", "a journey id"); + expect(stopMessage(stop({ stoppedReason: "platform_down", lastError }))).toContain( + MAY_HAVE_STARTED, + ); + }); + + it("leaves the line out after Studio's own 502", () => { + const lastError = studioError({ + status: 502, + payload: { data: { journeyId: "j-1", runIds: [], failed: [] } }, + }); + expect(lastError.gateway).toBe(false); + expect(stopMessage(stop({ stoppedReason: "platform_down", lastError }))).toEqual([DOWN]); + }); + + it("leaves the line out when no error is known", () => { + expect(stopMessage(stop({ stoppedReason: "platform_down" }))).toEqual([DOWN]); + }); + + it("quotes an API error's message and says what keeps running", () => { + const lastError = studioError({ status: 500, payload: { message: "database is down" } }); + expect(stopMessage(stop({ stoppedReason: "api_error", lastError }))).toEqual([ + "The platform answered with an error, so launching stopped: ora request failed (500): database is down", + "6 runs were not launched. Runs that started keep running.", + ]); + }); + + it("strips an escape sequence from the quoted error", () => { + const lastError = new Error(`${ESC}[2Jcleared`); + const lines = stopMessage(stop({ stoppedReason: "api_error", lastError })); + expect(lines[0]).not.toMatch(CONTROL); + expect(lines[0]).toContain("cleared"); + }); + + it("says launching was interrupted", () => { + expect(stopMessage(stop({ stoppedReason: "interrupted" }))).toEqual([ + "Launching stopped. 6 runs were not launched. Runs that started keep running.", + ]); + }); + + it("never names a command: the Next lines do", () => { + for (const reason of ["quota", "platform_down", "api_error", "interrupted"] as const) { + for (const line of stopMessage(stop({ stoppedReason: reason }))) { + expect(line).not.toMatch(/ax journey/); + } + } + }); +}); + +describe("readFailureMessage", () => { + it("says reading kept failing, with the last error when there is one", () => { + expect(readFailureMessage()).toEqual(["Reading from the platform kept failing."]); + const lines = readFailureMessage(new Error(`${ESC}[31mdown`)); + expect(lines[1]).toBe("The last error: [31mdown"); + expect(lines[1]).not.toContain(ESC); + }); +}); + +describe("copy rules", () => { + it("uses no dash characters and no time estimates in any string", () => { + const all: string[] = [ + GET_A_KEY, + PRODUCT_NOT_GRANTED_MESSAGE, + confirmPrompt(3), + closeTerminalLine("j-abc123def4"), + progressLine({ queued: 1, running: 1, finished: 1, waitingForSlot: true }), + ...renderPreview(go({}, {}, ["Use Linear to create an issue"])), + ...(["quota", "platform_down", "api_error", "interrupted"] as const).flatMap((reason) => + stopMessage({ launchedRuns: 1, notLaunched: 2, stoppedReason: reason }), + ), + ...readFailureMessage(new Error("down")), + ]; + for (const text of all) { + expect(text).not.toMatch(/[\u2013\u2014]/); + expect(text).not.toMatch(/\b(minutes?|seconds?|hours?|about|takes)\b/i); + } + }); +}); diff --git a/src/journey-set/render-preview.ts b/src/journey-set/render-preview.ts new file mode 100644 index 0000000..a0610fb --- /dev/null +++ b/src/journey-set/render-preview.ts @@ -0,0 +1,247 @@ +// The words of a set before it launches: the preview, the refusals, the confirm prompt, the key +// copy, the lines a launch stops with, and the error after reads kept failing. Pure functions +// that return strings; the command prints (and colors) them. Every string the server supplied +// (agent names, a catalog reason, an error message) goes through `plain()` before it is put in a +// line. + +import { StudioApiError, StudioConnectionError, StudioShapeError } from "../api/studio"; +import { plain } from "../ui/ansi"; +import type { FollowProgress } from "./follow"; +import type { LaunchResult } from "./launch"; +import type { PreviewResult, Refusal } from "./preview"; +import { DEFAULT_HARNESS, type PlannedCell, type SetSpec, type SkippedCell } from "./spec"; + +/** Shared copy: the README and the journeys docs page reproduce this word for word. */ +export const GET_A_KEY = + "`ax journey` runs on your Front account. Sign up at app.ora.ai with a work email (personal addresses are not accepted), then create an API key in your account settings with these scopes: `journeys:write`, `journeys:read`, `runs:read`, `runs:write`, `intents:read`. Set it as `ORA_API_KEY`. The Free plan includes a monthly number of runs; before a set of journeys launches, its preview shows what is left."; + +/** The Next line's reason when ax refused the platform URL a key would go to. */ +export const KEY_DESTINATION_WHY = "once ORA_PLATFORM_URL is fixed as said above"; + +/** The code of a 403 that says the key was made for another product. */ +export const PRODUCT_NOT_GRANTED_CODE = "PRODUCT_NOT_GRANTED"; + +export const PRODUCT_NOT_GRANTED_MESSAGE = `This API key was made for another product, so it cannot run a set of journeys. Create a key in your Front account settings. ${GET_A_KEY}`; + +/** True for the 403 that PRODUCT_NOT_GRANTED_MESSAGE answers. */ +export function isProductNotGranted(error: unknown): boolean { + return ( + error instanceof StudioApiError && + error.status === 403 && + error.code === PRODUCT_NOT_GRANTED_CODE + ); +} + +/** `1 run`, `2 runs`. */ +function count(n: number, noun: string): string { + return `${n} ${noun}${n === 1 ? "" : "s"}`; +} + +/** `Dec 1`: en-US short month and day, read in UTC so the date is the same on every machine. */ +function resetDate(resetsAt: string | null): string | null { + if (resetsAt === null) return null; + const date = new Date(resetsAt); + if (Number.isNaN(date.getTime())) return null; + return date.toLocaleDateString("en-US", { month: "short", day: "numeric", timeZone: "UTC" }); +} + +/** ` (resets Dec 1)` or nothing, for the sentences that carry the reset date. */ +function resetClause(resetsAt: string | null): string { + const date = resetDate(resetsAt); + return date === null ? "" : ` (resets ${date})`; +} + +type GoPreview = Extract; + +export interface PreviewRender { + spec: SetSpec; + cells: readonly PlannedCell[]; + skipped: readonly SkippedCell[]; + preview: GoPreview; +} + +/** Levels per goal, when every goal runs the same number of them; null when they differ. */ +function uniformLevels(cells: readonly PlannedCell[]): { goals: number; levels: number } | null { + const perGoal = new Map(); + for (const cell of cells) perGoal.set(cell.goalIndex, (perGoal.get(cell.goalIndex) ?? 0) + 1); + const levels = new Set(perGoal.values()); + if (levels.size !== 1) return null; + return { goals: perGoal.size, levels: [...levels][0] }; +} + +function totalsSentence({ spec, cells, preview }: PreviewRender): string { + const { runs } = preview.totals; + const agents = count(preview.agents.length, "agent"); + const repeats = count(spec.repeat, "repeat"); + const shape = uniformLevels(cells); + const grid = shape + ? `${count(shape.goals, "goal")} x ${count(shape.levels, "level")}` + : `${count(cells.length, "goal and level pair")}`; + return `This set uses ${count(runs, "run")} (${grid} x ${agents} x ${repeats}).`; +} + +function allowanceSentence(allowance: GoPreview["allowance"]): string { + if (allowance.kind === "unlimited") return "Your plan has no monthly run limit."; + const left = `Your plan has ${count(allowance.remaining, "run")} left this month`; + return `${left}${resetClause(allowance.resetsAt)}.`; +} + +function trackedSentence(site: string, journeys: number): string { + return `It adds ${plain(site)} to your Front account's tracked sites if it is not there yet (with a daily ora score scan, which does not count against your plan) and ${count(journeys, "journey")} to your journeys list.`; +} + +/** The lines shown before the confirm prompt. The command prints the blank lines between groups. */ +export function renderPreview(input: PreviewRender): string[] { + const { spec, skipped, preview } = input; + const lines = [totalsSentence(input)]; + if (preview.defaultedAgent) { + const [agent] = preview.agents; + const why = + agent.harness === DEFAULT_HARNESS + ? "the Claude Code agent your plan offers" + : "the first one your plan offers, since it offers no Claude Code agent"; + lines.push( + `Agent: ${plain(agent.harnessName)} with ${plain(agent.modelName)}, ${why}. Choose others with --agents.`, + ); + } + if (spec.levels) { + lines.push(`Brand in the prompts: ${plain(spec.brand)}. Change it with --brand.`); + } + for (const skip of skipped) lines.push(plain(skip.note)); + lines.push(allowanceSentence(preview.allowance)); + lines.push(trackedSentence(spec.site, preview.totals.journeys)); + return lines; +} + +/** Printed once the record exists, with its real id. */ +export function closeTerminalLine(id: string): string { + return `You can close the terminal; \`ax journey view ${id}\` picks it up.`; +} + +/** The question asked before launching. Ends with a space so the answer types after it. */ +export function confirmPrompt(runs: number): string { + return `Launch ${count(runs, "run")}? [y/N] `; +} + +function overAllowanceLines(refusal: Extract): string[] { + const { runs, remaining, resetsAt, fits } = refusal; + const lines = [ + `This set needs ${count(runs, "run")} and your plan has ${count(remaining, "run")} left this month${resetClause(resetsAt)}.`, + ]; + if (fits === null) { + lines.push( + "Not even one repeat fits in what is left. Cut goals or agents, or wait for the reset.", + ); + return lines; + } + if (refusal.oneJourney) { + lines.push("1 run fits. Without --repeat, ax runs it as one journey and draws it live."); + return lines; + } + const verb = fits.repeat === 1 ? "fits" : "fit"; + lines.push( + `${count(fits.repeat, "repeat")} ${verb}: ${count(fits.runs, "run")}. Try --repeat ${fits.repeat}.`, + ); + return lines; +} + +/** One refusal in plain words, as one or more lines. */ +export function renderRefusal(refusal: Refusal): string[] { + switch (refusal.kind) { + case "missing_scopes": + return [`Your API key is missing these scopes: ${refusal.missing.join(", ")}.`, GET_A_KEY]; + case "unknown_agent": + return [ + `No agent is named ${plain(refusal.requested)}. --agents takes harness or harness/model.`, + ]; + case "unavailable_agent": { + const reason = refusal.reason ? plain(refusal.reason).trim().replace(/\.$/, "") : ""; + const why = reason ? `: ${reason}` : ""; + return [`The agent ${plain(refusal.requested)} is not available to your account${why}.`]; + } + case "duplicate_agent": + return [ + `${refusal.requested.map(plain).join(" and ")} name the same agent, ${plain(refusal.label)}. Name it once.`, + ]; + case "no_agent_offered": + return ["No agent is available to your account right now. Check your plan at app.ora.ai."]; + case "over_cap": + return [ + `This set would use ${count(refusal.runs, "run")}, and a set is capped at ${refusal.maxRuns}. Cut goals, agents or repeats.`, + ]; + case "over_allowance": + return overAllowanceLines(refusal); + } +} + +/** Every refusal, in the order the preview found them. */ +export function renderRefusals(refusals: readonly Refusal[]): string[] { + return refusals.flatMap(renderRefusal); +} + +/** `2 queued, 1 running, 0 finished`, with the slot clause when a run waits for one. */ +export function progressLine(progress: FollowProgress): string { + const line = `${progress.queued} queued, ${progress.running} running, ${progress.finished} finished`; + return progress.waitingForSlot ? `${line}; waiting for your plan's run slot` : line; +} + +/** True when the create that failed may have landed on the platform anyway. */ +function lastCreateMayHaveLanded(error: unknown): boolean { + if (error instanceof StudioApiError) return error.gateway; + return error instanceof StudioConnectionError || error instanceof StudioShapeError; +} + +function errorText(error: unknown): string { + return plain(error instanceof Error ? error.message : String(error)); +} + +function notLaunched(n: number): string { + return `${count(n, "run")} ${n === 1 ? "was" : "were"} not launched.`; +} + +const KEEPS_RUNNING = "Runs that started keep running."; + +/** + * Why launching stopped; empty for a launch that ran to the end. The command's Next lines name the + * commands to run (`view`, `cancel`), so no message here repeats them. After a quota stop the + * command goes on to follow what started. + */ +export function stopMessage(result: LaunchResult): string[] { + switch (result.stoppedReason) { + case undefined: + return []; + case "quota": + return [ + `Your plan's run limit was reached, so launching stopped. ${notLaunched(result.notLaunched)}`, + result.launchedRuns > 0 + ? `Following the ${count(result.launchedRuns, "run")} that started.` + : "No run started.", + ]; + case "platform_down": { + const lines = [`The platform is likely down, so launching stopped. ${KEEPS_RUNNING}`]; + if (lastCreateMayHaveLanded(result.lastError)) { + lines.push( + "The last journey may have started its runs anyway; they will show in your account's journeys list.", + ); + } + return lines; + } + case "api_error": + return [ + `The platform answered with an error, so launching stopped: ${errorText(result.lastError)}`, + `${notLaunched(result.notLaunched)} ${KEEPS_RUNNING}`, + ]; + case "interrupted": + return [`Launching stopped. ${notLaunched(result.notLaunched)} ${KEEPS_RUNNING}`]; + } +} + +/** + * After reads kept failing, the last error: the one thing the report's incomplete line cannot say. + * The command prints it on stderr, before the report or, with no report, before the view line. + */ +export function readFailureMessage(lastError?: unknown): string[] { + const lines = ["Reading from the platform kept failing."]; + if (lastError !== undefined) lines.push(`The last error: ${errorText(lastError)}`); + return lines; +} diff --git a/src/journey-set/render-report.test.ts b/src/journey-set/render-report.test.ts new file mode 100644 index 0000000..1aebc54 --- /dev/null +++ b/src/journey-set/render-report.test.ts @@ -0,0 +1,543 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import studioOraAi from "./__fixtures__/studio-trajectories/run-49ab0c88.json"; +import { type CountedRun, countSet, RATE_FLOOR, type SetShape } from "./count"; +import { deriveSetFlags, type SetStep } from "./flags"; +import type { FollowEnd } from "./follow"; +import { LEVEL_LABELS, type Level } from "./levels"; +import type { Bucket } from "./outcome"; +import { appendJourney, createRecord, markStopped, type StoppedReason } from "./record"; +import { NEED_LEVEL_NOTE, renderSetReport } from "./render-report"; +import type { ReportInput } from "./report-json"; +import { parseSetSpec, planCells } from "./spec"; +import blockedFirst403 from "./vendor/__fixtures__/run-flags/blocked-first-403.json"; + +// Every input is built in this file and fed through the production code: the spec through +// parseSetSpec and planCells, the record through createRecord and appendJourney in a temp +// directory, the counts through countSet. Runs are CountedRuns (the real type from count.ts) +// with a bucket named for what the case isolates. Flags come from deriveSetFlags over the +// fixtures count.test.ts uses (a blocked first request, and a run that stayed off linear.app). +// Server strings use the shapes outcome.test.ts uses: "timeout" is an excluded status, and +// "run gate unavailable" is Studio's start error (oramono apps/studio-service/src/journeys/ +// journeys-service.test.ts:629). + +const A = "claude-code/sonnet"; +const B = "codex/gpt-5"; +const FAILED_TO_START = "run gate unavailable"; + +const BLOCKED = deriveSetFlags( + blockedFirst403.trajectory.steps as SetStep[], + blockedFirst403.domain, +); +const NEVER_VISITED = deriveSetFlags(studioOraAi.steps, "linear.app"); + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ax-set-report-")); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +interface Scenario { + /** Defaults to true: most cases climb the ladder. */ + levels?: boolean; + goals?: string[]; + agents?: string[]; + repeat?: number; + docs?: string; + exact?: string; + runs: CountedRun[]; + stopped?: { reason: StoppedReason; notLaunched: number }; + ended?: FollowEnd; +} + +const NEED_TASK = "Find how to create an issue from the API"; +const BRAND_TASK = "Find how to create a Linear issue from the API"; + +function build(scenario: Scenario): ReportInput { + const agents = scenario.agents ?? [A]; + const parsed = parseSetSpec({ + site: "linear.app", + levels: scenario.levels ?? true, + goals: scenario.goals ?? [NEED_TASK], + agents: agents.join(","), + docs: scenario.docs, + exact: scenario.exact, + repeat: String(scenario.repeat ?? 1), + }); + if (!parsed.ok) throw new Error(parsed.error); + const { spec } = parsed; + const { cells } = planCells(spec); + let record = createRecord( + { + spec, + agents: agents.map((label) => { + const [harness, model] = label.split("/"); + return { harness, model }; + }), + cells, + }, + dir, + ); + for (const cell of cells) { + record = appendJourney( + record.id, + { + journeyId: `journey-${cell.goalIndex}-${cell.level}`, + goalIndex: cell.goalIndex, + level: cell.level, + repeat: 1, + runIds: [], + failed: [], + createdAt: record.createdAt, + }, + dir, + ); + } + if (scenario.stopped) { + record = markStopped(record.id, scenario.stopped.reason, scenario.stopped.notLaunched, dir); + } + const shape: SetShape = { + goals: spec.goals, + levelsByGoal: spec.goals.map((_, goalIndex) => + cells.filter((cell) => cell.goalIndex === goalIndex).map((cell) => cell.level), + ), + agents, + plannedPerCell: spec.repeat, + }; + return { + record, + count: countSet(scenario.runs, shape), + ended: scenario.ended ?? "settled", + }; +} + +function run(level: Level, bucket: Bucket, fields: Partial = {}): CountedRun { + return { goalIndex: 0, level, agent: A, repeat: 1, bucket, flags: null, ...fields }; +} + +/** `n` runs of one bucket in one cell, repeats numbered from 1. */ +function many( + level: Level, + bucket: Bucket, + n: number, + fields: Partial = {}, +): CountedRun[] { + return Array.from({ length: n }, (_, index) => + run(level, bucket, { repeat: index + 1, ...fields }), + ); +} + +// biome-ignore lint/suspicious/noControlCharactersInRegex: strips the color codes picocolors may add +const COLOR = /\u001b\[[0-9;]*m/g; + +function text(input: ReportInput): string { + return renderSetReport(input).join("\n").replace(COLOR, ""); +} + +function rowOf(report: string, label: string): string { + const row = report.split("\n").find((line) => line.trim().startsWith(label)); + if (!row) throw new Error(`no row ${label} in:\n${report}`); + return row; +} + +describe("a set without levels", () => { + it("prints one As written row per goal, with no level notes", () => { + const report = text( + build({ + levels: false, + goals: [NEED_TASK, BRAND_TASK], + runs: [run("written", "satisfied"), run("written", "unsatisfied", { goalIndex: 1 })], + }), + ); + const [first, second] = report.split("Goal 2"); + expect(first).toContain(`Goal 1: ${NEED_TASK}`); + expect(rowOf(first, "As written")).toContain("1 of 1"); + expect(second).toContain(BRAND_TASK); + expect(rowOf(second, "As written")).toContain("0 of 1"); + expect(report).not.toMatch(/skipped|Need only/); + expect(report).toContain("journey-1-written Goal 2, As written, repeat 1"); + }); +}); + +describe("the matrix", () => { + it("lays out a mixed set by goal, level and agent, with every bucket and flag", () => { + const input = build({ + goals: [NEED_TASK, BRAND_TASK], + agents: [A, B], + repeat: 5, + exact: "https://linear.app/developers", + runs: [ + // Goal 1, need: A has 5 gradable runs; B has 2 gradable and one of each other bucket. + ...many("need", "satisfied", 1), + ...many("need", "unsatisfied", 4, { repeat: 2 }), + run("need", "partial", { agent: B, repeat: 1 }), + run("need", "unsatisfied", { agent: B, repeat: 2 }), + run("need", "excluded", { agent: B, repeat: 3, rawStatus: "timeout" }), + run("need", "excluded", { agent: B, repeat: 4, reason: FAILED_TO_START }), + run("need", "not_graded_yet", { agent: B, repeat: 5 }), + // Goal 1, brand: A is all satisfied, one run blocked and one never visited; B has no runs. + ...many("brand", "satisfied", 3), + run("brand", "satisfied", { repeat: 4, flags: BLOCKED }), + run("brand", "satisfied", { repeat: 5, flags: NEVER_VISITED }), + // Goal 2 has no need level (it names the brand). + ...many("brand", "satisfied", 2, { goalIndex: 1 }), + ], + }); + const report = text(input); + expect(report).toContain("linear.app"); + expect(report).toContain(`set ${input.record.id}`); + expect(report).toContain(`Goal 1: ${NEED_TASK}`); + expect(report).toContain(`Goal 2: ${BRAND_TASK}`); + expect(report).toMatch(/Level\s+claude-code\/sonnet\s+codex\/gpt-5/); + const need = report.split("Goal 2")[0]; + expect(rowOf(need, "Need only")).toContain("20% (1 of 5)"); + expect(need).toContain("0 of 2"); + expect(need).toContain("excluded: timeout (1)"); + expect(need).toContain(`never started: ${FAILED_TO_START} (1)`); + expect(need).toContain("not graded yet 1"); + expect(rowOf(need, "Brand named")).toContain("100% (5 of 5)"); + expect(need).toContain("blocked 1"); + expect(need).toContain("never visited 1"); + const second = report.split("Goal 2")[1].split("Journeys")[0]; + expect(second).not.toMatch(/^\s+Need only\s{2,}/m); + expect(rowOf(second, "Brand named")).toContain("2 of 2"); + }); + + it("shows counts below the gradable floor and a percent from it", () => { + const below = text( + build({ + repeat: RATE_FLOOR, + runs: [...many("need", "satisfied", 3), run("need", "unsatisfied", { repeat: 4 })], + }), + ); + expect(rowOf(below, "Need only")).toContain("3 of 4"); + expect(rowOf(below, "Need only")).not.toContain("%"); + const at = text( + build({ + repeat: RATE_FLOOR, + runs: [...many("need", "satisfied", 3), ...many("need", "unsatisfied", 2, { repeat: 4 })], + }), + ); + expect(rowOf(at, "Need only")).toContain("60% (3 of 5)"); + }); + + it("rounds the rate to a whole percent", () => { + const report = text( + build({ + repeat: 6, + runs: [...many("need", "satisfied", 4), ...many("need", "unsatisfied", 2, { repeat: 5 })], + }), + ); + expect(rowOf(report, "Need only")).toContain("67% (4 of 6)"); + }); + + it("prints a hyphen for a cell with no data", () => { + const report = text(build({ agents: [A, B], runs: [run("need", "satisfied")] })); + expect(rowOf(report, "Need only")).toMatch(/1 of 1\s+-$/); + }); + + it("marks a level that launched fewer runs than planned", () => { + const report = text( + build({ + agents: [A, B], + repeat: 2, + runs: [ + run("need", "satisfied"), + run("need", "satisfied", { repeat: 2 }), + run("need", "satisfied", { agent: B }), + run("need", "unsatisfied", { agent: B, repeat: 2 }), + run("brand", "satisfied"), + ], + }), + ); + expect(report).toContain("Fewer runs than planned: Brand named (1 of 4)."); + expect(report).not.toContain("Need only ("); + }); + + it("says excluded runs never count against the site, only when some were excluded", () => { + const some = text(build({ runs: [run("need", "excluded", { rawStatus: "failed" })] })); + expect(some).toContain( + "Excluded runs never count against linear.app, although the platform's own verdict counts some of them as failed.", + ); + const none = text(build({ runs: [run("need", "satisfied")] })); + expect(none).not.toContain("Excluded runs never count"); + }); + + it("tallies not gradable runs by their raw verdict and in flight runs", () => { + const report = text( + build({ + runs: [ + run("need", "not_gradable", { rawVerdict: "inconclusive" }), + run("need", "in_flight", { repeat: 2 }), + ], + ended: "wait_limit", + }), + ); + expect(report).toContain("not gradable: inconclusive (1)"); + expect(report).toContain("in flight 1"); + }); +}); + +describe("the need level note", () => { + it("prints once, only when the need level ran", () => { + const ran = text(build({ runs: [run("need", "satisfied")] })); + expect(ran.split(NEED_LEVEL_NOTE)).toHaveLength(2); + const skipped = text(build({ goals: [BRAND_TASK], runs: [run("brand", "satisfied")] })); + expect(skipped).not.toContain(NEED_LEVEL_NOTE); + }); + + it("claims nothing about how the platform judges", () => { + expect(NEED_LEVEL_NOTE).not.toMatch(/judge|credit|counts as/i); + expect(NEED_LEVEL_NOTE).not.toMatch(/[\u2013\u2014]/); + }); +}); + +describe("gap lines", () => { + it("names the Discovery layer for a gap that closes at the brand level", () => { + const report = text( + build({ + repeat: 5, + runs: [...many("need", "unsatisfied", 5), ...many("brand", "satisfied", 5)], + }), + ); + expect(report).toContain( + "linear.app loses agents at the Discovery layer: 0 of 5 runs satisfied at Need only, 5 of 5 at Brand named.", + ); + expect(report).not.toContain("early signal"); + expect(report).not.toMatch(/Agent \d|claude-code\/sonnet (loses|did)/); + }); + + it("names the Access and Usability layers for a gap that closes at the exact level", () => { + const report = text( + build({ + goals: [BRAND_TASK], + repeat: 5, + exact: "https://linear.app/developers", + runs: [...many("brand", "unsatisfied", 5), ...many("exact", "satisfied", 5)], + }), + ); + expect(report).toContain( + "linear.app loses agents at the Access and Usability layers: 0 of 5 runs satisfied at Brand named, 5 of 5 at Exact target.", + ); + expect(report).not.toContain("the Access and Usability layer:"); + }); + + it("names Discovery, singular, for the docs level", () => { + const report = text( + build({ + goals: [BRAND_TASK], + repeat: 5, + docs: "https://linear.app/docs", + runs: [...many("brand", "unsatisfied", 5), ...many("docs", "satisfied", 5)], + }), + ); + expect(report).toContain("loses agents at the Discovery layer:"); + expect(report).not.toContain("Discovery layers"); + }); + + it("reads a negative difference as worse with help, with both counts and no layer", () => { + const report = text( + build({ + repeat: 5, + runs: [...many("need", "satisfied", 5), ...many("brand", "unsatisfied", 5)], + }), + ); + expect(report).toContain( + "On linear.app, agents did worse with the help at Brand named than at Need only: 5 of 5 runs satisfied at Need only, 0 of 5 at Brand named.", + ); + expect(report).not.toContain("loses agents"); + expect(report).not.toContain("layer"); + }); + + it("tags an early signal when either level has fewer than the floor of gradable runs", () => { + const report = text( + build({ + repeat: 2, + runs: [...many("need", "unsatisfied", 2), ...many("brand", "satisfied", 2)], + }), + ); + expect(report).toContain( + `(early signal: fewer than ${RATE_FLOOR} gradable runs at one of these levels)`, + ); + }); + + it("sums over agents, so a gap line never names one", () => { + const report = text( + build({ + agents: [A, B], + repeat: 3, + runs: [ + ...many("need", "unsatisfied", 3), + ...many("need", "satisfied", 1, { agent: B }), + ...many("need", "unsatisfied", 2, { agent: B, repeat: 2 }), + ...many("brand", "satisfied", 3), + ...many("brand", "satisfied", 3, { agent: B }), + ], + }), + ); + const gap = report.split("\n").find((line) => line.includes("loses agents")); + expect(gap).toContain("1 of 6 runs satisfied at Need only, 6 of 6 at Brand named"); + expect(gap).not.toMatch(/claude-code|codex/); + }); + + it("prints a gap under its own goal", () => { + const report = text( + build({ + goals: [NEED_TASK, "Plan a sprint with the API"], + repeat: 5, + runs: [ + ...many("need", "unsatisfied", 5), + ...many("brand", "satisfied", 5), + ...many("need", "satisfied", 5, { goalIndex: 1 }), + ...many("brand", "satisfied", 5, { goalIndex: 1 }), + ], + }), + ); + const [first, second] = report.split("Goal 2"); + expect(first).toContain("loses agents"); + expect(second).not.toContain("loses agents"); + }); +}); + +describe("journeys and skipped levels", () => { + it("lists journey ids as plain ids with no links", () => { + const report = text( + build({ + exact: "https://linear.app/developers", + runs: [run("need", "satisfied")], + }), + ); + expect(report).toContain("Journeys"); + expect(report).toMatch(/journey-0-need\s+Goal 1, Need only, repeat 1/); + expect(report).toMatch(/journey-0-exact\s+Goal 1, Exact target, repeat 1/); + expect(report).not.toMatch(/https?:\/\/(?!linear\.app)/); + }); + + it("prints each skip note under its goal, verbatim", () => { + const report = text(build({ goals: [BRAND_TASK], runs: [run("brand", "satisfied")] })); + expect(report).toContain( + "The goal names Linear, so the need level is skipped. Write the goal without it to include that level.", + ); + expect(report).toContain( + "No --docs URL, so the docs level is skipped. Add --docs to include it.", + ); + }); +}); + +describe("complete and the incomplete line", () => { + it("is complete when launch ran, following settled and nothing is in flight or ungraded", () => { + const input = build({ runs: [run("need", "satisfied")] }); + expect(text(input)).not.toContain("not complete"); + }); + + it("is incomplete after a quota stop, and says the plan's run limit and how many runs never started", () => { + const input = build({ + runs: [run("need", "satisfied")], + stopped: { reason: "quota", notLaunched: 7 }, + }); + expect(text(input)).toContain( + "This report is not complete: launching stopped at your plan's run limit. 7 runs never started.", + ); + }); + + it("is incomplete at the wait limit, and says how many runs are still going", () => { + const input = build({ + runs: [run("need", "satisfied"), run("need", "in_flight", { repeat: 2 })], + repeat: 2, + ended: "wait_limit", + }); + expect(text(input)).toContain( + "This report is not complete: the wait limit ran out with 1 run still going.", + ); + }); + + it("is incomplete when a grade is pending, even if following said settled", () => { + const input = build({ runs: [run("need", "not_graded_yet")] }); + expect(text(input)).toContain("This report is not complete:"); + }); + + it.each<[FollowEnd, string]>([ + ["settle_limit", "no grade yet and the wait for grading ran out."], + ["detached", "you stopped following before every run finished."], + ["read_failed", "the platform could not be read, so some runs may be missing."], + ])("says why when following ended %s", (ended, why) => { + const input = build({ runs: [run("need", "not_graded_yet")], ended }); + expect(text(input)).toContain(why); + }); + + it.each<[StoppedReason, string]>([ + ["platform_down", "launching stopped because the platform did not answer."], + ["api_error", "launching stopped on an error from the platform."], + ["interrupted", "launching was interrupted."], + ])("says why when launching stopped with %s", (reason, why) => { + const input = build({ + runs: [run("need", "satisfied")], + stopped: { reason, notLaunched: 0 }, + }); + expect(text(input)).toContain(why); + }); + + it("puts the stop reason ahead of the follow ending", () => { + const input = build({ + runs: [run("need", "in_flight")], + stopped: { reason: "quota", notLaunched: 1 }, + ended: "wait_limit", + }); + const line = text(input) + .split("\n") + .find((l) => l.includes("not complete")); + expect(line).toContain("run limit"); + expect(line).not.toContain("wait limit"); + }); +}); + +describe("server strings", () => { + const ESCAPE = "\u001b[2J"; + const EVIL_AGENT = `evil/mod${ESCAPE}el`; + + function hostile(): ReportInput { + return build({ + agents: [EVIL_AGENT], + runs: [ + run("need", "excluded", { agent: EVIL_AGENT, rawStatus: `time${ESCAPE}out` }), + run("need", "excluded", { agent: EVIL_AGENT, repeat: 2, reason: `gate${ESCAPE}down` }), + run("need", "not_gradable", { + agent: EVIL_AGENT, + repeat: 3, + rawVerdict: `odd${ESCAPE}verdict`, + }), + ], + }); + } + + it("strips terminal control sequences in the text form", () => { + const report = text(hostile()); + expect(report).not.toContain("\u001b"); + expect(report).toContain("mod [2Jel"); + expect(report).toContain("excluded: time [2Jout (1)"); + expect(report).toContain("never started: gate [2Jdown (1)"); + expect(report).toContain("not gradable: odd [2Jverdict (1)"); + }); +}); + +describe("labels", () => { + it("prints the level labels the ladder pins", () => { + const report = text( + build({ + repeat: 1, + docs: "https://linear.app/docs", + exact: "https://linear.app/developers", + runs: [run("need", "satisfied")], + }), + ); + for (const level of ["need", "brand", "docs", "exact"] as const) { + expect(report).toContain(LEVEL_LABELS[level]); + } + }); +}); diff --git a/src/journey-set/render-report.ts b/src/journey-set/render-report.ts new file mode 100644 index 0000000..0d173e0 --- /dev/null +++ b/src/journey-set/render-report.ts @@ -0,0 +1,236 @@ +// The finished set as text: one matrix per goal (levels down, agents across), the gap lines +// between levels, and the journey ids. Everything here is arithmetic count.ts already did, laid +// out for a terminal; it derives no grade and ranks no agent. Server strings (agent labels, raw +// statuses, raw verdicts, never started reasons, journey ids) go through plain() on the way out. +// The Next line is not built here: the command renders it with axCall. The JSON form lives in +// report-json.ts and keeps raw values. + +import pc from "picocolors"; +import { plain, squeeze } from "../ui/ansi"; +import { type CellCount, type GapLine, RATE_FLOOR, type Tally } from "./count"; +import { LEVEL_LABELS } from "./levels"; +import type { StoppedReason } from "./record"; +import { + agentsOf, + isComplete, + type ReportInput, + runsAwaitingGrade, + runsInFlight, +} from "./report-json"; +import { planCells } from "./spec"; + +/** + * What satisfied means at the need level. Provisional: it states only what the prompt does, and + * claims nothing about how the platform judges a run that used another product. The first real + * need level run grounds it (plan M9.2). + */ +export const NEED_LEVEL_NOTE = + "Need only prompts do not name the site, so an agent can reach the goal without using it. Satisfied there is the platform's verdict on the goal."; + +const GOAL_TEXT_MAX = 120; +const GAP = " "; + +/** `3 of 4` below RATE_FLOOR gradable runs, `75% (3 of 4)` from it, a hyphen with none. */ +function headline(cell: CellCount): string { + if (cell.gradable === 0) return "-"; + const counts = `${cell.satisfied} of ${cell.gradable}`; + return cell.rate === null ? counts : `${Math.round(cell.rate * 100)}% (${counts})`; +} + +/** `excluded: failed (1), timeout (2)`; the values are server strings. */ +function tallyLine(label: string, tally: Tally, empty: string): string { + const parts = tally.map((entry) => `${plain(entry.value).trim() || empty} (${entry.count})`); + return `${label}: ${parts.join(", ")}`; +} + +/** The lines under a cell's headline: counts, never warnings. Zero counts are left out. */ +function detailLines(cell: CellCount): string[] { + const lines: string[] = []; + const flags = [ + cell.flags.blocked > 0 ? `blocked ${cell.flags.blocked}` : "", + cell.flags.never_visited > 0 ? `never visited ${cell.flags.never_visited}` : "", + ].filter(Boolean); + if (flags.length > 0) lines.push(flags.join(", ")); + if (cell.excluded.length > 0) lines.push(tallyLine("excluded", cell.excluded, "no status")); + if (cell.neverStarted.length > 0) { + lines.push(tallyLine("never started", cell.neverStarted, "no reason given")); + } + if (cell.notGradedYet > 0) lines.push(`not graded yet ${cell.notGradedYet}`); + if (cell.notGradable.length > 0) { + lines.push(tallyLine("not gradable", cell.notGradable, "no value")); + } + if (cell.inFlight > 0) lines.push(`in flight ${cell.inFlight}`); + return lines; +} + +/** Columns padded on plain text; color goes on whole padded segments, so widths never skew. */ +function table(header: string[], rows: string[][][]): string[] { + const widths = header.map((title, col) => + Math.max(title.length, ...rows.map((row) => Math.max(...row[col].map((line) => line.length)))), + ); + const line = (segments: string[], paint: (value: string) => string) => + ` ${segments.map((text, col) => paint(text.padEnd(widths[col]))).join(GAP)}`.trimEnd(); + const out = [line(header, pc.bold)]; + for (const row of rows) { + const height = Math.max(...row.map((cell) => cell.length)); + for (let index = 0; index < height; index++) { + out.push( + line( + row.map((cell) => cell[index] ?? ""), + index === 0 ? (v) => v : pc.dim, + ), + ); + } + } + return out; +} + +function levelRows(cells: readonly CellCount[], agents: readonly string[]): string[][][] { + const levels = [...new Set(cells.map((cell) => cell.level))]; + return levels.map((level) => [ + [LEVEL_LABELS[level]], + ...agents.map((agent) => { + const cell = cells.find((c) => c.level === level && c.agent === agent); + return cell ? [headline(cell), ...detailLines(cell)] : ["-"]; + }), + ]); +} + +/** The layer a gap sits at; `ax audit` prints Access and Usability as two layers. */ +function layerPhrase(layer: GapLine["layer"]): string { + return layer === "Access and Usability" + ? "the Access and Usability layers" + : `the ${layer} layer`; +} + +function counts(total: { satisfied: number; gradable: number }): string { + return `${total.satisfied} of ${total.gradable}`; +} + +/** About the site, never an agent: positive loses agents at a layer, negative claims no layer. */ +function gapLine(gap: GapLine, site: string): string { + const lower = LEVEL_LABELS[gap.lower.level]; + const upper = LEVEL_LABELS[gap.upper.level]; + const early = gap.earlySignal + ? ` (early signal: fewer than ${RATE_FLOOR} gradable runs at one of these levels)` + : ""; + if (gap.difference > 0) { + return `${site} loses agents at ${layerPhrase(gap.layer)}: ${counts(gap.lower)} runs satisfied at ${lower}, ${counts(gap.upper)} at ${upper}.${early}`; + } + return `On ${site}, agents did worse with the help at ${upper} than at ${lower}: ${counts(gap.lower)} runs satisfied at ${lower}, ${counts(gap.upper)} at ${upper}.${early}`; +} + +function shortLine(input: ReportInput, goalIndex: number): string[] { + const short = input.count.shortLevels.filter((level) => level.goalIndex === goalIndex); + if (short.length === 0) return []; + const parts = short.map( + (level) => `${LEVEL_LABELS[level.level]} (${level.launched} of ${level.planned})`, + ); + return [pc.dim(` Fewer runs than planned: ${parts.join(", ")}.`)]; +} + +function goalBlock(input: ReportInput, goalIndex: number, skipped: string[]): string[] { + const cells = input.count.cells.filter((cell) => cell.goalIndex === goalIndex); + const text = squeeze(plain(input.record.spec.goals[goalIndex] ?? ""), GOAL_TEXT_MAX); + const lines = ["", pc.bold(`Goal ${goalIndex + 1}: ${text}`)]; + lines.push( + ...table( + ["Level", ...agentsOf(input.count).map(plain)], + levelRows(cells, agentsOf(input.count)), + ), + ); + lines.push(...shortLine(input, goalIndex)); + for (const gap of input.count.gaps.filter((g) => g.goalIndex === goalIndex)) { + lines.push("", ` ${gapLine(gap, input.record.spec.site)}`); + } + for (const note of skipped) lines.push(pc.dim(` ${note}`)); + return lines; +} + +function journeyLines(input: ReportInput): string[] { + const listed = input.record.journeys.flatMap((entry) => + entry.journeyId === null ? [] : [{ ...entry, journeyId: plain(entry.journeyId) }], + ); + if (listed.length === 0) return []; + const width = Math.max(...listed.map((entry) => entry.journeyId.length)); + return [ + "", + pc.bold("Journeys"), + ...listed.map( + (entry) => + ` ${entry.journeyId.padEnd(width)} Goal ${entry.goalIndex + 1}, ${LEVEL_LABELS[entry.level]}, repeat ${entry.repeat}`, + ), + ]; +} + +function hasExcluded(input: ReportInput): boolean { + return input.count.cells.some((cell) => cell.excluded.length > 0 || cell.neverStarted.length > 0); +} + +function ranNeedLevel(input: ReportInput): boolean { + return input.count.cells.some((cell) => cell.level === "need"); +} + +function stopReason(reason: StoppedReason, notLaunched: number | undefined): string { + const left = notLaunched + ? ` ${notLaunched} ${notLaunched === 1 ? "run" : "runs"} never started.` + : ""; + switch (reason) { + case "quota": + return `launching stopped at your plan's run limit.${left}`; + case "platform_down": + return `launching stopped because the platform did not answer.${left}`; + case "api_error": + return `launching stopped on an error from the platform.${left}`; + case "interrupted": + return `launching was interrupted.${left}`; + } +} + +function followReason(input: ReportInput): string { + const going = runsInFlight(input.count); + const ungraded = runsAwaitingGrade(input.count); + switch (input.ended) { + case "settle_limit": + return `${ungraded} ${ungraded === 1 ? "run has" : "runs have"} no grade yet and the wait for grading ran out.`; + case "wait_limit": + return `the wait limit ran out with ${going} ${going === 1 ? "run" : "runs"} still going.`; + case "detached": + return "you stopped following before every run finished."; + case "read_failed": + return "the platform could not be read, so some runs may be missing."; + case "settled": + return `${going + ungraded} ${going + ungraded === 1 ? "run is" : "runs are"} still going or waiting for a grade.`; + } +} + +/** One line: the stop reason when launching stopped, otherwise how following ended. */ +function incompleteLine(input: ReportInput): string { + const { stoppedReason, notLaunched } = input.record; + const why = stoppedReason ? stopReason(stoppedReason, notLaunched) : followReason(input); + return `This report is not complete: ${why}`; +} + +/** The report for stdout, one string per line. Prints nothing itself. */ +export function renderSetReport(input: ReportInput): string[] { + const { record } = input; + const site = record.spec.site; + const skipped = planCells(record.spec).skipped; + const lines = [`${pc.bold(site)} ${pc.dim(`set ${record.id}`)}`]; + for (const goalIndex of record.spec.goals.keys()) { + const notes = skipped.filter((cell) => cell.goalIndex === goalIndex).map((cell) => cell.note); + lines.push(...goalBlock(input, goalIndex, notes)); + } + if (hasExcluded(input)) { + lines.push( + "", + `Excluded runs never count against ${site}, although the platform's own verdict counts some of them as failed.`, + ); + } + if (ranNeedLevel(input)) lines.push("", NEED_LEVEL_NOTE); + if (!isComplete(input)) { + lines.push("", pc.yellow(incompleteLine(input))); + } + lines.push(...journeyLines(input)); + return lines; +} diff --git a/src/journey-set/report-flow.test.ts b/src/journey-set/report-flow.test.ts new file mode 100644 index 0000000..8d0a612 --- /dev/null +++ b/src/journey-set/report-flow.test.ts @@ -0,0 +1,203 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import quota402 from "../api/__fixtures__/studio/error-402-quota.json"; +import { PlatformApiError } from "../api/platform"; +import { createStudioClient, StudioConnectionError, StudioShapeError } from "../api/studio"; +import { studioError } from "../api/studio-errors"; +import { createRecord, markStopped, readRecord, type SetRecord } from "./record"; +import { + cancelStep, + failureCode, + followAndReport, + launchAndReport, + pickUpRecord, + viewStep, + withInterrupt, +} from "./report-flow"; +import { parseSetSpec, planCells } from "./spec"; + +// The path from a written record to its report, below the commands. The full paths (launch, +// follow, every exit) run in src/commands/set.test.ts; these cases pin the pieces that test +// reaches only indirectly. The client is a real createStudioClient over a mocked fetch; error +// bodies are the fixtures in src/api/__fixtures__/studio/ (provenance in studio.test.ts). + +const BASE = "https://platform.test"; +const SITE = "linear.app"; +const AGENTS = [ + { harness: "claude-code", model: "claude-haiku-4-5-20251001" }, + { harness: "claude-code", model: "claude-sonnet-5-5" }, +]; +const RUN_CALL = "ax journey 'Create an issue' --domain linear.app --levels --yes"; + +const asJson = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); + +/** Answers the key exchange and each create from `creates` in turn; any other call fails loudly. */ +function studioMock(creates: Array<() => Response> = []) { + const calls: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string | URL, init: RequestInit = {}) => { + const at = String(url).slice(BASE.length); + if (at === "/auth/token") return asJson({ token: "bearer" }); + calls.push(`${init.method ?? "GET"} ${at}`); + const reply = at === "/studio/v1/journeys" ? creates[calls.length - 1] : undefined; + if (!reply) throw new Error(`unrouted ${init.method} ${at}`); + return reply(); + }), + ); + return { calls }; +} + +const client = () => createStudioClient({ apiKey: "ora_sk_local", baseUrl: BASE }); + +let dir: string; + +function newSet(): SetRecord { + const parsed = parseSetSpec({ + site: SITE, + goals: ["Create an issue"], + agents: "claude-code", + levels: true, + }); + if (!parsed.ok) throw new Error(parsed.error); + const { cells } = planCells(parsed.spec); + return createRecord({ spec: parsed.spec, agents: AGENTS, cells }, dir); +} + +const flow = { waitMinutes: 30, json: false, now: () => 0, sleep: async () => {} }; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ax-report-flow-")); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + rmSync(dir, { recursive: true, force: true }); +}); + +const logged = () => vi.mocked(console.log).mock.calls.map((c) => String(c[0])); +const errored = () => vi.mocked(console.error).mock.calls.map((c) => String(c[0])); + +describe("withInterrupt", () => { + it("aborts the phase's signal on Ctrl-C and takes its listener off after", async () => { + const before = process.listenerCount("SIGINT"); + const aborted = await withInterrupt(async (signal) => { + expect(process.listenerCount("SIGINT")).toBe(before + 1); + process.emit("SIGINT"); + return signal.aborted; + }); + expect(aborted).toBe(true); + expect(process.listenerCount("SIGINT")).toBe(before); + }); + + it("takes its listener off when the phase throws", async () => { + const before = process.listenerCount("SIGINT"); + await expect( + withInterrupt(async () => { + throw new RangeError("a bug in the phase"); + }), + ).rejects.toThrow(RangeError); + expect(process.listenerCount("SIGINT")).toBe(before); + }); +}); + +describe("failureCode", () => { + it("is Studio's code, or the CLI's own for a failure that carries none", () => { + expect( + failureCode(studioError({ status: 500, payload: { error: "boom", code: "internal" } })), + ).toBe("internal"); + expect(failureCode(studioError({ status: 502, payload: null }))).toBe("API_ERROR"); + expect(failureCode(new StudioConnectionError(BASE, new TypeError("fetch failed")))).toBe( + "API_UNREACHABLE", + ); + expect(failureCode(new StudioShapeError("GET /studio/v1/usage", "a runs meter"))).toBe( + "API_BAD_ANSWER", + ); + const rejected = new PlatformApiError("ora rejected the API key (401). Check ORA_API_KEY.", { + status: 401, + payload: null, + }); + expect(failureCode(rejected)).toBe("API_ERROR"); + }); +}); + +describe("viewStep and cancelStep", () => { + it("name the set and carry --json when it was given", () => { + expect(viewStep("j-0123456789", false).call).toBe("ax journey view j-0123456789"); + expect(viewStep("j-0123456789", true).call).toBe("ax journey view j-0123456789 --json"); + expect(cancelStep("j-0123456789", true).call).toBe("ax journey cancel j-0123456789 --json"); + }); +}); + +describe("pickUpRecord", () => { + it("reads the record back and prints nothing", () => { + const record = newSet(); + expect(pickUpRecord(record.id, false, dir)).toEqual(record); + expect(errored()).toEqual([]); + }); +}); + +describe("followAndReport", () => { + it("reports a set with no journey without a read, the stop on its incomplete line", async () => { + const record = markStopped(newSet().id, "quota", 2, dir); + const mock = studioMock(); + + expect(await followAndReport({ ...flow, client: client(), record })).toBe("reported"); + + expect(mock.calls).toEqual([]); + expect(logged()).toContain( + "This report is not complete: launching stopped at your plan's run limit. 2 runs never started.", + ); + expect(logged().at(-1)).toMatch(/^Next: ax audit linear\.app {2}# /); + }); +}); + +describe("launchAndReport", () => { + const launchInput = (record: SetRecord) => ({ + ...flow, + client: client(), + record, + agents: AGENTS, + runCall: RUN_CALL, + dir, + }); + + it("a gateway 502 on the first create: no journey, so Next is the run again", async () => { + const record = newSet(); + studioMock([() => new Response("bad gateway", { status: 502 })]); + + expect(await launchAndReport(launchInput(record))).toBe("failed"); + + expect(readRecord(record.id, dir)).toMatchObject({ + journeys: [], + stoppedReason: "platform_down", + }); + expect(errored().at(-1)).toBe(`Next: ${RUN_CALL} # launches the same set again`); + expect(errored().join("\n")).not.toContain("ax journey cancel"); + expect(logged()).toEqual([]); + }); + + it("a 402 on the first create says no run started, then reports, with no read", async () => { + const record = newSet(); + const mock = studioMock([() => asJson(quota402, 402)]); + + expect(await launchAndReport(launchInput(record))).toBe("reported"); + + // Two levels (need, brand) times two agents, and none of the four started. + expect(mock.calls).toEqual(["POST /studio/v1/journeys"]); + expect(errored()).toContain( + "Your plan's run limit was reached, so launching stopped. 4 runs were not launched.", + ); + expect(errored()).toContain("No run started."); + expect(logged()).toContain( + "This report is not complete: launching stopped at your plan's run limit. 4 runs never started.", + ); + }); +}); diff --git a/src/journey-set/report-flow.ts b/src/journey-set/report-flow.ts new file mode 100644 index 0000000..87617fe --- /dev/null +++ b/src/journey-set/report-flow.ts @@ -0,0 +1,287 @@ +import { join } from "node:path"; +import { StudioApiError, StudioConnectionError, StudioShapeError } from "../api/studio"; +import type { StudioAgentPair, StudioClient } from "../api/studio-types"; +import { reportError } from "../report/json-error"; +import { axCall, type NextStep, placeholder, printNext } from "../report/next-line"; +import { plain } from "../ui/ansi"; +import { collectSet } from "./collect"; +import { countSet, type SetCount } from "./count"; +import { type FollowInput, followJourneys } from "./follow"; +import { type LaunchResult, launchSet } from "./launch"; +import { + CorruptRecordError, + DEFAULT_SETS_DIR, + isSetId, + readRecord, + type SetRecord, +} from "./record"; +import { progressLine, readFailureMessage, stopMessage } from "./render-preview"; +import { renderSetReport } from "./render-report"; +import { reportJson, runsAwaitingGrade, runsInFlight } from "./report-json"; + +// A set from its written record to its report, the path a set takes once the person said yes +// and `ax journey view` takes from a record id: pick the record up (view only), launch (run +// only), follow the recorded journeys, collect and count what they read, and print the report or +// the --json document with its Next line. Ctrl-C is wired per phase. Lines for a person go to +// stderr; stdout carries only the report or the document. Exit codes stay with the commands. + +/** `reported`: a report printed. `interrupted`: Ctrl-C. `failed`: the platform, and no report. */ +export type FlowEnd = "reported" | "interrupted" | "failed"; + +export interface FollowFlowInput { + client: Pick; + /** The record as it is on disk: its journeys and any launch stop. */ + record: SetRecord; + waitMinutes: number; + json: boolean; + now: () => number; + sleep: FollowInput["sleep"]; +} + +export interface LaunchFlowInput extends Omit { + client: Pick; + /** The resolved agents; every create sends all of them. */ + agents: readonly StudioAgentPair[]; + /** The run command as typed: the Next line when launching stopped before any journey. */ + runCall: string; + /** The record directory; tests pass a temp one. */ + dir?: string; +} + +const DETACHED = "Stopped following. Runs that started keep running."; + +/** Runs whose steps could not be read carry no flags, so the report would undercount them. */ +function unflaggedLine(runs: number): string { + const noun = runs === 1 ? "1 run" : `${runs} runs`; + return `The steps of ${noun} could not be read, so the report does not flag them as blocked or never visited.`; +} + +function jsonWords(json: boolean): string[] { + return json ? ["--json"] : []; +} + +/** A set of journeys in its general form, for a Next line that has no typed call to repeat. */ +export const SET_USAGE_CALL = axCall([ + "journey", + placeholder("goal"), + "--domain", + placeholder("site"), + "--levels", +]); + +export function viewStep(id: string, json: boolean): NextStep { + return { + call: axCall(["journey", "view", id, ...jsonWords(json)]), + why: "picks the set up and prints its report", + }; +} + +export function cancelStep(id: string, json: boolean): NextStep { + return { + call: axCall(["journey", "cancel", id, ...jsonWords(json)]), + why: "stops the runs that started", + }; +} + +/** + * The --json envelope's code for one of the platform's failures, for every set command: the + * platform's own code when it sent one, else `API_ERROR` as the other commands use, and a code + * each for no connection and a malformed answer. + */ +export function failureCode(error: unknown): string { + if (error instanceof StudioConnectionError) return "API_UNREACHABLE"; + if (error instanceof StudioShapeError) return "API_BAD_ANSWER"; + if (error instanceof StudioApiError && error.code) return error.code; + return "API_ERROR"; +} + +/** + * Run one phase with Ctrl-C wired to its signal. The listener is added once, so a second Ctrl-C + * finds none and gets Node's default exit, and it comes off when the phase ends either way. + */ +export async function withInterrupt(phase: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + const onSignal = () => controller.abort(); + process.once("SIGINT", onSignal); + try { + return await phase(controller.signal); + } finally { + process.removeListener("SIGINT", onSignal); + } +} + +/** One stderr line that changes as a phase runs: repainted on a terminal, printed on change otherwise. */ +function statusLine(): { show(text: string): void; end(): void } { + const live = Boolean(process.stderr.isTTY); + let last = ""; + return { + show(text) { + if (text === last) return; + last = text; + if (live) process.stderr.write(`\r\x1b[K${text}`); + else console.error(text); + }, + end() { + if (live && last) process.stderr.write("\n"); + last = ""; + }, + }; +} + +async function withStatus(phase: (show: (text: string) => void) => Promise): Promise { + const status = statusLine(); + try { + return await phase((text) => status.show(text)); + } finally { + status.end(); + } +} + +/** View while runs are still going or awaiting a grade, which a later view picks up; otherwise the audit. */ +function reportNext(record: SetRecord, count: SetCount, json: boolean): NextStep { + if (runsInFlight(count) + runsAwaitingGrade(count) > 0) return viewStep(record.id, json); + return { + call: axCall(["audit", record.spec.site, ...jsonWords(json)]), + why: "the readiness audit: which checks explain what the agents hit", + }; +} + +function follow(input: FollowFlowInput) { + const journeyIds = input.record.journeys.flatMap((entry) => + entry.journeyId === null ? [] : [entry.journeyId], + ); + return withStatus((show) => + withInterrupt((signal) => + followJourneys({ + client: input.client, + journeyIds, + waitMinutes: input.waitMinutes, + signal, + now: input.now, + sleep: input.sleep, + onProgress: (progress) => show(progressLine(progress)), + }), + ), + ); +} + +/** + * Follow, then report. A Ctrl-C while following prints the view line and no report: collecting + * would make more platform calls after the person asked to stop. Reads that kept failing print + * their last error first, then the report of what was read, or only the view line when no read + * ever worked. The report's own incomplete line is the one place that says why it is incomplete. + */ +export async function followAndReport(input: FollowFlowInput): Promise { + const { client, record, json } = input; + const followed = await follow(input); + const view = viewStep(record.id, json); + if (followed.ended === "detached") { + reportError({ code: "INTERRUPTED", message: DETACHED, next: view }, { json }); + return "interrupted"; + } + if (followed.ended === "read_failed") { + const lines = readFailureMessage(followed.lastError); + if (followed.journeys.size === 0) { + const code = failureCode(followed.lastError); + reportError({ code, message: lines.join("\n"), next: view }, { json }); + return "failed"; + } + for (const line of lines) console.error(line); + } + const journeys = [...followed.journeys.values()]; + const collected = await collectSet({ record, journeys, client }); + if (collected.trajectoryFailures > 0) console.error(unflaggedLine(collected.trajectoryFailures)); + const count = countSet(collected.runs, collected.shape); + const report = { record, count, ended: followed.ended }; + if (json) process.stdout.write(`${JSON.stringify(reportJson(report), null, 2)}\n`); + else for (const line of renderSetReport(report)) console.log(line); + printNext(reportNext(record, count, json), { stderr: json }); + return "reported"; +} + +function launch(input: LaunchFlowInput): Promise { + const { record } = input; + return withStatus((show) => + withInterrupt((signal) => + launchSet({ + client: input.client, + recordId: record.id, + site: record.spec.site, + cells: record.cells, + agents: input.agents, + repeat: record.spec.repeat, + signal, + onProgress: ({ created, planned }) => + show(`${created} of ${planned} ${planned === 1 ? "journey" : "journeys"} created`), + now: () => new Date(input.now()), + dir: input.dir, + }), + ), + ); +} + +function stopCode(result: LaunchResult): string { + if (result.stoppedReason === "interrupted") return "INTERRUPTED"; + if (result.stoppedReason === "platform_down") return "PLATFORM_DOWN"; + return failureCode(result.lastError); +} + +/** Launching stopped with no report to follow: the stop, then view, and cancel once a run started. */ +function launchStopped(result: LaunchResult, record: SetRecord, input: LaunchFlowInput): FlowEnd { + const { json } = input; + const steps: NextStep[] = + record.journeys.length === 0 + ? [{ call: input.runCall, why: "launches the same set again" }] + : [viewStep(record.id, json)]; + if (result.launchedRuns > 0) steps.push(cancelStep(record.id, json)); + const [next, ...more] = steps; + reportError({ code: stopCode(result), message: stopMessage(result).join("\n"), next }, { json }); + for (const step of more) printNext(step, { stderr: true }); + return result.stoppedReason === "interrupted" ? "interrupted" : "failed"; +} + +const LIST_STEP: NextStep = { + call: axCall(["journey", "list"]), + why: "see the sets recorded on this machine", +}; + +/** The record for `id`, for every set command; null once the refusal (an invalid, unknown or damaged record) is reported. */ +export function pickUpRecord(id: string, json: boolean, dir?: string): SetRecord | null { + const refuse = (code: string, message: string) => { + reportError({ code, message, next: LIST_STEP }, { json }); + return null; + }; + if (!isSetId(id)) { + return refuse( + "INVALID_SET_ID", + `${JSON.stringify(plain(id))} is not a set id. Ids look like j-0123456789.`, + ); + } + try { + return readRecord(id, dir) ?? refuse("UNKNOWN_SET", `No set ${id} on this machine.`); + } catch (error) { + if (!(error instanceof CorruptRecordError)) throw error; + const file = `${id}.json`; + const where = plain(join(dir ?? DEFAULT_SETS_DIR, file)); + return refuse( + "CORRUPT_SET_RECORD", + `The record ${file} is damaged, so ax cannot read it. It is at ${where}.`, + ); + } +} + +/** + * Launch into the written record, then follow and report. A quota stop prints why and follows + * what started; any other stop prints why and the commands to pick the set up, without + * following: the platform is failing, or the person pressed Ctrl-C. + */ +export async function launchAndReport(input: LaunchFlowInput): Promise { + const result = await launch(input); + const record = readRecord(input.record.id, input.dir); + if (!record) throw new Error(`the record of set ${input.record.id} is gone`); + if (result.stoppedReason && result.stoppedReason !== "quota") { + return launchStopped(result, record, input); + } + for (const line of stopMessage(result)) console.error(line); + return followAndReport({ ...input, record }); +} diff --git a/src/journey-set/report-json.test.ts b/src/journey-set/report-json.test.ts new file mode 100644 index 0000000..502c395 --- /dev/null +++ b/src/journey-set/report-json.test.ts @@ -0,0 +1,301 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { type CountedRun, countSet, type SetShape } from "./count"; +import type { FollowEnd } from "./follow"; +import type { Level } from "./levels"; +import type { Bucket } from "./outcome"; +import { appendJourney, createRecord, markStopped, type StoppedReason } from "./record"; +import { agentsOf, isComplete, type ReportInput, reportJson } from "./report-json"; +import { parseSetSpec, planCells } from "./spec"; + +// The inputs are built the way render-report.test.ts builds them (a spec from parseSetSpec, a +// record from createRecord and appendJourney in a temp directory, counts from the real countSet) +// with the runs trimmed to what the document and the completeness rule read. The server strings +// are the shapes outcome.test.ts uses: "timeout" is an excluded status, and "run gate unavailable" +// is Studio's start error (oramono apps/studio-service/src/journeys/journeys-service.test.ts:629). + +const A = "claude-code/sonnet"; +const B = "codex/gpt-5"; +const FAILED_TO_START = "run gate unavailable"; +const NEED_TASK = "Find how to create an issue from the API"; +const BRAND_TASK = "Find how to create a Linear issue from the API"; + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ax-set-json-")); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +interface Scenario { + /** Defaults to true: most cases climb the ladder. */ + levels?: boolean; + goals?: string[]; + agents?: string[]; + repeat?: number; + exact?: string; + runs: CountedRun[]; + stopped?: { reason: StoppedReason; notLaunched: number }; + ended?: FollowEnd; +} + +function build(scenario: Scenario): ReportInput { + const agents = scenario.agents ?? [A]; + const parsed = parseSetSpec({ + site: "linear.app", + levels: scenario.levels ?? true, + goals: scenario.goals ?? [NEED_TASK], + agents: agents.join(","), + exact: scenario.exact, + repeat: String(scenario.repeat ?? 1), + }); + if (!parsed.ok) throw new Error(parsed.error); + const { spec } = parsed; + const { cells } = planCells(spec); + let record = createRecord( + { + spec, + agents: agents.map((label) => { + const [harness, model] = label.split("/"); + return { harness, model }; + }), + cells, + }, + dir, + ); + for (const cell of cells) { + record = appendJourney( + record.id, + { + journeyId: `journey-${cell.goalIndex}-${cell.level}`, + goalIndex: cell.goalIndex, + level: cell.level, + repeat: 1, + runIds: [], + failed: [], + createdAt: record.createdAt, + }, + dir, + ); + } + if (scenario.stopped) { + record = markStopped(record.id, scenario.stopped.reason, scenario.stopped.notLaunched, dir); + } + const shape: SetShape = { + goals: spec.goals, + levelsByGoal: spec.goals.map((_, goalIndex) => + cells.filter((cell) => cell.goalIndex === goalIndex).map((cell) => cell.level), + ), + agents, + plannedPerCell: spec.repeat, + }; + return { record, count: countSet(scenario.runs, shape), ended: scenario.ended ?? "settled" }; +} + +function run(level: Level, bucket: Bucket, fields: Partial = {}): CountedRun { + return { goalIndex: 0, level, agent: A, repeat: 1, bucket, flags: null, ...fields }; +} + +function many( + level: Level, + bucket: Bucket, + n: number, + fields: Partial = {}, +): CountedRun[] { + return Array.from({ length: n }, (_, index) => + run(level, bucket, { repeat: index + 1, ...fields }), + ); +} + +describe("reportJson", () => { + it("has the documented keys, raw values and the fraction rate", () => { + const input = build({ + goals: [NEED_TASK, BRAND_TASK], + agents: [A, B], + repeat: 5, + exact: "https://linear.app/developers", + runs: [ + ...many("need", "satisfied", 4), + ...many("need", "unsatisfied", 1, { repeat: 5 }), + run("need", "excluded", { agent: B, rawStatus: "timeout" }), + run("need", "not_gradable", { agent: B, repeat: 2, rawVerdict: "inconclusive" }), + run("brand", "excluded", { agent: B, reason: FAILED_TO_START }), + ], + stopped: { reason: "quota", notLaunched: 3 }, + ended: "detached", + }); + const doc = reportJson(input); + expect(Object.keys(doc)).toEqual([ + "setId", + "site", + "createdAt", + "complete", + "levels", + "skippedLevels", + "agents", + "cells", + "gaps", + "shortLevels", + "journeys", + "notLaunched", + "stoppedReason", + "ended", + ]); + expect(doc.setId).toBe(input.record.id); + expect(doc.site).toBe("linear.app"); + expect(doc.createdAt).toBe(input.record.createdAt); + expect(doc.agents).toEqual([A, B]); + expect(doc.levels).toEqual([ + { goalIndex: 0, text: NEED_TASK, levels: ["need", "brand", "exact"] }, + { goalIndex: 1, text: BRAND_TASK, levels: ["brand", "exact"] }, + ]); + expect(doc.skippedLevels).toEqual([ + expect.objectContaining({ goalIndex: 0, level: "docs" }), + expect.objectContaining({ goalIndex: 1, level: "need" }), + expect.objectContaining({ goalIndex: 1, level: "docs" }), + ]); + const first = doc.cells.find((c) => c.goalIndex === 0 && c.level === "need" && c.agent === A); + expect(first?.rate).toBe(0.8); + const second = doc.cells.find((c) => c.goalIndex === 0 && c.level === "need" && c.agent === B); + expect(second?.excluded).toEqual([{ value: "timeout", count: 1 }]); + expect(second?.notGradable).toEqual([{ value: "inconclusive", count: 1 }]); + const brand = doc.cells.find((c) => c.goalIndex === 0 && c.level === "brand" && c.agent === B); + expect(brand?.neverStarted).toEqual([{ value: FAILED_TO_START, count: 1 }]); + expect(doc.journeys[0]).toEqual({ + journeyId: "journey-0-need", + goalIndex: 0, + level: "need", + repeat: 1, + }); + expect(Object.keys(doc.journeys[0])).toEqual(["journeyId", "goalIndex", "level", "repeat"]); + expect(doc.notLaunched).toBe(3); + expect(doc.stoppedReason).toBe("quota"); + expect(doc.ended).toBe("detached"); + expect(doc.complete).toBe(false); + expect(doc.gaps).toEqual(input.count.gaps); + expect(doc.shortLevels).toEqual(input.count.shortLevels); + }); + + it("reports no stop as null and zero runs not launched, and a null rate below the floor", () => { + const doc = reportJson(build({ runs: [run("need", "satisfied")] })); + expect(doc.stoppedReason).toBeNull(); + expect(doc.notLaunched).toBe(0); + expect(doc.ended).toBe("settled"); + expect(doc.cells[0].rate).toBeNull(); + }); + + it("keeps a journey with no id as null", () => { + const input = build({ runs: [run("need", "satisfied")] }); + const record = appendJourney( + input.record.id, + { + journeyId: null, + goalIndex: 0, + level: "need", + repeat: 2, + runIds: [], + failed: [], + createdAt: input.record.createdAt, + }, + dir, + ); + const withNull = { ...input, record }; + expect(reportJson(withNull).journeys.at(-1)).toEqual({ + journeyId: null, + goalIndex: 0, + level: "need", + repeat: 2, + }); + }); + + it("round trips through JSON", () => { + const doc = reportJson(build({ runs: [run("need", "satisfied")] })); + expect(JSON.parse(JSON.stringify(doc))).toEqual(doc); + }); + + it("keeps server strings raw, esc sequences included", () => { + const esc = "\u001b[2J"; + const agent = `evil/mod${esc}el`; + const doc = reportJson( + build({ + agents: [agent], + runs: [ + run("need", "excluded", { agent, rawStatus: `time${esc}out` }), + run("need", "excluded", { agent, repeat: 2, reason: `gate${esc}down` }), + run("need", "not_gradable", { agent, repeat: 3, rawVerdict: `odd${esc}verdict` }), + ], + }), + ); + expect(doc.agents).toEqual([agent]); + expect(doc.cells[0].excluded[0].value).toBe(`time${esc}out`); + expect(doc.cells[0].neverStarted[0].value).toBe(`gate${esc}down`); + expect(doc.cells[0].notGradable[0].value).toBe(`odd${esc}verdict`); + }); +}); + +describe("a set without levels", () => { + it("lists each goal's one written level and skips nothing", () => { + const doc = reportJson(build({ levels: false, runs: [run("written", "satisfied")] })); + expect(doc.levels).toEqual([{ goalIndex: 0, text: NEED_TASK, levels: ["written"] }]); + expect(doc.skippedLevels).toEqual([]); + expect(doc.cells.map((cell) => [cell.level, cell.satisfied])).toEqual([["written", 1]]); + expect(doc.journeys.map((journey) => journey.level)).toEqual(["written"]); + }); +}); + +describe("isComplete", () => { + it("is true when launch ran, following settled and nothing is in flight or ungraded", () => { + expect(isComplete(build({ runs: [run("need", "satisfied")] }))).toBe(true); + expect(reportJson(build({ runs: [run("need", "satisfied")] })).complete).toBe(true); + }); + + it.each(["quota", "platform_down", "api_error", "interrupted"])( + "is false after a %s stop, however following ended", + (reason) => { + const input = build({ + runs: [run("need", "satisfied")], + stopped: { reason, notLaunched: 1 }, + }); + expect(isComplete(input)).toBe(false); + }, + ); + + it.each(["settle_limit", "wait_limit", "detached", "read_failed"])( + "is false when following ended %s", + (ended) => { + expect(isComplete(build({ runs: [run("need", "satisfied")], ended }))).toBe(false); + }, + ); + + it("is false with a run in flight, even if following said settled", () => { + expect(isComplete(build({ runs: [run("need", "in_flight")] }))).toBe(false); + }); + + it("is false with a run awaiting a grade, even if following said settled", () => { + expect(isComplete(build({ runs: [run("need", "not_graded_yet")] }))).toBe(false); + }); + + it("stays true with excluded and not gradable runs, which are final", () => { + const input = build({ + repeat: 3, + runs: [ + run("need", "satisfied"), + run("need", "excluded", { repeat: 2, rawStatus: "timeout" }), + run("need", "not_gradable", { repeat: 3, rawVerdict: "inconclusive" }), + ], + }); + expect(isComplete(input)).toBe(true); + }); +}); + +describe("agentsOf", () => { + it("lists the columns in order, extra labels included", () => { + const input = build({ agents: [A, B], runs: [] }); + expect(agentsOf(input.count)).toEqual([A, B]); + }); +}); diff --git a/src/journey-set/report-json.ts b/src/journey-set/report-json.ts new file mode 100644 index 0000000..9e00d58 --- /dev/null +++ b/src/journey-set/report-json.ts @@ -0,0 +1,127 @@ +// The `--json` document for a set, and the one rule for when a report is complete. ax builds +// this document itself because no single server payload exists for a set (it is many journeys +// and a local record); every value is raw: statuses, verdicts and reasons as the platform sent +// them, `rate` as a fraction from 0 to 1 (null below the gradable floor). The text report reads +// `isComplete` and `agentsOf` from here so the two forms cannot disagree. + +import type { CellCount, GapLine, SetCount, ShortLevel } from "./count"; +import type { FollowEnd } from "./follow"; +import { LEVELS, type Level } from "./levels"; +import type { SetRecord, StoppedReason } from "./record"; +import { planCells, type SkippedCell } from "./spec"; + +/** What the command has once it has followed, collected and counted a set. */ +export interface ReportInput { + record: SetRecord; + count: SetCount; + /** How following ended (`FollowResult.ended`). */ + ended: FollowEnd; +} + +/** One goal and the levels that ran for it, in ladder order. `goalIndex` indexes the record's goals. */ +export interface GoalLevels { + goalIndex: number; + text: string; + levels: Level[]; +} + +export interface JourneyRef { + /** Null when a create answered with no journey (see the record). */ + journeyId: string | null; + /** 0-based index into the set's goals, as in `levels[].goalIndex`. */ + goalIndex: number; + level: Level; + /** 1-based. */ + repeat: number; +} + +export interface ReportDocument { + setId: string; + site: string; + createdAt: string; + /** Launch was not stopped, following ended `settled`, and no run is in flight or awaiting a grade. */ + complete: boolean; + /** The levels each goal ran, in ladder order, with the goal's text. */ + levels: GoalLevels[]; + /** The levels left out of the plan, each with the note the preview printed. */ + skippedLevels: SkippedCell[]; + /** The report's columns, `harness/model`; also the `agent` of every cell. */ + agents: string[]; + cells: CellCount[]; + gaps: GapLine[]; + shortLevels: ShortLevel[]; + journeys: JourneyRef[]; + /** Runs that never started because launching stopped; 0 when it did not. */ + notLaunched: number; + stoppedReason: StoppedReason | null; + ended: FollowEnd; +} + +/** The set's agents in column order. Cells list every agent for every level, so the first level's do. */ +export function agentsOf(count: SetCount): string[] { + const agents: string[] = []; + for (const cell of count.cells) { + if (!agents.includes(cell.agent)) agents.push(cell.agent); + } + return agents; +} + +function sum(cells: readonly CellCount[], pick: (cell: CellCount) => number): number { + return cells.reduce((total, cell) => total + pick(cell), 0); +} + +/** Runs still going, summed over every cell. */ +export function runsInFlight(count: SetCount): number { + return sum(count.cells, (cell) => cell.inFlight); +} + +/** Runs that finished but have no grade yet. */ +export function runsAwaitingGrade(count: SetCount): number { + return sum(count.cells, (cell) => cell.notGradedYet); +} + +/** True only when launching ran to its end, following settled, and nothing is in flight or ungraded. */ +export function isComplete(input: ReportInput): boolean { + return ( + !input.record.stoppedReason && + input.ended === "settled" && + runsInFlight(input.count) === 0 && + runsAwaitingGrade(input.count) === 0 + ); +} + +function levelsRan(record: SetRecord): GoalLevels[] { + return record.spec.goals.map((text, goalIndex) => ({ + goalIndex, + text, + levels: LEVELS.filter((level) => + record.cells.some((cell) => cell.goalIndex === goalIndex && cell.level === level), + ), + })); +} + +/** The `--json` document: raw values throughout, so scripts key on what the platform said. */ +export function reportJson(input: ReportInput): ReportDocument { + const { record, count } = input; + return { + setId: record.id, + site: record.spec.site, + createdAt: record.createdAt, + complete: isComplete(input), + levels: levelsRan(record), + skippedLevels: planCells(record.spec).skipped, + agents: agentsOf(count), + cells: count.cells, + gaps: count.gaps, + shortLevels: count.shortLevels, + journeys: record.journeys.map(({ journeyId, goalIndex, level, repeat }) => ({ + journeyId, + goalIndex, + level, + repeat, + })), + notLaunched: record.notLaunched ?? 0, + stoppedReason: record.stoppedReason || null, + ended: input.ended, + }; +} diff --git a/src/journey-set/spec.test.ts b/src/journey-set/spec.test.ts new file mode 100644 index 0000000..03fd163 --- /dev/null +++ b/src/journey-set/spec.test.ts @@ -0,0 +1,436 @@ +import { describe, expect, it } from "vitest"; +import { DOCS_SKIP_NOTE, EXACT_SKIP_NOTE, needSkipNote } from "./levels"; +import { + DEFAULT_REPEAT, + DEFAULT_WAIT_MINUTES, + exceedsRunCap, + journeyMode, + MAX_AGENTS, + MAX_GOALS, + MAX_REPEAT, + MAX_RUNS, + MISSING_DOMAIN, + parseSetSpec, + parseWaitMinutes, + planCells, + plannedRuns, + type SetSpec, + type SetSpecInput, +} from "./spec"; + +const GOAL = "Find how to create an issue from the API"; +const BASE: SetSpecInput = { site: "linear.app", goals: [GOAL], levels: true }; + +function parsed(input: Partial): SetSpec { + const result = parseSetSpec({ ...BASE, ...input }); + if (!result.ok) throw new Error(`expected a valid spec, got: ${result.error}`); + return result.spec; +} + +function errorOf(input: Partial): string { + const result = parseSetSpec({ ...BASE, ...input }); + if (result.ok) throw new Error("expected a usage error, got a valid spec"); + return result.error; +} + +describe("caps and defaults", () => { + it("pins the shared numbers", () => { + expect({ + MAX_GOALS, + MAX_AGENTS, + MAX_REPEAT, + MAX_RUNS, + DEFAULT_REPEAT, + DEFAULT_WAIT_MINUTES, + }).toEqual({ + MAX_GOALS: 5, + MAX_AGENTS: 4, + MAX_REPEAT: 10, + MAX_RUNS: 200, + DEFAULT_REPEAT: 1, + DEFAULT_WAIT_MINUTES: 60, + }); + }); +}); + +describe("parseSetSpec: defaults", () => { + it("fills every optional value", () => { + expect(parsed({})).toEqual({ + site: "linear.app", + goals: [GOAL], + agents: [], + levels: true, + brand: "Linear", + repeat: 1, + waitMinutes: 60, + }); + }); +}); + +describe("parseSetSpec: levels", () => { + it("is off without --levels, --brand, --docs or --exact", () => { + expect(parsed({ levels: false }).levels).toBe(false); + }); + + it("is on with --levels, and with any value a level prompt needs", () => { + expect(parsed({ levels: true }).levels).toBe(true); + expect(parsed({ levels: false, brand: "Linear" }).levels).toBe(true); + expect(parsed({ levels: false, docs: "https://linear.app/docs" }).levels).toBe(true); + expect(parsed({ levels: false, exact: "https://linear.app/developers" }).levels).toBe(true); + }); +}); + +describe("parseSetSpec: site", () => { + it("keeps a bare domain", () => { + expect(parsed({ site: "linear.app" }).site).toBe("linear.app"); + }); + + it("reduces a URL to its host and lowercases it", () => { + expect(parsed({ site: "https://Linear.App/developers?ref=x" }).site).toBe("linear.app"); + expect(parsed({ site: "http://linear.app:8080/" }).site).toBe("linear.app"); + expect(parsed({ site: " LINEAR.app/docs " }).site).toBe("linear.app"); + }); + + it("drops a leading www, so on-site steps on the apex count", () => { + expect(parsed({ site: "https://www.linear.app/developers" }).site).toBe("linear.app"); + expect(parsed({ site: "WWW.linear.app" }).site).toBe("linear.app"); + expect(parsed({ site: "https://www.linear.app" }).brand).toBe("Linear"); + }); + + it("keeps every other subdomain, since the set targets that host", () => { + expect(parsed({ site: "docs.stripe.com" }).site).toBe("docs.stripe.com"); + expect(parsed({ site: "www2.example.com" }).site).toBe("www2.example.com"); + }); + + it("folds a fullwidth spelling to its plain form", () => { + expect(parsed({ site: "ｌｉｎｅａｒ．ａｐｐ" }).site).toBe("linear.app"); + }); + + it("rejects anything that is not a domain or an http(s) URL", () => { + for (const site of ["", " ", "linear", "linear app", "ftp://linear.app", "https://"]) { + expect(errorOf({ site })).toMatch(/^--domain must be a domain like linear\.app/); + } + }); + + it("needs --domain at all", () => { + expect(errorOf({ site: undefined })).toBe(MISSING_DOMAIN); + expect(MISSING_DOMAIN).toMatch(/--domain/); + }); +}); + +describe("parseSetSpec: goals", () => { + it("trims each goal", () => { + expect(parsed({ goals: [" Find the pricing page "] }).goals).toEqual([ + "Find the pricing page", + ]); + }); + + it("accepts 1 and 5 goals", () => { + expect(parsed({ goals: ["a"] }).goals).toHaveLength(1); + expect(parsed({ goals: ["a", "b", "c", "d", "e"] }).goals).toHaveLength(5); + }); + + it("rejects 0 and 6 goals", () => { + expect(errorOf({ goals: [] })).toBe("at least one goal is required"); + expect(errorOf({ goals: ["a", "b", "c", "d", "e", "f"] })).toMatch(/at most 5/); + }); + + it("rejects a goal that is empty after trimming, naming which one", () => { + expect(errorOf({ goals: ["Find the pricing page", " "] })).toMatch(/goal 2/); + }); +}); + +describe("parseSetSpec: agents", () => { + it("is empty without --agents, meaning the catalog default", () => { + expect(parsed({}).agents).toEqual([]); + }); + + it("reads comma separated harness[/model] pairs", () => { + expect(parsed({ agents: "claude-code/claude-sonnet-5, codex" }).agents).toEqual([ + { harness: "claude-code", model: "claude-sonnet-5" }, + { harness: "codex" }, + ]); + }); + + it("splits on the first slash, so a model may contain one", () => { + expect(parsed({ agents: "eve/anthropic/claude-sonnet-5" }).agents).toEqual([ + { harness: "eve", model: "anthropic/claude-sonnet-5" }, + ]); + }); + + it("accepts 1 and 4 agents", () => { + expect(parsed({ agents: "codex" }).agents).toHaveLength(1); + expect( + parsed({ + agents: "claude-code,codex/gpt-5.5,eve/claude-sonnet-5,claude-code/claude-sonnet-5", + }).agents, + ).toHaveLength(4); + }); + + it("rejects 5 agents", () => { + expect( + errorOf({ + agents: "claude-code,codex,eve,codex/gpt-5.5,eve/claude-sonnet-5", + }), + ).toMatch(/--agents/); + }); + + it("rejects an empty --agents and empty entries", () => { + for (const agents of ["", " ", "codex,", ",codex", "codex,,eve"]) { + expect(errorOf({ agents })).toMatch(/--agents/); + } + }); + + it("rejects a pair with an empty harness or model", () => { + expect(errorOf({ agents: "/gpt-5.5" })).toMatch(/--agents/); + expect(errorOf({ agents: "codex/" })).toMatch(/--agents/); + }); + + it("rejects the same agent listed twice", () => { + expect(errorOf({ agents: "codex/gpt-5.5, codex/gpt-5.5" })).toMatch(/twice/); + }); +}); + +describe("parseSetSpec: brand", () => { + it("defaults to the site's name label, capitalized", () => { + expect(parsed({ site: "stripe.com" }).brand).toBe("Stripe"); + }); + + it("takes --brand over the default, trimmed", () => { + expect(parsed({ brand: " Linear App " }).brand).toBe("Linear App"); + }); + + it("rejects an empty --brand", () => { + expect(errorOf({ brand: " " })).toMatch(/--brand/); + }); +}); + +describe("parseSetSpec: docs and exact URLs", () => { + it("keeps http and https URLs as written, trimmed", () => { + const spec = parsed({ + docs: " https://linear.app/docs ", + exact: "http://linear.app/developers", + }); + expect(spec.docs).toBe("https://linear.app/docs"); + expect(spec.exact).toBe("http://linear.app/developers"); + }); + + it("rejects anything that is not an http or https URL", () => { + for (const value of [ + "", + "linear.app/docs", + "ftp://linear.app/docs", + "javascript:alert(1)", + "https://linear.app/a b", + ]) { + expect(errorOf({ docs: value })).toMatch(/--docs/); + expect(errorOf({ exact: value })).toMatch(/--exact/); + } + }); +}); + +describe("parseSetSpec: repeat", () => { + it("defaults to 1", () => { + expect(parsed({}).repeat).toBe(1); + }); + + it("accepts 1 and 10", () => { + expect(parsed({ repeat: "1" }).repeat).toBe(1); + expect(parsed({ repeat: "10" }).repeat).toBe(10); + }); + + it("rejects 0, 11, fractions and non numbers", () => { + for (const repeat of ["0", "11", "2.5", "", " ", "three", "-1"]) { + expect(errorOf({ repeat })).toMatch(/--repeat must be an integer between 1 and 10/); + } + }); +}); + +describe("parseSetSpec: wait", () => { + it("defaults to 60 minutes", () => { + expect(parsed({}).waitMinutes).toBe(60); + }); + + it("accepts any positive whole number of minutes", () => { + expect(parsed({ wait: "1" }).waitMinutes).toBe(1); + expect(parsed({ wait: "240" }).waitMinutes).toBe(240); + }); + + it("rejects 0, negatives, fractions and non numbers", () => { + for (const wait of ["0", "-5", "1.5", "", "soon"]) { + expect(errorOf({ wait })).toMatch(/--wait/); + } + }); + + it("is the same parser `ax journey view` uses on its own", () => { + expect(parseWaitMinutes(undefined)).toEqual({ value: 60 }); + expect(parseWaitMinutes("15")).toEqual({ value: 15 }); + expect(parseWaitMinutes("0")).toEqual({ error: expect.stringMatching(/--wait/) }); + }); +}); + +describe("planCells", () => { + it("plans every level in ladder order when nothing is skipped", () => { + const spec = parsed({ + docs: "https://linear.app/docs", + exact: "https://linear.app/developers", + }); + expect(planCells(spec)).toEqual({ + cells: [ + { goalIndex: 0, level: "need", prompt: GOAL }, + { goalIndex: 0, level: "brand", prompt: `${GOAL}. Use Linear.` }, + { + goalIndex: 0, + level: "docs", + prompt: `${GOAL}. Use Linear. Its documentation is at https://linear.app/docs.`, + }, + { + goalIndex: 0, + level: "exact", + prompt: `${GOAL}. Start at https://linear.app/developers.`, + }, + ], + skipped: [], + }); + }); + + it("skips docs and exact without their URLs, each with its note", () => { + expect(planCells(parsed({}))).toEqual({ + cells: [ + { goalIndex: 0, level: "need", prompt: GOAL }, + { goalIndex: 0, level: "brand", prompt: `${GOAL}. Use Linear.` }, + ], + skipped: [ + { goalIndex: 0, level: "docs", note: DOCS_SKIP_NOTE }, + { goalIndex: 0, level: "exact", note: EXACT_SKIP_NOTE }, + ], + }); + }); + + it("skips the need level when the goal names the brand", () => { + const plan = planCells(parsed({ goals: ["Find the Linear API rate limits"] })); + expect(plan.cells.map((cell) => cell.level)).toEqual(["brand"]); + expect(plan.skipped[0]).toEqual({ goalIndex: 0, level: "need", note: needSkipNote("Linear") }); + }); + + it("names the domain in the note when only the domain matched", () => { + const plan = planCells( + parsed({ brand: "Acme", goals: ["Open linear.app and find the rate limits"] }), + ); + expect(plan.skipped[0]).toEqual({ + goalIndex: 0, + level: "need", + note: needSkipNote("linear.app"), + }); + }); + + it("checks the leak against --brand when given, not the default", () => { + const plan = planCells(parsed({ brand: "Acme", goals: ["Find the Linear API rate limits"] })); + expect(plan.cells.map((cell) => cell.level)).toEqual(["need", "brand"]); + expect(plan.cells[1].prompt).toBe("Find the Linear API rate limits. Use Acme."); + }); + + it("plans each goal on its own, so two goals can run different levels", () => { + const plan = planCells( + parsed({ + goals: ["Find the Linear API rate limits", GOAL], + exact: "https://linear.app/developers", + }), + ); + expect(plan.cells.map((cell) => [cell.goalIndex, cell.level])).toEqual([ + [0, "brand"], + [0, "exact"], + [1, "need"], + [1, "brand"], + [1, "exact"], + ]); + expect(plan.skipped.map((skip) => [skip.goalIndex, skip.level])).toEqual([ + [0, "need"], + [0, "docs"], + [1, "docs"], + ]); + }); + + it("without levels, plans one written cell per goal and skips nothing", () => { + const plan = planCells( + parsed({ levels: false, goals: ["Find the Linear API rate limits", ` ${GOAL} `] }), + ); + expect(plan).toEqual({ + cells: [ + { goalIndex: 0, level: "written", prompt: "Find the Linear API rate limits" }, + { goalIndex: 1, level: "written", prompt: GOAL }, + ], + skipped: [], + }); + }); +}); + +describe("journeyMode", () => { + const one = { goals: [GOAL], levels: false }; + + it("is one journey for one goal, at most one agent, no repeat above 1 and levels off", () => { + expect(journeyMode(one)).toEqual({ value: { set: false, agents: [] } }); + expect(journeyMode({ ...one, repeat: "1" })).toEqual({ value: { set: false, agents: [] } }); + expect(journeyMode({ ...one, agents: "codex/gpt-5.5" })).toEqual({ + value: { set: false, agents: [{ harness: "codex", model: "gpt-5.5" }] }, + }); + }); + + it("is a set for several goals, several agents or a repeat above 1", () => { + expect(journeyMode({ ...one, goals: [GOAL, "Plan a sprint"] })).toMatchObject({ + value: { set: true }, + }); + expect(journeyMode({ ...one, agents: "codex,claude-code" })).toMatchObject({ + value: { set: true }, + }); + expect(journeyMode({ ...one, repeat: "2" })).toMatchObject({ value: { set: true } }); + }); + + it("is a set whenever levels are on", () => { + for (const flags of [ + { levels: true }, + { brand: "Linear" }, + { docs: "https://linear.app/docs" }, + { exact: "https://linear.app/developers" }, + ]) { + expect(journeyMode({ ...one, ...flags })).toMatchObject({ value: { set: true } }); + } + }); + + it("refuses a bad --agents or --repeat, since the mode cannot be told without them", () => { + expect(journeyMode({ ...one, agents: "codex," })).toEqual({ + error: expect.stringMatching(/--agents/), + }); + expect(journeyMode({ ...one, repeat: "0" })).toEqual({ + error: expect.stringMatching(/--repeat/), + }); + }); +}); + +describe("plannedRuns and the run cap", () => { + it("multiplies cells by agents by repeats", () => { + const { cells } = planCells(parsed({ exact: "https://linear.app/developers" })); + expect(cells).toHaveLength(3); + expect(plannedRuns(cells, 2, 3)).toBe(18); + expect(plannedRuns([], 4, 10)).toBe(0); + }); + + it("allows exactly 200 runs and refuses 201", () => { + expect(exceedsRunCap(200)).toBe(false); + expect(exceedsRunCap(201)).toBe(true); + }); + + it("puts a set at the cap within it and one more repeat over it", () => { + const goals = [ + "a Linear goal", + "b Linear goal", + "c Linear goal", + "d Linear goal", + "e Linear goal", + ]; + const { cells } = planCells(parsed({ goals, exact: "https://linear.app/developers" })); + expect(cells).toHaveLength(10); + expect(exceedsRunCap(plannedRuns(cells, 4, 5))).toBe(false); + expect(exceedsRunCap(plannedRuns(cells, 4, 6))).toBe(true); + }); +}); diff --git a/src/journey-set/spec.ts b/src/journey-set/spec.ts new file mode 100644 index 0000000..c67946c --- /dev/null +++ b/src/journey-set/spec.ts @@ -0,0 +1,331 @@ +// A set's spec: the command-line values checked against the caps, and the cells it plans (goal x +// level, with the prompt each one sends). Every value is validated here, so a bad flag is a usage +// error before anything launches. The rule for whether `ax journey` runs one journey or a set +// lives here too, so the command and the spec read the flags the same way. + +import { + brandPrompt, + DOCS_SKIP_NOTE, + defaultBrand, + docsPrompt, + EXACT_SKIP_NOTE, + exactPrompt, + LADDER, + type LadderLevel, + type Level, + needPrompt, + needSkipNote, + siteTermIn, + writtenPrompt, +} from "./levels"; +import { normalizeDomain } from "./vendor/domain-normalize"; + +export const MAX_GOALS = 5; +export const MAX_AGENTS = 4; +export const MAX_REPEAT = 10; +export const MAX_RUNS = 200; +export const DEFAULT_REPEAT = 1; +export const DEFAULT_WAIT_MINUTES = 60; + +/** The harness a single journey runs, and the one a set prefers when no agent is named. */ +export const DEFAULT_HARNESS = "claude-code"; + +/** The flags that turn levels on: `--levels` itself, or any value a level prompt needs. */ +export interface LevelFlags { + levels: boolean; + brand?: string; + docs?: string; + exact?: string; +} + +/** The values as the command line gave them, unchecked. */ +export interface SetSpecInput extends LevelFlags { + /** --domain: a bare domain or a URL. A set needs one. */ + site?: string; + /** Every goal, in order. */ + goals: readonly string[]; + /** Raw --agents: comma separated `harness[/model]`. */ + agents?: string; + /** Raw --repeat. */ + repeat?: string; + /** Raw --wait, in minutes. */ + wait?: string; +} + +export interface AgentChoice { + harness: string; + model?: string; +} + +export interface SetSpec { + /** The host alone: NFKC, trimmed, lowercase, without a leading `www.`. */ + site: string; + goals: string[]; + /** Empty when --agents was not given: the catalog default, resolved by the preview. */ + agents: AgentChoice[]; + /** True when each goal climbs the ladder; false runs each goal once, as written. */ + levels: boolean; + /** --brand, or defaultBrand(site). Only the level prompts use it. */ + brand: string; + docs?: string; + exact?: string; + repeat: number; + waitMinutes: number; +} + +export type ParsedSetSpec = { ok: true; spec: SetSpec } | { ok: false; error: string }; + +export type Parsed = { value: T } | { error: string }; + +/** One planned cell. `goalIndex` is the index into `SetSpec.goals`. */ +export interface PlannedCell { + goalIndex: number; + level: Level; + prompt: string; +} + +export interface SkippedCell { + goalIndex: number; + level: Level; + note: string; +} + +export interface CellPlan { + cells: PlannedCell[]; + skipped: SkippedCell[]; +} + +export const MISSING_DOMAIN = + "a set of journeys needs --domain, the site the agents work on (e.g. --domain linear.app)"; + +function isHttp(url: URL): boolean { + return url.protocol === "http:" || url.protocol === "https:"; +} + +/** The set's site from a domain or URL; a free `ax journey` reduces its target with it too. */ +export function parseSetSite(raw: string): Parsed { + const trimmed = raw.trim(); + const url = + trimmed && !/\s/.test(trimmed) + ? URL.parse(trimmed.includes("://") ? trimmed : `https://${trimmed}`) + : null; + const host = (url && isHttp(url) && normalizeDomain(url.hostname)) || ""; + const site = host.replace(/^www\./, ""); + if (!site.includes(".")) { + return { + error: `--domain must be a domain like linear.app or an http(s) URL, got ${JSON.stringify(raw)}`, + }; + } + return { value: site }; +} + +function parseGoals(raw: readonly string[]): Parsed { + if (raw.length === 0) return { error: "at least one goal is required" }; + if (raw.length > MAX_GOALS) { + return { error: `at most ${MAX_GOALS} goals, got ${raw.length}` }; + } + const goals = raw.map((goal) => goal.trim()); + const empty = goals.indexOf(""); + if (empty !== -1) return { error: `goal ${empty + 1} is empty` }; + return { value: goals }; +} + +function parseAgent(entry: string): AgentChoice | undefined { + const slash = entry.indexOf("/"); + if (slash === -1) return entry ? { harness: entry } : undefined; + const harness = entry.slice(0, slash).trim(); + const model = entry.slice(slash + 1).trim(); + return harness && model ? { harness, model } : undefined; +} + +export function parseAgents(raw: string | undefined): Parsed { + if (raw === undefined) return { value: [] }; + const usage = `--agents takes 1 to ${MAX_AGENTS} agents as harness[/model], comma separated, got ${JSON.stringify(raw)}`; + const entries = raw.split(",").map((entry) => entry.trim()); + if (entries.length > MAX_AGENTS) return { error: usage }; + const agents: AgentChoice[] = []; + const seen = new Set(); + for (const entry of entries) { + const agent = parseAgent(entry); + if (!agent) return { error: usage }; + const key = `${agent.harness}/${agent.model ?? ""}`; + if (seen.has(key)) return { error: `--agents lists ${entry} twice` }; + seen.add(key); + agents.push(agent); + } + return { value: agents }; +} + +function parseBrand(raw: string | undefined, site: string): Parsed { + if (raw === undefined) return { value: defaultBrand(site) }; + const brand = raw.trim(); + return brand ? { value: brand } : { error: "--brand must not be empty" }; +} + +function parseHttpUrl(raw: string | undefined, flag: string): Parsed { + if (raw === undefined) return { value: undefined }; + const value = raw.trim(); + const url = value && !/\s/.test(value) ? URL.parse(value) : null; + if (!url || !isHttp(url)) { + return { error: `${flag} must be an http or https URL, got ${JSON.stringify(raw)}` }; + } + return { value }; +} + +export function parseRepeat(raw: string | undefined): Parsed { + if (raw === undefined) return { value: DEFAULT_REPEAT }; + const value = Number(raw); + if (!Number.isInteger(value) || value < 1 || value > MAX_REPEAT) { + return { + error: `--repeat must be an integer between 1 and ${MAX_REPEAT}, got ${JSON.stringify(raw)}`, + }; + } + return { value }; +} + +/** Raw --wait to minutes; `ax journey view` takes --wait without a spec, so it calls this directly. */ +export function parseWaitMinutes(raw: string | undefined): Parsed { + if (raw === undefined) return { value: DEFAULT_WAIT_MINUTES }; + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) { + return { + error: `--wait must be a positive whole number of minutes, got ${JSON.stringify(raw)}`, + }; + } + return { value }; +} + +/** Levels are on with --levels, and with any of --brand, --docs or --exact, which imply it. */ +export function levelsOn(flags: LevelFlags): boolean { + return ( + flags.levels || + flags.brand !== undefined || + flags.docs !== undefined || + flags.exact !== undefined + ); +} + +/** The flags that decide between one journey and a set, unchecked. */ +export interface ModeInput extends LevelFlags { + goals: readonly string[]; + agents?: string; + repeat?: string; +} + +/** Whether the flags ask for a set, and the agents --agents named (empty without it). */ +export interface JourneyMode { + set: boolean; + agents: AgentChoice[]; +} + +/** + * One journey, drawn live, for one goal and at most one agent, with no --repeat above 1 and levels + * off; a set of journeys for anything more. A bad --agents or --repeat is the usage error, since + * the mode cannot be told without them. + */ +export function journeyMode(input: ModeInput): Parsed { + const agents = parseAgents(input.agents); + if ("error" in agents) return agents; + const repeat = parseRepeat(input.repeat); + if ("error" in repeat) return repeat; + const set = + input.goals.length > 1 || agents.value.length > 1 || repeat.value > 1 || levelsOn(input); + return { value: { set, agents: agents.value } }; +} + +function fail(part: { error: string }): ParsedSetSpec { + return { ok: false, error: part.error }; +} + +/** The first invalid value is the usage error; otherwise the spec, defaults filled. */ +export function parseSetSpec(input: SetSpecInput): ParsedSetSpec { + if (input.site === undefined) return fail({ error: MISSING_DOMAIN }); + const site = parseSetSite(input.site); + if ("error" in site) return fail(site); + const goals = parseGoals(input.goals); + if ("error" in goals) return fail(goals); + const agents = parseAgents(input.agents); + if ("error" in agents) return fail(agents); + const brand = parseBrand(input.brand, site.value); + if ("error" in brand) return fail(brand); + const docs = parseHttpUrl(input.docs, "--docs"); + if ("error" in docs) return fail(docs); + const exact = parseHttpUrl(input.exact, "--exact"); + if ("error" in exact) return fail(exact); + const repeat = parseRepeat(input.repeat); + if ("error" in repeat) return fail(repeat); + const wait = parseWaitMinutes(input.wait); + if ("error" in wait) return fail(wait); + const spec: SetSpec = { + site: site.value, + goals: goals.value, + agents: agents.value, + levels: levelsOn(input), + brand: brand.value, + ...(docs.value !== undefined && { docs: docs.value }), + ...(exact.value !== undefined && { exact: exact.value }), + repeat: repeat.value, + waitMinutes: wait.value, + }; + return { ok: true, spec }; +} + +/** + * One ladder level of one goal: its prompt, or the note saying why it is skipped. + * Need runs only when the goal does not name the site; brand always runs; + * docs only with --docs; exact only with --exact. + */ +function planLevel( + level: LadderLevel, + goal: string, + spec: SetSpec, +): { prompt: string } | { note: string } { + switch (level) { + case "need": { + const term = siteTermIn(goal, spec.site, spec.brand); + return term === null ? { prompt: needPrompt(goal) } : { note: needSkipNote(term) }; + } + case "brand": + return { prompt: brandPrompt(goal, spec.brand) }; + case "docs": + return spec.docs + ? { prompt: docsPrompt(goal, spec.brand, spec.docs) } + : { note: DOCS_SKIP_NOTE }; + case "exact": + return spec.exact ? { prompt: exactPrompt(goal, spec.exact) } : { note: EXACT_SKIP_NOTE }; + } +} + +/** + * Every goal's cells: with levels on, its ladder levels in order, plus the ones skipped with their + * notes; with levels off, one written cell per goal and nothing skipped. + */ +export function planCells(spec: SetSpec): CellPlan { + const plan: CellPlan = { cells: [], skipped: [] }; + for (const [goalIndex, text] of spec.goals.entries()) { + if (!spec.levels) { + plan.cells.push({ goalIndex, level: "written", prompt: writtenPrompt(text) }); + continue; + } + for (const level of LADDER) { + const planned = planLevel(level, text, spec); + if ("prompt" in planned) plan.cells.push({ goalIndex, level, prompt: planned.prompt }); + else plan.skipped.push({ goalIndex, level, note: planned.note }); + } + } + return plan; +} + +/** Runs a set launches: every planned cell, for every agent, every repeat. */ +export function plannedRuns( + cells: readonly PlannedCell[], + agentCount: number, + repeat: number, +): number { + return cells.length * agentCount * repeat; +} + +/** True when the runs are more than one set may launch (MAX_RUNS), whatever the plan allows. */ +export function exceedsRunCap(runs: number): boolean { + return runs > MAX_RUNS; +} diff --git a/src/journey-set/vendor/__fixtures__/run-flags/blocked-first-403.json b/src/journey-set/vendor/__fixtures__/run-flags/blocked-first-403.json new file mode 100644 index 0000000..056f294 --- /dev/null +++ b/src/journey-set/vendor/__fixtures__/run-flags/blocked-first-403.json @@ -0,0 +1,54 @@ +{ + "run_id": "0cc1a662-4ab6-4dba-b4b2-8be2e2cb4ddd", + "domain": "fiverr.com", + "trajectory": { + "steps": [ + { + "id": 0, + "type": "tool_call", + "url_host": "fiverr.com", + "status": 403 + }, + { + "id": 1, + "type": "tool_call", + "url_host": "developers.fiverr.com" + }, + { + "id": 2, + "type": "tool_call" + }, + { + "id": 3, + "type": "tool_call", + "url_host": "docs.enterprise.fiverr.com", + "status": 200 + }, + { + "id": 4, + "type": "tool_call" + }, + { + "id": 5, + "type": "tool_call", + "url_host": "stoke.docs.apiary.io", + "status": 200 + }, + { + "id": 6, + "type": "text" + } + ] + }, + "run_signals": { + "answer_basis": { + "sections_total": 7, + "from_site": 1, + "from_external": 3, + "from_search": 2, + "from_memory": 1, + "site_share": 0.14, + "memory_share": 0.14 + } + } +} diff --git a/src/journey-set/vendor/__fixtures__/run-flags/clean-onsite.json b/src/journey-set/vendor/__fixtures__/run-flags/clean-onsite.json new file mode 100644 index 0000000..b677bd4 --- /dev/null +++ b/src/journey-set/vendor/__fixtures__/run-flags/clean-onsite.json @@ -0,0 +1,100 @@ +{ + "run_id": "d0a55cc0-f961-4d26-8d07-310c02c3e01e", + "domain": "linear.app", + "trajectory": { + "steps": [ + { + "id": 0, + "type": "text" + }, + { + "id": 1, + "type": "tool_call", + "url_host": "linear.app", + "status": 200 + }, + { + "id": 2, + "type": "tool_call", + "url_host": "linear.app" + }, + { + "id": 3, + "type": "tool_call" + }, + { + "id": 4, + "type": "text" + }, + { + "id": 5, + "type": "tool_call", + "url_host": "linear.app", + "status": 200 + }, + { + "id": 6, + "type": "tool_call", + "url_host": "linear.app", + "status": 200 + }, + { + "id": 7, + "type": "tool_call", + "url_host": "linear.app", + "status": 200 + }, + { + "id": 8, + "type": "text" + }, + { + "id": 9, + "type": "tool_call", + "url_host": "docs.linear.app", + "status": 200 + }, + { + "id": 10, + "type": "tool_call", + "url_host": "docs.linear.app", + "status": 200 + }, + { + "id": 11, + "type": "text" + }, + { + "id": 12, + "type": "tool_call", + "url_host": "docs.linear.app" + }, + { + "id": 13, + "type": "text" + }, + { + "id": 14, + "type": "tool_call" + }, + { + "id": 15, + "type": "tool_call" + }, + { + "id": 16, + "type": "text" + }, + { + "id": 17, + "type": "tool_call", + "url_host": "docs.linear.app", + "status": 200 + }, + { + "id": 18, + "type": "text" + } + ] + } +} diff --git a/src/journey-set/vendor/__fixtures__/run-flags/memory-answer.json b/src/journey-set/vendor/__fixtures__/run-flags/memory-answer.json new file mode 100644 index 0000000..757e24b --- /dev/null +++ b/src/journey-set/vendor/__fixtures__/run-flags/memory-answer.json @@ -0,0 +1,61 @@ +{ + "run_id": "0201b819-0f11-4ab7-b8c0-b09e0ff20ca1", + "domain": "bino.church", + "trajectory": { + "steps": [ + { + "id": 0, + "type": "tool_call", + "url_host": "bino.church", + "status": 200 + }, + { + "id": 1, + "type": "text" + }, + { + "id": 2, + "type": "tool_call", + "url_host": "bino.church", + "status": 404 + }, + { + "id": 3, + "type": "tool_call", + "url_host": "bino.church", + "status": 404 + }, + { + "id": 4, + "type": "tool_call", + "url_host": "bino.church", + "status": 404 + }, + { + "id": 5, + "type": "tool_call" + }, + { + "id": 6, + "type": "tool_call", + "url_host": "github.com", + "status": 200 + }, + { + "id": 7, + "type": "text" + } + ] + }, + "run_signals": { + "answer_basis": { + "sections_total": 6, + "from_site": 1, + "from_external": 1, + "from_search": 1, + "from_memory": 3, + "site_share": 0.17, + "memory_share": 0.5 + } + } +} diff --git a/src/journey-set/vendor/__fixtures__/run-flags/offsite-only.json b/src/journey-set/vendor/__fixtures__/run-flags/offsite-only.json new file mode 100644 index 0000000..5f84565 --- /dev/null +++ b/src/journey-set/vendor/__fixtures__/run-flags/offsite-only.json @@ -0,0 +1,47 @@ +{ + "run_id": "a7aec281-5955-4291-8d44-173ace6d95d5", + "domain": "linear.com", + "trajectory": { + "steps": [ + { + "id": 0, + "type": "tool_call", + "url_host": "linear.app", + "status": 200 + }, + { + "id": 1, + "type": "tool_call", + "url_host": "linear.app", + "status": 200 + }, + { + "id": 2, + "type": "tool_call", + "url_host": "linear.app", + "status": 200 + }, + { + "id": 3, + "type": "tool_call", + "url_host": "linear.app", + "status": 200 + }, + { + "id": 4, + "type": "text" + } + ] + }, + "run_signals": { + "answer_basis": { + "sections_total": 5, + "from_site": 4, + "from_external": 0, + "from_search": 0, + "from_memory": 1, + "site_share": 0.8, + "memory_share": 0.2 + } + } +} diff --git a/src/journey-set/vendor/domain-normalize.ts b/src/journey-set/vendor/domain-normalize.ts new file mode 100644 index 0000000..4d7036b --- /dev/null +++ b/src/journey-set/vendor/domain-normalize.ts @@ -0,0 +1,14 @@ +// Copied from ora's journey code. DO NOT EDIT BY HAND. +// +// Source: ora's src/lib/journey/domain-normalize.ts at commit 075e9fe8f, +// verbatim. ./run-flags.ts matches hosts with it, and a set's spec reduces +// the site with it, so both agree with ora. Re-copy, never edit. Biome does +// not format this directory. + +/** Canonical domain normalisation for journey/simulation writes and site + * matching (the simulations route stores the same form it rate-limits on). + * Kept free of server imports so browser code can share it. */ +export function normalizeDomain(domain: string | null | undefined): string | null { + const d = domain?.normalize("NFKC").trim().toLowerCase(); + return d && d.length > 0 ? d : null; +} diff --git a/src/journey-set/vendor/run-flags.test.ts b/src/journey-set/vendor/run-flags.test.ts new file mode 100644 index 0000000..37fbda0 --- /dev/null +++ b/src/journey-set/vendor/run-flags.test.ts @@ -0,0 +1,213 @@ +// Copied from ora's journey code. DO NOT EDIT BY HAND. +// +// Source: ora's src/lib/journey/run-flags.spec.ts at commit 075e9fe8f, with +// its fixtures copied byte for byte into ./__fixtures__/run-flags/. These are +// ora's own cases for the rule in ./run-flags.ts, so a re-copy of the rule +// comes with a re-copy of this file. Renamed to .test.ts because ax's vitest +// collects that suffix; the only other deviations are the two source paths in +// the last describe, marked inline. Biome does not format this directory. + +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { normalizeDomain } from "./domain-normalize"; +import { deriveRunFlags, isSiteHost, onSiteSteps, raisedFlagReasons, reachedSite, runFlagReason } from "./run-flags"; + +// Real stored results, trimmed to step id/type/host/status and the answer basis: +// - blocked-first-403: prod run 0cc1a662 (fiverr.com). The first on-site request +// got a 403; only 1 of its 2 on-site requests with a status was refused. +// - memory-answer: prod run 0201b819 (bino.church). memory_share exactly 0.5, the +// most common value among flagged runs; three on-site 404s, no refusal. +// - clean-onsite: local.db run d0a55cc0 (linear.app, a mock replay). All on-site +// requests 200, no run_signals. +// - offsite-only: prod run a7aec281. Asked about linear.com, the agent only ever +// fetched linear.app. +import blockedFirst403 from "./__fixtures__/run-flags/blocked-first-403.json"; +import memoryAnswer from "./__fixtures__/run-flags/memory-answer.json"; +import cleanOnsite from "./__fixtures__/run-flags/clean-onsite.json"; +import offsiteOnly from "./__fixtures__/run-flags/offsite-only.json"; + +const NOT_FLAGGED = { flagged: false }; + +describe("isSiteHost", () => { + it("matches the site itself and any subdomain of it", () => { + expect(isSiteHost("fiverr.com", "fiverr.com")).toBe(true); + expect(isSiteHost("docs.enterprise.fiverr.com", "fiverr.com")).toBe(true); + }); + + it("ignores case on both sides and trims the site", () => { + expect(isSiteHost("Docs.Linear.App", " LINEAR.app ")).toBe(true); + }); + + it("rejects lookalikes, parent domains and sibling brands", () => { + expect(isSiteHost("notfiverr.com", "fiverr.com")).toBe(false); + expect(isSiteHost("fiverr.com.example.io", "fiverr.com")).toBe(false); + expect(isSiteHost("poupaonca.com.br", "staging.poupaonca.com.br")).toBe(false); + expect(isSiteHost("linear.app", "linear.com")).toBe(false); + }); + + it("is false when the host or the site is missing", () => { + expect(isSiteHost(undefined, "fiverr.com")).toBe(false); + expect(isSiteHost("fiverr.com", null)).toBe(false); + expect(isSiteHost("fiverr.com", " ")).toBe(false); + }); + + it("accepts every host the canonical journey domain normalization produces", () => { + for (const raw of [" Stripe.COM ", "linear.app", "ＦＩＶＥＲＲ.com"]) { + expect(isSiteHost(normalizeDomain(raw), raw)).toBe(true); + } + }); +}); + +describe("onSiteSteps and reachedSite", () => { + it("keeps only the steps on the site, in trajectory order", () => { + expect(onSiteSteps(blockedFirst403, blockedFirst403.domain).map((s) => s.id)).toEqual([0, 1, 3]); + expect(reachedSite(blockedFirst403, blockedFirst403.domain)).toBe(true); + }); + + it("reports no reach when the agent only fetched another host", () => { + expect(onSiteSteps(offsiteOnly, offsiteOnly.domain)).toEqual([]); + expect(reachedSite(offsiteOnly, offsiteOnly.domain)).toBe(false); + }); + + it("reports no reach without a site or a trajectory", () => { + expect(reachedSite(cleanOnsite, null)).toBe(false); + expect(reachedSite({}, "linear.app")).toBe(false); + }); +}); + +describe("deriveRunFlags on real runs", () => { + it("flags a run whose first on-site request was refused", () => { + expect(deriveRunFlags(blockedFirst403, blockedFirst403.domain)).toEqual({ + blocked: { flagged: true, reason: "The site refused the agent's first request with HTTP 403" }, + answered_from_memory: NOT_FLAGGED, + }); + }); + + it("flags an answer half drawn from memory, and never counts a 404 as a refusal", () => { + expect(deriveRunFlags(memoryAnswer, memoryAnswer.domain)).toEqual({ + blocked: NOT_FLAGGED, + answered_from_memory: { + flagged: true, + reason: "Half of this answer came from the model's memory, not from pages it fetched", + }, + }); + }); + + it("flags nothing on a clean on-site run without run signals", () => { + expect(deriveRunFlags(cleanOnsite, cleanOnsite.domain)).toEqual({ + blocked: NOT_FLAGGED, + answered_from_memory: NOT_FLAGGED, + }); + }); + + it("flags nothing on a run that never reached the site", () => { + expect(deriveRunFlags(offsiteOnly, offsiteOnly.domain)).toEqual({ + blocked: NOT_FLAGGED, + answered_from_memory: NOT_FLAGGED, + }); + }); +}); + +// Variants of the real fixtures that isolate one rule each; the edit is named +// in every test. +describe("deriveRunFlags rules", () => { + const steps = blockedFirst403.trajectory.steps; + + it("judges the first request that got a status, skipping steps without one", () => { + // fiverr.com with its status-less developers.fiverr.com step moved first. + const reordered = { trajectory: { steps: [steps[1], steps[0], ...steps.slice(2)] } }; + expect(deriveRunFlags(reordered, "fiverr.com").blocked).toEqual({ + flagged: true, + reason: "The site refused the agent's first request with HTTP 403", + }); + }); + + it("does not flag when the first answered request succeeded and only half were refused", () => { + // fiverr.com with the 200 from docs.enterprise.fiverr.com moved first. + const reordered = { trajectory: { steps: [steps[3], steps[0], steps[1]] } }; + expect(deriveRunFlags(reordered, "fiverr.com").blocked).toEqual(NOT_FLAGGED); + }); + + it("flags when more than half of the answered on-site requests were refused", () => { + // bino.church with its three on-site 404s turned into 403s: 3 of 4 refused. + const refused = { + trajectory: { + steps: memoryAnswer.trajectory.steps.map((s) => ("status" in s && s.status === 404 ? { ...s, status: 403 } : s)), + }, + }; + expect(deriveRunFlags(refused, "bino.church").blocked).toEqual({ + flagged: true, + reason: "The site refused 3 of the agent's 4 requests that got a response, with HTTP 403", + }); + }); + + it("never counts a refusal from another host", () => { + // The linear.com run with every linear.app response turned into a 403. + const refusedElsewhere = { + trajectory: { + steps: offsiteOnly.trajectory.steps.map((s) => ("status" in s ? { ...s, status: 403 } : s)), + }, + }; + expect(deriveRunFlags(refusedElsewhere, "linear.com").blocked).toEqual(NOT_FLAGGED); + }); + + it("does not flag memory below half", () => { + expect(blockedFirst403.run_signals.answer_basis.memory_share).toBeLessThan(0.5); + expect(deriveRunFlags(blockedFirst403, "fiverr.com").answered_from_memory).toEqual(NOT_FLAGGED); + }); +}); + +describe("raisedFlagReasons", () => { + it("lists the raised reasons, blocked first, and nothing for a clean run", () => { + expect(raisedFlagReasons(deriveRunFlags(blockedFirst403, blockedFirst403.domain))).toEqual([ + "The site refused the agent's first request with HTTP 403", + ]); + expect(raisedFlagReasons(deriveRunFlags(cleanOnsite, cleanOnsite.domain))).toEqual([]); + expect(raisedFlagReasons(undefined)).toEqual([]); + }); +}); + +describe("runFlagReason", () => { + it("names the status of a refused first request", () => { + expect(runFlagReason({ flag: "blocked", kind: "first_request", status: 429 })).toBe( + "The site refused the agent's first request with HTTP 429", + ); + }); + + it("counts the refusals and lists each refusing status once", () => { + expect( + runFlagReason({ flag: "blocked", kind: "most_requests", refused: 5, answered: 8, statuses: [401, 403] }), + ).toBe("The site refused 5 of the agent's 8 requests that got a response, with HTTP 401 and 403"); + expect( + runFlagReason({ flag: "blocked", kind: "most_requests", refused: 3, answered: 3, statuses: [401, 403, 429] }), + ).toBe("The site refused 3 of the agent's 3 requests that got a response, with HTTP 401, 403 and 429"); + }); + + it("says most only when memory supplied more than half of the answer", () => { + expect(runFlagReason({ flag: "answered_from_memory", memoryShare: 0.67 })).toBe( + "Most of this answer came from the model's memory, not from pages it fetched", + ); + expect(runFlagReason({ flag: "answered_from_memory", memoryShare: 0.5 })).toBe( + "Half of this answer came from the model's memory, not from pages it fetched", + ); + }); +}); + +describe("run-flags source", () => { + // ax deviation: the vendored path, not ora's src/lib/journey/run-flags.ts. + const source = readFileSync(join(process.cwd(), "src/journey-set/vendor/run-flags.ts"), "utf8"); + + it("computes reach from step hosts, never from the engine's anchor signals", () => { + expect(source).not.toMatch(/reached_anchor|anchor_relation/); + }); + + it("stays browser safe: type imports, plus only the browser safe domain normalizer", () => { + const imports = source.split("\n").filter((line) => /^import\s/.test(line)); + const valueImports = imports.filter((line) => !/^import type\s/.test(line)); + expect(valueImports).toEqual(['import { normalizeDomain } from "./domain-normalize";']); + // ax deviation: the vendored path, not ora's src/lib/journey/domain-normalize.ts. + const normalizer = readFileSync(join(process.cwd(), "src/journey-set/vendor/domain-normalize.ts"), "utf8"); + expect(normalizer).not.toMatch(/^import\s/m); + }); +}); diff --git a/src/journey-set/vendor/run-flags.ts b/src/journey-set/vendor/run-flags.ts new file mode 100644 index 0000000..c4552a8 --- /dev/null +++ b/src/journey-set/vendor/run-flags.ts @@ -0,0 +1,131 @@ +// Copied from ora's journey code. DO NOT EDIT BY HAND. +// +// Source: ora's src/lib/journey/run-flags.ts at commit 075e9fe8f. A set's +// "blocked" flag has to mean what the free journey's flag means, so this file +// is a COPY of ora's rule rather than a reimplementation of it. One deviation, +// marked inline: the step type. Re-copy the whole directory whenever ora's +// file changes. Biome does not format this directory. + +/** + * Run flags: two honest notes that sit beside a journey verdict and never + * change it. Derived only from the stored trajectory steps and run signals. + * + * - blocked: the site refused the agent's first request that got an HTTP + * status (401, 403 or 429), or more than half of those requests. Status + * codes only, because the engine sends no signal for bot challenge pages. + * 404 never counts: agents probe for files that do not exist. A step with + * no status (the fetch tool often reports none) is no evidence either way. + * - answered_from_memory: half or more of the answer's sections came from the + * model's memory rather than a page it fetched. + * + * Reach is computed here from each step's host, never from the engine's + * anchor signals: those key on the request's domain, which a run may omit. + * + * Pure and browser safe: the run page derives flags from the live result with + * the same functions REST and MCP use. Type imports only. + */ +import { normalizeDomain } from "./domain-normalize"; +// ax deviation: ora's engine-contract module is not vendored; this is the slice of its +// TrajectoryStep the flags read. Studio trajectories send a null status where ora's omit it. +type TrajectoryStep = { url_host?: string | null; status?: number | null }; + +type FlagStep = Pick; + +/** The fields the flags read. A stored RunResult and its public projection both fit. */ +export type RunFlagsInput = { + trajectory?: { steps?: readonly S[] }; + run_signals?: { answer_basis?: { memory_share: number } }; +}; + +export type RunFlag = { flagged: boolean; reason?: string }; + +export type RunFlags = { blocked: RunFlag; answered_from_memory: RunFlag }; + +/** Why a flag was raised; `runFlagReason` turns it into plain words. */ +export type RunFlagCause = + | { flag: "blocked"; kind: "first_request"; status: number } + | { flag: "blocked"; kind: "most_requests"; refused: number; answered: number; statuses: number[] } + | { flag: "answered_from_memory"; memoryShare: number }; + +const REFUSAL_STATUSES: ReadonlySet = new Set([401, 403, 429]); +const MEMORY_SHARE_THRESHOLD = 0.5; + +/** True when `host` is the site or one of its subdomains. */ +export function isSiteHost(host: string | null | undefined, site: string | null | undefined): boolean { + const normalizedSite = normalizeDomain(site); + const normalizedHost = host?.toLowerCase(); + if (!normalizedSite || !normalizedHost) return false; + return normalizedHost === normalizedSite || normalizedHost.endsWith(`.${normalizedSite}`); +} + +/** The run's steps on the site, in trajectory order. */ +export function onSiteSteps(result: RunFlagsInput, site: string | null | undefined): S[] { + return (result.trajectory?.steps ?? []).filter((step) => isSiteHost(step.url_host, site)); +} + +/** True when any step reached the site. */ +export function reachedSite(result: RunFlagsInput, site: string | null | undefined): boolean { + return (result.trajectory?.steps ?? []).some((step) => isSiteHost(step.url_host, site)); +} + +export function deriveRunFlags(result: RunFlagsInput, site: string | null | undefined): RunFlags { + return { + blocked: toFlag(blockedCause(result, site)), + answered_from_memory: toFlag(memoryCause(result)), + }; +} + +/** The reasons of the raised flags, in a fixed order (blocked first). */ +export function raisedFlagReasons(flags: RunFlags | null | undefined): string[] { + if (!flags) return []; + return [flags.blocked, flags.answered_from_memory].flatMap((flag) => + flag.flagged && flag.reason ? [flag.reason] : [], + ); +} + +/** The one label map from a flag's cause to the words every surface shows. */ +export function runFlagReason(cause: RunFlagCause): string { + if (cause.flag === "answered_from_memory") { + const portion = cause.memoryShare > MEMORY_SHARE_THRESHOLD ? "Most" : "Half"; + return `${portion} of this answer came from the model's memory, not from pages it fetched`; + } + if (cause.kind === "first_request") { + return `The site refused the agent's first request with HTTP ${cause.status}`; + } + return `The site refused ${cause.refused} of the agent's ${cause.answered} requests that got a response, with HTTP ${listStatuses(cause.statuses)}`; +} + +function blockedCause(result: RunFlagsInput, site: string | null | undefined): RunFlagCause | null { + const statuses = onSiteSteps(result, site).flatMap((step) => + typeof step.status === "number" ? [step.status] : [], + ); + if (statuses.length === 0) return null; + if (REFUSAL_STATUSES.has(statuses[0])) { + return { flag: "blocked", kind: "first_request", status: statuses[0] }; + } + const refused = statuses.filter((status) => REFUSAL_STATUSES.has(status)); + if (refused.length * 2 <= statuses.length) return null; + return { + flag: "blocked", + kind: "most_requests", + refused: refused.length, + answered: statuses.length, + statuses: [...new Set(refused)].sort((a, b) => a - b), + }; +} + +function memoryCause(result: RunFlagsInput): RunFlagCause | null { + const memoryShare = result.run_signals?.answer_basis?.memory_share; + if (typeof memoryShare !== "number" || memoryShare < MEMORY_SHARE_THRESHOLD) return null; + return { flag: "answered_from_memory", memoryShare }; +} + +function toFlag(cause: RunFlagCause | null): RunFlag { + return cause ? { flagged: true, reason: runFlagReason(cause) } : { flagged: false }; +} + +/** [403] -> "403", [401, 403] -> "401 and 403", [401, 403, 429] -> "401, 403 and 429". */ +function listStatuses(statuses: readonly number[]): string { + if (statuses.length <= 1) return statuses.join(""); + return `${statuses.slice(0, -1).join(", ")} and ${statuses[statuses.length - 1]}`; +} diff --git a/src/main.ts b/src/main.ts index 8a70ca1..8a6b2d2 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,13 +1,17 @@ -import { defineCommand, runMain, showUsage } from "citty"; +import { defineCommand, renderUsage, runMain, showUsage } from "citty"; import pc from "picocolors"; import pkg from "../package.json"; +import { platformBase } from "./api/platform"; +import { resolveBase } from "./api/skills"; import { auditCommand } from "./commands/audit"; import { deepJourneyCommand } from "./commands/deep-journey"; import { generateFilesCommand } from "./commands/generate-files"; -import { journeyCommand } from "./commands/journey"; +import { journey, journeyUsage, noStreamFlag } from "./commands/journey-cli"; +import { learnCommand } from "./commands/learn"; import { skillCommand } from "./commands/skill"; import { webmcpAuditCommand } from "./commands/webmcp-audit"; import { loadLocalEnv } from "./env"; +import { axCall, type NextStep, placeholder, printNext } from "./report/next-line"; const NAME = "ax"; @@ -86,43 +90,14 @@ const audit = defineCommand({ }, }); -const journey = defineCommand({ - meta: { - name: "journey", - description: "Send a real AI agent at a site and watch it navigate live", - }, - args: { - intent: { - type: "positional", - description: 'Goal for the agent (e.g. "Find the API docs and how to auth")', - required: true, - }, - domain: { type: "string", description: "Site the agent targets (e.g. stripe.com)" }, - harness: { - type: "string", - description: "Agent harness: claude-code, codex, openclaw, hermess, claude-agent, eve, …", - default: "claude-code", - }, - model: { type: "string", description: "Model override (harness default when omitted)" }, - json: { type: "boolean", description: "Output as JSON", default: false }, - }, - async run({ args }) { - // Same exitCode-over-exit rule as audit: let stdout drain before exiting. - process.exitCode = await journeyCommand({ - intent: args.intent as string, - domain: args.domain as string | undefined, - harness: args.harness as string, - model: args.model as string | undefined, - json: Boolean(args.json), - }); - }, -}); - +// The older name for `ax journey` with no goal of its own. It keeps working, flags and exit codes +// unchanged, with a note on stderr naming the new call, until a future major release removes it. const deepJourney = defineCommand({ meta: { name: "deep-journey", description: - "Run a real AI agent at a site on a curated task — or, with a partner API key, a free-text one — via ora's public API (exit codes: 0 ok, 1 run failed, 2 usage, 3 API error)", + "Deprecated: now ax journey --domain . Runs a real AI agent at a site on a curated task, or with a partner API key a free-text one, via ora's public API (exit codes: 0 ok, 1 run failed, 2 usage, 3 API error)", + hidden: true, }, args: { url: { @@ -161,6 +136,22 @@ const deepJourney = defineCommand({ }, }, async run({ args }) { + const task = args.task as string | undefined; + const intent = args.intent as string | undefined; + const agent = args.agent as string | undefined; + const now = axCall([ + "journey", + ...(task ? [task] : []), + "--domain", + args.url as string, + ...(intent && !task ? ["--intent", intent] : []), + ...(agent ? ["--agent", agent] : []), + ]); + console.error( + pc.dim( + `ax deep-journey is now ${now}. This name keeps working and will be removed in a future major release.`, + ), + ); // Same exitCode-over-exit rule as audit: let stdout drain before exiting. process.exitCode = await deepJourneyCommand({ url: args.url as string, @@ -169,7 +160,8 @@ const deepJourney = defineCommand({ apiKey: args["api-key"] as string | undefined, agent: args.agent as string | undefined, json: Boolean(args.json), - noStream: Boolean(args["no-stream"]), + noStream: noStreamFlag(args), + via: "deep-journey", }); }, }); @@ -296,6 +288,102 @@ const webmcpAudit = defineCommand({ }, }); +const learn = defineCommand({ + meta: { + name: "learn", + description: + "Print the ax workflow from ora's skill: score a site, read the report, run a journey, read its result", + }, + args: {}, + async run() { + process.exitCode = await learnCommand(); + }, +}); + +// Read at help time, after loadLocalEnv(), so help names the hosts this run +// would actually call. +const oraSite = () => `ora.ai at ${resolveBase()} (set by ORA_API_URL)`; +const frontAccount = () => + `your Front account at app.ora.ai, through ${platformBase()} (set by ORA_PLATFORM_URL), with ORA_API_KEY`; + +/** The chain every help screen ends with; `ax learn` prints the skill's own copy. */ +function workflowLines(): string[] { + const d = pc.dim; + return [ + ` ${d("Workflow:")}`, + ` 1. ax audit ${d("score the site and list what to fix")}`, + ` 2. read the report ${d("start with its Top fixes, ranked by ora")}`, + ` 3. ax journey --domain ${d("send a real agent at the site on a task")}`, + ` 4. read its result ${d("whether the agent finished, and where it got stuck")}`, + ` ${d("Every result and error ends with a Next: line, the call to make next.")}`, + ` ${d("ax learn prints these steps from ora's skill.")}`, + "", + ]; +} + +interface CommandHelp { + talksTo: () => string; + /** Replaces citty's generated usage, for a command whose forms it cannot show. */ + usage?: () => string; + /** The call to suggest when citty rejects the arguments (a missing positional). */ + usageNext: NextStep; +} + +const COMMAND_HELP = new Map([ + [ + audit, + { + talksTo: () => `${oraSite()}; with --tunnel ora, also ${frontAccount()}`, + usageNext: { call: axCall(["audit", placeholder("domain")]) }, + }, + ], + [ + webmcpAudit, + { + talksTo: () => `a Chrome on this machine for the capture, then ${oraSite()} to score it`, + usageNext: { call: axCall(["webmcp-audit", "http://localhost:3000"]) }, + }, + ], + [ + deepJourney, + { + talksTo: oraSite, + usageNext: { call: axCall(["journey", "--domain", placeholder("domain")]) }, + }, + ], + [ + journey, + { + talksTo: () => + `${oraSite()} with no goal of your own (free); ${frontAccount()} with a goal; journey list reads the set records in ~/.ax/journeys and calls nothing`, + usage: journeyUsage, + usageNext: { + call: axCall(["journey", placeholder("goal"), "--domain", placeholder("domain")]), + why: "quote the goal", + }, + }, + ], + [ + generateFiles, + { + talksTo: oraSite, + usageNext: { call: axCall(["generate-files", placeholder("domain")]) }, + }, + ], + [ + skill, + { talksTo: () => `ora's skill registry on ${oraSite()}`, usageNext: { call: "ax skill" } }, + ], + [ + learn, + { talksTo: () => `ora's skill registry on ${oraSite()}`, usageNext: { call: "ax learn" } }, + ], +]); + +function commandFooter(help: CommandHelp): string { + return [` ${pc.dim("Talks to:")} ${help.talksTo()}`, "", ...workflowLines()].join("\n"); +} + function helpScreen(): string { const g = pc.green("$"); const d = pc.dim; @@ -307,10 +395,18 @@ function helpScreen(): string { ` ${d("Commands:")}`, ` audit ${d("")} ${d("Score a site's agent readiness")}`, ` webmcp-audit ${d("")} ${d("Audit a WebMCP surface in a real browser, localhost included")}`, - ` deep-journey ${d("")} ${d("Run a real agent at a site on a curated or free-text task")}`, - ` journey ${d("")} ${d("Send a real agent at a site and watch it live (needs a platform key)")}`, + ` journey ${d("--domain ")} ${d("Send a real agent at a site on one of ora's curated tasks, drawn live; free, no key")}`, + ` journey ${d("")} ${d("The same on a goal of your own; --levels, --agents or --repeat run a set of journeys and count the verdicts (needs a Front API key)")}`, + ` journey ${d("view|list|cancel")} ${d("Pick up, list, or stop a recorded set of journeys")}`, ` skill ${d("[name]")} ${d("List, print, or install ora's agent skills")}`, ` generate-files ${d("")} ${d("Draft a domain's agent files: ARD catalog, poppy.json, llms.txt, robots.txt edit, …")}`, + ` learn ${d("Print the ax workflow from ora's skill")}`, + "", + ` ${d("Talks to:")}`, + ` ${oraSite()}`, + ` ${d("audit, webmcp-audit, journey with no goal, generate-files, skill, learn")}`, + ` ${frontAccount()}`, + ` ${d("journey with a goal, and audit --tunnel ora")}`, "", ` ${d("Examples:")}`, ` ${g} ${NAME} audit https://docs.example.com`, @@ -318,9 +414,11 @@ function helpScreen(): string { ` ${g} ${NAME} audit https://docs.example.com --json`, ` ${g} ${NAME} audit localhost:3000 --tunnel ora ${d("# audit a local dev server")}`, ` ${g} ${NAME} webmcp-audit http://localhost:3000`, - ` ${g} ${NAME} deep-journey stripe.com --intent pricing`, + ` ${g} ${NAME} journey --domain stripe.com --intent pricing ${d("# free, no key")}`, ` ${g} ${NAME} journey "Find the API docs and how to authenticate" --domain stripe.com`, + ` ${g} ${NAME} journey "Find how to create an issue from the API" --domain linear.app --exact https://linear.app/developers --repeat 3`, ` ${g} ${NAME} generate-files example.com --out public`, + ` ${g} ${NAME} learn`, "", ` ${d("audit options:")}`, ` --min-score ${d("Exit 1 when the score is below n (0-100); the CI gate")}`, @@ -346,16 +444,6 @@ function helpScreen(): string { ` ${d("scored by ora, stored nowhere: the result is never published or ranked")}`, ` ${d("exit codes: 0 ok · 1 below --min-score · 2 usage · 3 API error")}`, "", - ` ${d("deep-journey options:")}`, - ` --intent ${d("Curated task id, e.g. pricing, signup, api-docs (server default when omitted)")}`, - ` --task ${d("Free-text task (needs a partner API key; replaces --intent)")}`, - ` --api-key ${d("ora partner API key: unlocks --task + 1000 runs/24h; also read from ORA_PARTNER_API_KEY, then ORA_SCAN_API_KEY")}`, - ` --agent ${d("Agent to run (default: cas-haiku — Claude Code on the Haiku model ora's roster lists)")}`, - ` --no-stream ${d("Hide the live trajectory view and print only the final result")}`, - ` --json ${d("Print the terminal run detail as JSON")}`, - ` ${d("no key needed · public caps: 100 runs/24h per target and task, 20 runs/24h per IP")}`, - ` ${d("exit codes: 0 run succeeded (any verdict) · 1 run failed · 2 usage · 3 API unreachable/rate-limited")}`, - "", ` ${d("generate-files options:")}`, ` --out ${d("Your site's web root: writes .well-known/*, robots.txt, llms.txt there (default: .)")}`, ` --force ${d("Overwrite files that already exist (otherwise nothing is written)")}`, @@ -368,13 +456,37 @@ function helpScreen(): string { ` --dir ${d("Install directory (default .claude/skills)")}`, ` --json ${d("Print the raw registry index as JSON")}`, "", - ` ${d("journey options:")}`, - ` --domain ${d("Site the agent targets (e.g. stripe.com)")}`, + ` ${d("journey options, free (no goal of your own):")}`, + ` --domain ${d("Site the agent works on (e.g. stripe.com)")}`, + ` --intent ${d("Curated task id, e.g. pricing, signup, api-docs (server default when omitted)")}`, + ` --agent ${d("Agent to run (default: cas-haiku, Claude Code on the Haiku model ora's roster lists)")}`, + ` --api-key ${d("ora partner API key: a goal of your own + 1000 runs/24h; also read from ORA_PARTNER_API_KEY, then ORA_SCAN_API_KEY")}`, + ` --no-stream ${d("Hide the live trajectory view and print only the final result")}`, + ` --json ${d("Print the terminal run detail as JSON")}`, + ` ${d("public caps: 100 runs/24h per target and task, 20 runs/24h per IP")}`, + ` ${d("exit codes: 0 run succeeded (any verdict) · 1 run failed · 2 usage · 3 API unreachable/rate-limited")}`, + ` ${d("ax deep-journey still runs this, and will be removed in a future major release")}`, + "", + ` ${d("journey options, on your Front account (a goal of your own):")}`, + ` --domain ${d("Site the agent works on (e.g. stripe.com); a set of journeys needs it")}`, ` --harness ${d("Agent harness: claude-code (default), codex, openclaw, hermess, claude-agent, eve, …")}`, ` --model ${d("Model override (harness default when omitted)")}`, + ` --agents ${d("Agents as harness or harness/model, comma separated (up to 4; a set defaults to your plan's Claude Code agent)")}`, + ` --levels ${d("Run each goal at the levels of help: need only, brand named, docs linked, exact target")}`, + ` --brand ${d("Brand for the brand level (default: the site's name); turns levels on")}`, + ` --docs ${d("Documentation URL; adds the docs level and turns levels on")}`, + ` --exact ${d("URL the agent starts at; adds the exact level and turns levels on")}`, + ` --repeat ${d("Runs for each goal, level and agent (1 to 10; default 1)")}`, + ` --yes ${d("Launch a set without asking after the preview")}`, + ` --wait ${d("Minutes to follow a set's runs before printing what has finished (default 60)")}`, ` --json ${d("Output as JSON")}`, - ` ${d("requires ORA_API_KEY (see .env.example)")}`, + ` ${d("one goal and one agent run one journey, drawn live; more goals or agents, --repeat or levels run a set")}`, + ` ${d("ax journey view [--wait ] [--json] ax journey list [--json] ax journey cancel [--json]")}`, + ` ${d("requires ORA_API_KEY, a Front API key (see .env.example); sign up at app.ora.ai")}`, + ` ${d("exit codes, one journey: 0 success · 1 finished otherwise · 2 error")}`, + ` ${d("exit codes, a set: 0 report printed · 2 usage or refusal before launch · 3 API error · 130 Ctrl-C")}`, "", + ...workflowLines(), ].join("\n"); } @@ -391,6 +503,7 @@ const root = defineCommand({ journey, skill, "generate-files": generateFiles, + learn, }, setup() { // Bare invocation gets the curated screen. Top-level --help is handled in @@ -406,12 +519,29 @@ const root = defineCommand({ // at call time, so the file has to be in place first. loadLocalEnv(); +const HELP_FLAGS = new Set(["--help", "-h"]); + // citty intercepts --help itself and prints its generated usage, so the root // command's help is routed to the curated screen here. Subcommand --help -// (`ax audit --help`) keeps citty's per-command usage. +// (`ax audit --help`) keeps citty's per-command usage, or the command's own +// (`ax journey`), followed by what the command talks to and the workflow. +// citty also lands here when it rejects the arguments (a missing positional), +// before printing its error and exiting 1; that path gets a Next line on +// stderr instead of the footer. runMain(root, { showUsage: async (cmd, parent) => { + const asked = process.argv.slice(2).some((arg) => HELP_FLAGS.has(arg)); + const help = COMMAND_HELP.get(cmd); if (cmd === root) console.log(helpScreen()); + else if (asked && help) { + const usage = help.usage ? help.usage() : (await renderUsage(cmd, parent)).trimEnd(); + console.log(`${usage}\n\n${commandFooter(help)}`); + } else if (help?.usage) console.log(`${help.usage()}\n`); else await showUsage(cmd, parent); + if (!asked) { + printNext(help?.usageNext ?? { call: axCall(["audit", placeholder("domain")]) }, { + stderr: true, + }); + } }, }); diff --git a/src/report/json-error.test.ts b/src/report/json-error.test.ts new file mode 100644 index 0000000..671a466 --- /dev/null +++ b/src/report/json-error.test.ts @@ -0,0 +1,105 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { jsonErrorEnvelope, reportError } from "./json-error"; + +const ERROR = { + code: "INVALID_TARGET", + message: 'Not a scannable URL or domain: "not-a-domain"', + next: { call: "ax audit ", why: "a public URL or domain" }, +}; + +beforeEach(() => { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("jsonErrorEnvelope", () => { + it("carries the code, the message, and the next call without its reason", () => { + expect(jsonErrorEnvelope(ERROR)).toEqual({ + error: { + code: "INVALID_TARGET", + message: 'Not a scannable URL or domain: "not-a-domain"', + next: "ax audit ", + }, + }); + }); +}); + +describe("reportError", () => { + it("under --json writes only the envelope to stdout and the notes to stderr", () => { + reportError(ERROR, { json: true }); + expect(process.stdout.write).toHaveBeenCalledTimes(1); + expect(JSON.parse(String(vi.mocked(process.stdout.write).mock.calls[0][0]))).toEqual( + jsonErrorEnvelope(ERROR), + ); + expect(console.log).not.toHaveBeenCalled(); + expect(vi.mocked(console.error).mock.calls).toEqual([ + ['Not a scannable URL or domain: "not-a-domain"'], + ["Next: ax audit # a public URL or domain"], + ]); + }); + + it("without --json writes nothing to stdout", () => { + reportError(ERROR, { json: false, display: () => "Audit failed: bad target" }); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(console.log).not.toHaveBeenCalled(); + expect(vi.mocked(console.error).mock.calls).toEqual([ + ["Audit failed: bad target"], + ["Next: ax audit # a public URL or domain"], + ]); + }); + + it("under --json passes a served error body through unchanged instead of the envelope", () => { + const served = { + error: "invalid_domain", + message: "not a public domain", + next_action: { call: "ax audit " }, + }; + reportError(ERROR, { json: true, served }); + expect(process.stdout.write).toHaveBeenCalledTimes(1); + expect(process.stdout.write).toHaveBeenCalledWith(`${JSON.stringify(served, null, 2)}\n`); + expect(console.log).not.toHaveBeenCalled(); + expect(vi.mocked(console.error).mock.calls).toEqual([ + ['Not a scannable URL or domain: "not-a-domain"'], + ["Next: ax audit # a public URL or domain"], + ]); + }); + + it("under --json falls back to the envelope when the served body is null", () => { + reportError(ERROR, { json: true, served: null }); + expect(JSON.parse(String(vi.mocked(process.stdout.write).mock.calls[0][0]))).toEqual( + jsonErrorEnvelope(ERROR), + ); + }); + + it("without --json ignores a served body", () => { + reportError(ERROR, { json: false, served: { error: "invalid_domain" } }); + expect(process.stdout.write).not.toHaveBeenCalled(); + }); + + // A message can carry text the server wrote. On a terminal, an escape sequence in it would + // run (clear the screen, set the clipboard) instead of being read. + it("drops terminal control bytes from the message and keeps its line breaks", () => { + const message = "bad\u001b]52;c;cGF5bG9hZA==\u0007 target\r\n second line"; + reportError({ ...ERROR, message }, { json: false }); + const printed = String(vi.mocked(console.error).mock.calls[0][0]); + for (const control of ["\u001b", "\u0007", "\r"]) expect(printed).not.toContain(control); + expect(printed).toContain("\n second line"); + }); + + it("hands display the cleaned message, so the caller's colour survives and the server's does not", () => { + const message = "bad\u001b[2J target"; + reportError( + { ...ERROR, message }, + { json: false, display: (m) => `\u001b[31mfailed: ${m}\u001b[39m` }, + ); + const printed = String(vi.mocked(console.error).mock.calls[0][0]); + expect(printed.startsWith("\u001b[31mfailed: bad")).toBe(true); + expect(printed.endsWith("\u001b[39m")).toBe(true); + expect(printed).not.toContain("\u001b[2J"); + }); +}); diff --git a/src/report/json-error.ts b/src/report/json-error.ts new file mode 100644 index 0000000..3845376 --- /dev/null +++ b/src/report/json-error.ts @@ -0,0 +1,50 @@ +import { plainLines } from "../ui/ansi"; +import { type NextStep, printNext } from "./next-line"; + +// Under --json, stdout carries one JSON document and nothing else. When ora +// (or the platform) answered with an error body, that body is the document, +// unchanged: it is the contract, the same as a result payload (invariant 4). +// Only a failure with no server body to show (a usage mistake caught here, a +// network failure, a timeout, a response that did not parse) gets this +// envelope instead. Every line written for a person (the error itself, the +// Next line, progress, warnings) goes to stderr, in both modes. + +export interface CommandError { + /** ora's code when it named one, otherwise the CLI's own (e.g. INVALID_TARGET). */ + code: string; + /** What went wrong, as one human message. */ + message: string; + next: NextStep; +} + +export interface JsonErrorEnvelope { + error: { code: string; message: string; next: string }; +} + +export function jsonErrorEnvelope(error: CommandError): JsonErrorEnvelope { + return { error: { code: error.code, message: error.message, next: error.next.call } }; +} + +/** + * Report a failed command: the message (or `display`, its painted or prefixed + * form) and the Next line on stderr. Under --json, stdout gets `served`, the + * error body the server answered with, when there is one (a body that parsed + * to null carries nothing to pass through), and the envelope otherwise. + * + * The message can quote a server, so stderr gets it with the terminal control + * bytes dropped, and `display` paints that cleaned text rather than the raw one. + */ +export function reportError( + error: CommandError, + options: { json: boolean; display?: (message: string) => string; served?: unknown }, +): void { + const message = plainLines(error.message); + console.error(options.display ? options.display(message) : message); + printNext(error.next, { stderr: true }); + if (!options.json) return; + const document = + options.served === undefined || options.served === null + ? jsonErrorEnvelope(error) + : options.served; + process.stdout.write(`${JSON.stringify(document, null, 2)}\n`); +} diff --git a/src/report/next-line.test.ts b/src/report/next-line.test.ts new file mode 100644 index 0000000..20a127a --- /dev/null +++ b/src/report/next-line.test.ts @@ -0,0 +1,136 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import runningDetail from "../api/__fixtures__/deep-journey-detail-running.json"; +import refusals from "../api/__fixtures__/deep-journey-refusals.json"; +import type { JourneyNextAction } from "../contract"; +import { axCall, nextLine, placeholder, printNext, servedNextStep, shellWord } from "./next-line"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("shellWord", () => { + it("leaves a word bare when a shell would read it unchanged", () => { + expect(shellWord("stripe.com")).toBe("stripe.com"); + expect(shellWord("https://docs.example.com/v2")).toBe("https://docs.example.com/v2"); + expect(shellWord("--min-score")).toBe("--min-score"); + }); + + it("single quotes anything a shell would split or expand", () => { + expect(shellWord("Find pricing")).toBe("'Find pricing'"); + expect(shellWord("$HOME")).toBe("'$HOME'"); + expect(shellWord("ngrok http 3000 --log stdout")).toBe("'ngrok http 3000 --log stdout'"); + }); + + it("escapes a single quote inside the word", () => { + expect(shellWord("what's new")).toBe("'what'\\''s new'"); + }); +}); + +describe("axCall", () => { + it("renders placeholders unquoted and real arguments quoted", () => { + expect(axCall(["deep-journey", "stripe.com", "--task", "Find the pricing page"])).toBe( + "ax deep-journey stripe.com --task 'Find the pricing page'", + ); + expect(axCall(["audit", placeholder("domain")])).toBe("ax audit "); + expect(axCall(["audit", "stripe.com", "--api-key", placeholder("key")])).toBe( + "ax audit stripe.com --api-key ", + ); + }); +}); + +describe("nextLine", () => { + it("puts the reason after the call as a shell comment", () => { + expect(nextLine({ call: "ax audit stripe.com", why: "score the site" })).toBe( + "Next: ax audit stripe.com # score the site", + ); + expect(nextLine({ call: "ax learn" })).toBe("Next: ax learn"); + }); + + it("blanks control characters a server value could carry", () => { + expect(nextLine({ call: "ax audit evil.com\u001b[2J", why: "a\nb" })).toBe( + "Next: ax audit evil.com [2J # a b", + ); + }); +}); + +describe("servedNextStep", () => { + // Next actions as ora served them (real responses, see src/api/deep-journey.test.ts). + const STREAM = runningDetail.next_action as JourneyNextAction; + const INVALID_DOMAIN = refusals.INVALID_DOMAIN.next_action as JourneyNextAction; + const AGENTS = refusals.UNSUPPORTED_AGENT.next_action as JourneyNextAction; + const KEY_DOCS = refusals.CUSTOM_INTENT_REQUIRES_KEY.next_action as JourneyNextAction; + const SAME = { + method: "POST", + url: "/api/journey/runs", + call: "ax deep-journey linear.app --intent pricing", + }; + + it("writes a GET as curl against the base, with ora's reason as the why", () => { + expect(servedNextStep(STREAM, { base: "https://ora.ai" })).toEqual({ + call: `curl -s https://ora.ai${runningDetail.stream_url}`, + why: "Hold the stream open to the end; only a followed stream saves the result.", + }); + expect(servedNextStep(AGENTS, { base: "http://localhost:4214", axEquivalent: SAME })).toEqual({ + call: "curl -s http://localhost:4214/api/journey/agents", + why: AGENTS.reason, + }); + }); + + it("keeps an absolute API URL as served", () => { + const api = { ...AGENTS, url: "https://ora.ai/api/journey/agents" }; + expect(servedNextStep(api, { base: "http://localhost:4214" }).call).toBe( + "curl -s https://ora.ai/api/journey/agents", + ); + }); + + it("never curls a docs page: the call is the ax call to make after reading it, the page goes in the why", () => { + const keyed = + "ORA_PARTNER_API_KEY= ax deep-journey linear.app --task 'find pricing'"; + expect(servedNextStep(KEY_DOCS, { base: "http://localhost:4214", pageCall: keyed })).toEqual({ + call: keyed, + why: `${KEY_DOCS.reason} ${KEY_DOCS.url}`, + }); + expect(servedNextStep(KEY_DOCS, { base: "http://localhost:4214" })).toEqual({ + call: KEY_DOCS.url, + why: KEY_DOCS.reason, + }); + }); + + it("writes the request ax itself sends as that ax call", () => { + expect(servedNextStep(INVALID_DOMAIN, { base: "https://ora.ai", axEquivalent: SAME })).toEqual({ + call: SAME.call, + why: INVALID_DOMAIN.reason, + }); + }); + + it("puts the served wait in front as a sleep", () => { + const retry = { ...INVALID_DOMAIN, wait_seconds: 30 }; + expect(servedNextStep(retry, { base: "https://ora.ai", axEquivalent: SAME }).call).toBe( + `sleep 30 && ${SAME.call}`, + ); + const odd = { ...INVALID_DOMAIN, wait_seconds: 1.5 }; + expect(servedNextStep(odd, { base: "https://ora.ai", axEquivalent: SAME }).call).toBe( + SAME.call, + ); + }); + + it("writes a POST with no ax equivalent as curl, its body quoted", () => { + const replay = { ...INVALID_DOMAIN, body: { intent: { custom: "what's new" } } }; + expect(servedNextStep(replay, { base: "https://ora.ai" }).call).toBe( + `curl -s -X POST https://ora.ai/api/journey/runs -H 'content-type: application/json' -d '{"intent":{"custom":"what'\\''s new"}}'`, + ); + }); +}); + +describe("printNext", () => { + it("goes to stdout after a report and to stderr when stdout carries a payload", () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + printNext({ call: "ax learn" }, { stderr: false }); + expect(log).toHaveBeenCalledWith("Next: ax learn"); + expect(error).not.toHaveBeenCalled(); + printNext({ call: "ax learn" }, { stderr: true }); + expect(error).toHaveBeenCalledWith("Next: ax learn"); + expect(log).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/report/next-line.ts b/src/report/next-line.ts new file mode 100644 index 0000000..8128bbc --- /dev/null +++ b/src/report/next-line.ts @@ -0,0 +1,111 @@ +import type { JourneyNextAction } from "../contract"; +import { plain } from "../ui/ansi"; + +// The one renderer for the line that ends every ax result and error: the next +// call to make, written out with its arguments so an agent can run it as-is. +// A value the CLI cannot know (the user's domain, a key) is a ; +// a secret the user passed is never echoed, only its placeholder. When ora +// served a `next_action`, that request is the next call (servedNextStep); +// the CLI's own suggestion is only for payloads that carry none. + +export interface NextStep { + /** The call, e.g. `ax journey --domain stripe.com --intent pricing`. */ + call: string; + /** Why, in one line; printed as a shell comment after the call. */ + why?: string; +} + +/** A word the user fills in, shown as `` and never shell-quoted. */ +export interface Placeholder { + placeholder: string; +} + +export type Word = string | Placeholder; + +export function placeholder(name: string): Placeholder { + return { placeholder: name }; +} + +const SHELL_SAFE = /^[A-Za-z0-9_@%+=:,./-]+$/; + +/** One shell word: bare when it needs no quoting, single quoted otherwise. */ +export function shellWord(value: string): string { + if (SHELL_SAFE.test(value)) return value; + return `'${value.replaceAll("'", `'\\''`)}'`; +} + +/** An `ax` invocation, each real argument quoted the way a shell needs it. */ +export function axCall(words: readonly Word[]): string { + const rendered = words.map((word) => + typeof word === "string" ? shellWord(word) : `<${word.placeholder}>`, + ); + return ["ax", ...rendered].join(" "); +} + +export interface ServedCallOptions { + /** The origin a relative API path in the action resolves against. */ + base: string; + /** + * The ax call that sends `method url` itself. A request ax makes is + * written as that ax call, which also follows what it starts; any other + * request is written as curl. + */ + axEquivalent?: { method: string; url: string; call: string }; + /** + * The ax call to make once a page ora links to has been read (a docs + * page, never an API request). Without one, the page URL is the call. + */ + pageCall?: string; +} + +/** A page to read rather than a request to send: an absolute URL off the API. */ +function isPage(action: JourneyNextAction): boolean { + if (action.method !== "GET" || !/^https?:\/\//.test(action.url)) return false; + return !new URL(action.url).pathname.startsWith("/api/"); +} + +function curlCall(action: JourneyNextAction, base: string): string { + const url = /^https?:\/\//.test(action.url) ? action.url : `${base}${action.url}`; + if (action.method === "GET") return `curl -s ${shellWord(url)}`; + const body = action.body + ? ` -H ${shellWord("content-type: application/json")} -d ${shellWord(JSON.stringify(action.body))}` + : ""; + return `curl -s -X ${shellWord(action.method)} ${shellWord(url)}${body}`; +} + +/** + * The Next step for a next action ora served (a `next_action` on a journey + * payload): its request as a call, its wait as a `sleep` in front, and ora's + * reason, verbatim, as the why. + */ +export function servedNextStep(action: JourneyNextAction, options: ServedCallOptions): NextStep { + const same = options.axEquivalent; + if (isPage(action)) { + return options.pageCall + ? { call: options.pageCall, why: `${action.reason} ${action.url}` } + : { call: action.url, why: action.reason }; + } + const call = + same && action.method === same.method && action.url === same.url + ? same.call + : curlCall(action, options.base); + const seconds = action.wait_seconds; + const wait = Number.isInteger(seconds) && (seconds ?? 0) > 0 ? `sleep ${seconds} && ` : ""; + return { call: `${wait}${call}`, why: action.reason }; +} + +/** `Next: `, plus ` # ` when there is one. Control characters are blanked. */ +export function nextLine(step: NextStep): string { + const line = step.why ? `Next: ${step.call} # ${step.why}` : `Next: ${step.call}`; + return plain(line); +} + +/** + * Print the Next line: on stdout after a report rendered for a person, on + * stderr whenever stdout carries a payload (`--json`, raw skill bytes) that + * must stay clean. + */ +export function printNext(step: NextStep, options: { stderr: boolean }): void { + if (options.stderr) console.error(nextLine(step)); + else console.log(nextLine(step)); +} diff --git a/src/tunnel.ts b/src/tunnel.ts index 122fd82..d4f547c 100644 --- a/src/tunnel.ts +++ b/src/tunnel.ts @@ -8,8 +8,19 @@ import { spawn } from "node:child_process"; // already has works, as long as it prints its public URL to stdout or stderr // (e.g. `ngrok http 3000 --log stdout`). -/** Tunnel setup failure - a usage-class error (exit 2), never an API error. */ -export class TunnelError extends Error {} +/** + * Tunnel setup failure - a usage-class error (exit 2), never an API error. + * `payload` is the platform's error body exactly as served when a control + * plane call failed with JSON (what `--json` prints), undefined otherwise. + */ +export class TunnelError extends Error { + readonly payload: unknown; + + constructor(message: string, options: { payload?: unknown } = {}) { + super(message); + this.payload = options.payload; + } +} export interface Tunnel { /** The public https origin the tunnel serves. */ diff --git a/src/tunnel/ora.ts b/src/tunnel/ora.ts index f16c289..6e6ff80 100644 --- a/src/tunnel/ora.ts +++ b/src/tunnel/ora.ts @@ -2,7 +2,7 @@ import { randomBytes } from "node:crypto"; import { once } from "node:events"; import { type ClientRequest, request as httpRequest, type IncomingMessage } from "node:http"; import pkg from "../../package.json"; -import { exchangeKey, platformBase } from "../api/platform"; +import { exchangeKey, PlatformApiError, platformBase } from "../api/platform"; import { type Tunnel, TunnelError, waitRoutable } from "../tunnel"; import { type HeaderMap, @@ -109,6 +109,7 @@ export async function openOraTunnel( } catch (cause) { throw new TunnelError( `${cause instanceof Error ? cause.message : String(cause)}\n(--tunnel ora needs a platform key with the ${ORA_TUNNEL_SCOPES} scopes)`, + { payload: cause instanceof PlatformApiError ? cause.payload : undefined }, ); } if (opts.signal?.aborted) throw interrupted(); @@ -204,14 +205,18 @@ async function createTunnel(base: string, bearer: string): Promise & { error?: string; message?: string } = {}; + let served: unknown; try { payload = JSON.parse(text); + served = payload; } catch { // non-JSON error page - the status alone is the message } if (!res.ok) { const detail = payload.error ?? payload.message ?? text.trim().slice(0, 200); - throw new TunnelError(`${createHint(res.status)}${detail ? `: ${detail}` : ""}`); + throw new TunnelError(`${createHint(res.status)}${detail ? `: ${detail}` : ""}`, { + payload: served, + }); } // Without an id there is nothing to delete and nothing to connect; every // other shape problem is checked after the caller can delete the row. diff --git a/src/ui/ansi.ts b/src/ui/ansi.ts index 009e430..61ee4e0 100644 --- a/src/ui/ansi.ts +++ b/src/ui/ansi.ts @@ -57,3 +57,9 @@ export function plain(value: string): string { // biome-ignore lint/suspicious/noControlCharactersInRegex: removing them is the point return value.replace(/[\u0000-\u001f\u007f-\u009f]/g, " "); } + +/** `plain` for a message printed as it is, not reflowed: tabs and line breaks stay. */ +export function plainLines(value: string): string { + // biome-ignore lint/suspicious/noControlCharactersInRegex: removing them is the point + return value.replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/g, " "); +} diff --git a/src/webmcp/ingest.test.ts b/src/webmcp/ingest.test.ts index 5285aeb..7446634 100644 --- a/src/webmcp/ingest.test.ts +++ b/src/webmcp/ingest.test.ts @@ -1,5 +1,6 @@ import { readFileSync } from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { KeyDestinationError } from "../api/key-destination"; import { ingestCapture, preflightCapture, prepareForIngest, WebmcpIngestError } from "./ingest"; import type { PublicWebmcpAudit } from "./vendor/projection"; import type { WebmcpAuditEvent, WebmcpCaptureResult } from "./vendor/types"; @@ -175,6 +176,10 @@ describe("ingestCapture", () => { expect(err.code).toBe("SHIM_VERSION_MISMATCH"); expect(err.message).toMatch(/Upgrade/); expect(err.message).not.toMatch(/ingest failed/); + expect(err.payload).toEqual({ + code: "SHIM_VERSION_MISMATCH", + error: "capture is shim 1, server scores 3", + }); }); it("surfaces retry_after_ms from a 429", async () => { @@ -231,3 +236,17 @@ describe("ingestCapture", () => { ); }); }); + +describe("ingestCapture key destination", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("never sends an api key over plain http to a host other than this machine", async () => { + const fetchMock = vi.fn(async () => sse([{ type: "done", audit: AUDIT }])); + vi.stubGlobal("fetch", fetchMock); + const keyed = { apiKey: "sk-test", baseUrl: "http://collector.example" }; + await expect(ingestCapture("http://localhost:3000", CAPTURE, keyed)).rejects.toThrow( + KeyDestinationError, + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/webmcp/ingest.ts b/src/webmcp/ingest.ts index 43aff3e..fb66aa1 100644 --- a/src/webmcp/ingest.ts +++ b/src/webmcp/ingest.ts @@ -17,6 +17,7 @@ * server would reject anyway. */ +import { assertKeyDestination } from "../api/key-destination"; import { errorBodyText, watchdog } from "../api/shared"; import { WEBMCP_SHIM_VERSION } from "./vendor/checks"; import type { PublicWebmcpAudit } from "./vendor/projection"; @@ -40,15 +41,21 @@ export class WebmcpIngestError extends Error { readonly code: string | null; /** Milliseconds to wait, from a 429. */ readonly retryAfterMs: number | null; + /** + * The error body or `error` frame exactly as ora served it, when it sent + * JSON (what `--json` prints); null or undefined otherwise. + */ + readonly payload: unknown; constructor( message: string, - options: { code?: string | null; retryAfterMs?: number | null } = {}, + options: { code?: string | null; retryAfterMs?: number | null; payload?: unknown } = {}, ) { super(message); this.name = "WebmcpIngestError"; this.code = options.code ?? null; this.retryAfterMs = options.retryAfterMs ?? null; + this.payload = options.payload; } } @@ -176,6 +183,7 @@ export async function ingestCapture( options: IngestOptions = {}, ): Promise { const base = (options.baseUrl ?? process.env.ORA_API_URL ?? PUBLIC_BASE).replace(/\/+$/, ""); + if (options.apiKey) assertKeyDestination(base, "ORA_API_URL", PUBLIC_BASE); const idleMs = options.idleMs ?? STREAM_IDLE_MS; const dog = watchdog(idleMs); const timeoutError = () => @@ -236,27 +244,27 @@ async function httpError(res: Response): Promise { const wait = retryAfterMs ? ` — retry in ${Math.ceil(retryAfterMs / 1000)}s` : ""; return new WebmcpIngestError( `ora rate limit exceeded${wait} (10/min and 20 per 24h per IP). An ora partner or scan API key lifts the daily limit: set ORA_PARTNER_API_KEY or pass --api-key.`, - { code: code ?? "RATE_LIMITED", retryAfterMs }, + { code: code ?? "RATE_LIMITED", retryAfterMs, payload }, ); } if (code === "SHIM_VERSION_MISMATCH") { return new WebmcpIngestError( `This ax build captures with a shim ora no longer scores, so the audit was refused.\nUpgrade with \`npm i -g ax@latest\` (or \`npx ax@latest webmcp-audit …\`) and run it again.\n\nora said: ${payload?.error ?? "shim version mismatch"}`, - { code }, + { code, payload }, ); } if (res.status === 503) { return new WebmcpIngestError( `ora cannot score right now: ${sentence(payload?.error ?? "scoring is unavailable")} Nothing was spent against your rate limit; try again shortly.`, - { code: code ?? "SCORING_UNAVAILABLE" }, + { code: code ?? "SCORING_UNAVAILABLE", payload }, ); } return new WebmcpIngestError( `ora ingest failed: ${payload?.error ?? (await errorBodyText(res))}`, - { code }, + { code, payload }, ); } @@ -297,6 +305,7 @@ async function consumeIngestStream( // the caller would only see "ended before completing". throw new WebmcpIngestError(`ora audit failed: ${event.message}`, { code: event.status, + payload: event, }); } if (event.type === "done") {