From 9719aa3669292b8dd79a6db3e0bf911dbea6d04a Mon Sep 17 00:00:00 2001 From: bunizao <102936102+bunizao@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:29:01 +1100 Subject: [PATCH 1/6] feat(nouns)!: share the courses and grades nouns across CLIs Add SHARED_NOUNS so every CLI names the cross-tool nouns the same way: courses (aliases units, projects) and grades. commandsJson now rejects a top-level command named after a shared alias, and a verb the shared noun does not offer. GRADE_STATUSES fixes the status vocabulary for grades rows. BREAKING CHANGE: a CLI whose top-level command is named "units" must rename it to "courses" and keep "units" as an alias. --- README.md | 44 ++++++++++++++++++++++++++++--------- dist/nouns.d.ts | 26 ++++++++++++++++++++++ dist/nouns.js | 19 ++++++++++++++++ dist/program.js | 10 +++++++++ src/nouns.ts | 22 +++++++++++++++++++ src/program.ts | 10 +++++++++ test/contract.test.js | 5 +++++ test/nouns.test.js | 20 ++++++++--------- test/program.test.js | 50 +++++++++++++++++++++++++++++++------------ 9 files changed, 172 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 85f83c9..849e5df 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ import { createProgram, insertDefaultVerb, mutating, + SHARED_NOUNS, type NounSpec, } from "@bunizao/cli-kit"; @@ -35,17 +36,19 @@ const program = createProgram({ description: "Example CLI", }); -const units = program.command("units").aliases(["courses", "projects"]); -units.command("list").action(listUnits); -units.command("show ").action(showUnit); -mutating(units.command("set ")).action(setUnitState); +const courses = program.command("courses").aliases(["units", "projects"]); +courses.command("list").action(listCourses); +courses.command("show ").action(showCourse); +const tasks = program.command("tasks"); +tasks.command("list ").action(listTasks); +mutating(tasks.command("set ")).action(setTaskState); const nouns: readonly NounSpec[] = [ + SHARED_NOUNS.courses, { - name: "units", - aliases: ["courses", "projects"], - verbs: ["list", "show", "set"], - defaultByArity: { 0: "list", 1: "show" }, + name: "tasks", + verbs: ["list", "set"], + defaultByArity: { 1: "list" }, valueFlags: [], }, ]; @@ -163,8 +166,29 @@ aliases, positional arguments, options, enum values, nested commands, and `mutat commands at noun depth must use a verb exported in `VERBS`; unsupported verbs throw. `auth` and `skills` are action groups rather than domain nouns and are not assigned a `verb` field. -The exit-code table, error vocabulary, and verb set are public versioned API. Changing one requires -a major package release. +A top-level command named after a shared noun's alias throws, and so does a verb that shared noun +does not offer. + +The exit-code table, error vocabulary, verb set, and shared nouns are public versioned API. Changing +one requires a major package release. + +## Shared nouns + +`SHARED_NOUNS` holds the nouns every CLI names the same way. A CLI that offers one uses its spec +as is, or with fewer verbs and its own `valueFlags`; domain nouns such as `tasks` or `lessons` +stay with each CLI. + +| Noun | Aliases | Verbs | Default by arity | +|---|---|---|---| +| `courses` | `units`, `projects` | list, show, get | 0 list, 1 show | +| `grades` | | list | 0 list, 1 list | + +`courses` is canonical because Ed and Moodle both call them courses; `units` is an Australian usage. + +Each `grades` row carries `grade` and `max` as numbers or `null`, and a `status` from +`GRADE_STATUSES`: `graded`, `ungraded`, `unreleased` (marked but not released to the student), +`not_submitted`, or `unavailable` (the platform refused or failed for that row). Display strings +such as a letter grade or a range are optional extra fields. The conformance suite is isolated under `conformance/`. Until all three CLIs have published their normalized releases, missing binaries are reported as skipped tests. Its CI workflow installs the diff --git a/dist/nouns.d.ts b/dist/nouns.d.ts index e0da090..cf7fb97 100644 --- a/dist/nouns.d.ts +++ b/dist/nouns.d.ts @@ -6,4 +6,30 @@ export interface NounSpec { /** Option flags whose following token is a value, not a positional argument. */ readonly valueFlags?: readonly string[]; } +/** + * Nouns every CLI names the same way. A CLI that offers one uses its spec as is, or with fewer + * verbs and its own valueFlags, so `moodle grades` and `edstem grades` read alike. + */ +export declare const SHARED_NOUNS: { + readonly courses: { + readonly name: "courses"; + readonly aliases: readonly ["units", "projects"]; + readonly verbs: readonly ["list", "show", "get"]; + readonly defaultByArity: { + readonly 0: "list"; + readonly 1: "show"; + }; + }; + readonly grades: { + readonly name: "grades"; + readonly verbs: readonly ["list"]; + readonly defaultByArity: { + readonly 0: "list"; + readonly 1: "list"; + }; + }; +}; +/** The `status` of one row in `grades` output. */ +export declare const GRADE_STATUSES: readonly ["graded", "ungraded", "unreleased", "not_submitted", "unavailable"]; +export type GradeStatus = (typeof GRADE_STATUSES)[number]; export declare function insertDefaultVerb(argv: readonly string[], nouns: readonly NounSpec[]): string[]; diff --git a/dist/nouns.js b/dist/nouns.js index 373f82d..48965f0 100644 --- a/dist/nouns.js +++ b/dist/nouns.js @@ -1,3 +1,22 @@ +/** + * Nouns every CLI names the same way. A CLI that offers one uses its spec as is, or with fewer + * verbs and its own valueFlags, so `moodle grades` and `edstem grades` read alike. + */ +export const SHARED_NOUNS = { + courses: { + name: "courses", + aliases: ["units", "projects"], + verbs: ["list", "show", "get"], + defaultByArity: { 0: "list", 1: "show" }, + }, + grades: { + name: "grades", + verbs: ["list"], + defaultByArity: { 0: "list", 1: "list" }, + }, +}; +/** The `status` of one row in `grades` output. */ +export const GRADE_STATUSES = ["graded", "ungraded", "unreleased", "not_submitted", "unavailable"]; const GLOBAL_VALUE_FLAGS = new Set(["--fields", "-o", "--output"]); export function insertDefaultVerb(argv, nouns) { const result = [...argv]; diff --git a/dist/program.js b/dist/program.js index 90eb805..4a0f08a 100644 --- a/dist/program.js +++ b/dist/program.js @@ -1,8 +1,10 @@ import { Command } from "commander"; import { colorEnabled } from "./color.js"; import { styledHelp } from "./help.js"; +import { SHARED_NOUNS } from "./nouns.js"; export const VERBS = ["list", "show", "read", "get", "search", "send", "submit", "set", "mark-read"]; const VERB_SET = new Set(VERBS); +const SHARED = Object.values(SHARED_NOUNS); const mutations = new WeakSet(); const ACTION_GROUPS = new Set(["auth", "skills"]); const ACTIONS = { @@ -53,9 +55,17 @@ function describe(command, noun) { if (noun && ACTION_GROUPS.has(noun) && !ACTIONS[noun]?.has(command.name())) { throw new Error(`Unsupported action "${command.name()}" on group "${noun}".`); } + const canonical = noun ? undefined : SHARED.find((spec) => spec.aliases?.includes(currentNoun)); + if (canonical) { + throw new Error(`Noun "${currentNoun}" is an alias; name the command "${canonical.name}".`); + } if (verb && !VERB_SET.has(verb)) { throw new Error(`Unsupported verb "${verb}" on noun "${noun}".`); } + const shared = SHARED.find((spec) => spec.name === noun); + if (verb && shared && !shared.verbs.includes(verb)) { + throw new Error(`Unsupported verb "${verb}" on shared noun "${noun}".`); + } return { name: command.name(), noun: currentNoun, diff --git a/src/nouns.ts b/src/nouns.ts index 445acd9..46e70d0 100644 --- a/src/nouns.ts +++ b/src/nouns.ts @@ -7,6 +7,28 @@ export interface NounSpec { readonly valueFlags?: readonly string[]; } +/** + * Nouns every CLI names the same way. A CLI that offers one uses its spec as is, or with fewer + * verbs and its own valueFlags, so `moodle grades` and `edstem grades` read alike. + */ +export const SHARED_NOUNS = { + courses: { + name: "courses", + aliases: ["units", "projects"], + verbs: ["list", "show", "get"], + defaultByArity: { 0: "list", 1: "show" }, + }, + grades: { + name: "grades", + verbs: ["list"], + defaultByArity: { 0: "list", 1: "list" }, + }, +} as const satisfies Readonly>; + +/** The `status` of one row in `grades` output. */ +export const GRADE_STATUSES = ["graded", "ungraded", "unreleased", "not_submitted", "unavailable"] as const; +export type GradeStatus = (typeof GRADE_STATUSES)[number]; + const GLOBAL_VALUE_FLAGS = new Set(["--fields", "-o", "--output"]); export function insertDefaultVerb(argv: readonly string[], nouns: readonly NounSpec[]): string[] { diff --git a/src/program.ts b/src/program.ts index 8c30188..2b35e35 100644 --- a/src/program.ts +++ b/src/program.ts @@ -2,10 +2,12 @@ import { Command, type Argument, type Option } from "commander"; import { colorEnabled } from "./color.js"; import { styledHelp } from "./help.js"; +import { SHARED_NOUNS, type NounSpec } from "./nouns.js"; export const VERBS = ["list", "show", "read", "get", "search", "send", "submit", "set", "mark-read"] as const; const VERB_SET = new Set(VERBS); +const SHARED: readonly NounSpec[] = Object.values(SHARED_NOUNS); const mutations = new WeakSet(); const ACTION_GROUPS = new Set(["auth", "skills"]); const ACTIONS: Readonly>> = { @@ -95,9 +97,17 @@ function describe(command: Command, noun?: string): CommandDescription { if (noun && ACTION_GROUPS.has(noun) && !ACTIONS[noun]?.has(command.name())) { throw new Error(`Unsupported action "${command.name()}" on group "${noun}".`); } + const canonical = noun ? undefined : SHARED.find((spec) => spec.aliases?.includes(currentNoun)); + if (canonical) { + throw new Error(`Noun "${currentNoun}" is an alias; name the command "${canonical.name}".`); + } if (verb && !VERB_SET.has(verb)) { throw new Error(`Unsupported verb "${verb}" on noun "${noun}".`); } + const shared = SHARED.find((spec) => spec.name === noun); + if (verb && shared && !shared.verbs.includes(verb)) { + throw new Error(`Unsupported verb "${verb}" on shared noun "${noun}".`); + } return { name: command.name(), diff --git a/test/contract.test.js b/test/contract.test.js index 0f8e3e2..609dcd5 100644 --- a/test/contract.test.js +++ b/test/contract.test.js @@ -5,6 +5,8 @@ import { CliError, ERROR_CODES, EXIT_CODES, + GRADE_STATUSES, + SHARED_NOUNS, VERBS, createTheme, exitCodeFor, @@ -20,6 +22,9 @@ test("exports the versioned CLI contract", () => { assert.deepEqual(ERROR_CODES, ["usage", "auth", "config", "not_found", "upstream", "network", "cancelled", "unexpected"]); assert.deepEqual(EXIT_CODES, { usage: 2, auth: 3, config: 1, not_found: 4, upstream: 5, network: 1, cancelled: 130, unexpected: 1 }); assert.equal(exitCodeFor("auth"), 3); + assert.deepEqual(Object.keys(SHARED_NOUNS), ["courses", "grades"]); + assert.deepEqual(SHARED_NOUNS.courses.aliases, ["units", "projects"]); + assert.deepEqual(GRADE_STATUSES, ["graded", "ungraded", "unreleased", "not_submitted", "unavailable"]); }); test("resolveFormat applies explicit flags and TTY defaults", () => { diff --git a/test/nouns.test.js b/test/nouns.test.js index f3fedc3..a6a6041 100644 --- a/test/nouns.test.js +++ b/test/nouns.test.js @@ -5,8 +5,8 @@ import { insertDefaultVerb } from "../dist/index.js"; const nouns = [ { - name: "units", - aliases: ["courses", "projects"], + name: "courses", + aliases: ["units", "projects"], verbs: ["list", "show", "get"], defaultByArity: { 0: "list", 1: "show" }, }, @@ -20,17 +20,17 @@ const nouns = [ const cases = [ ["zero arguments", [], []], - ["zero-arity noun", ["units"], ["units", "list"]], - ["alias resolution", ["courses", "FIT1045"], ["units", "show", "FIT1045"]], - ["explicit verb", ["projects", "get", "FIT1045"], ["units", "get", "FIT1045"]], - ["long help", ["courses", "--help"], ["units", "--help"]], - ["short help", ["projects", "-h"], ["units", "-h"]], + ["zero-arity noun", ["courses"], ["courses", "list"]], + ["alias resolution", ["units", "FIT1045"], ["courses", "show", "FIT1045"]], + ["explicit verb", ["projects", "get", "FIT1045"], ["courses", "get", "FIT1045"]], + ["long help", ["units", "--help"], ["courses", "--help"]], + ["short help", ["projects", "-h"], ["courses", "-h"]], ["interleaved flags", ["tasks", "FIT1045", "--json"], ["tasks", "list", "FIT1045", "--json"]], ["command option value", ["tasks", "FIT1045", "--status", "rediscuss"], ["tasks", "list", "FIT1045", "--status", "rediscuss"]], - ["unknown arity", ["units", "FIT1045", "extra"], ["units", "FIT1045", "extra"]], + ["unknown arity", ["courses", "FIT1045", "extra"], ["courses", "FIT1045", "extra"]], ["terminator", ["tasks", "FIT1045", "--", "-1"], ["tasks", "show", "FIT1045", "--", "-1"]], - ["global flag before noun", ["--json", "units"], ["--json", "units", "list"]], - ["global option before noun", ["--fields", "id,name", "units"], ["--fields", "id,name", "units", "list"]], + ["global flag before noun", ["--json", "courses"], ["--json", "courses", "list"]], + ["global option before noun", ["--fields", "id,name", "courses"], ["--fields", "id,name", "courses", "list"]], ["unknown noun", ["widgets"], ["widgets"]], ]; diff --git a/test/program.test.js b/test/program.test.js index 1d0d004..44da466 100644 --- a/test/program.test.js +++ b/test/program.test.js @@ -5,20 +5,20 @@ import { commandsJson, createProgram, mutating } from "../dist/index.js"; test("commandsJson describes commands and marks mutations", () => { const program = createProgram({ name: "demo", version: "1.2.3", description: "Demo CLI" }); - const units = program.command("units").alias("courses"); - units.command("list").option("--state ", "Filter state").addOption( - units.createOption("--kind ", "Filter kind").choices(["core", "elective"]), + const tasks = program.command("tasks").alias("assignments"); + tasks.command("list").option("--state ", "Filter state").addOption( + tasks.createOption("--kind ", "Filter kind").choices(["core", "elective"]), ); - mutating(units.command("set ").description("Change state")); + mutating(tasks.command("set ").description("Change state")); const value = commandsJson(program); assert.equal(value.name, "demo"); assert.equal(value.version, "1.2.3"); assert.deepEqual(value.commands[0], { - name: "units", - noun: "units", - aliases: ["courses"], + name: "tasks", + noun: "tasks", + aliases: ["assignments"], description: "", positionals: [], options: [], @@ -26,7 +26,7 @@ test("commandsJson describes commands and marks mutations", () => { commands: [ { name: "list", - noun: "units", + noun: "tasks", verb: "list", aliases: [], description: "", @@ -46,12 +46,12 @@ test("commandsJson describes commands and marks mutations", () => { }, { name: "set", - noun: "units", + noun: "tasks", verb: "set", aliases: [], description: "Change state", positionals: [ - { name: "unit", description: "", required: true, variadic: false }, + { name: "task", description: "", required: true, variadic: false }, { name: "state", description: "", required: true, variadic: false }, ], options: [], @@ -64,21 +64,43 @@ test("commandsJson describes commands and marks mutations", () => { test("commandsJson leaves hidden options out", () => { const program = createProgram({ name: "demo", version: "1.2.3", description: "Demo CLI" }); - const list = program.command("units").command("list").option("--limit ", "Maximum rows"); + const list = program.command("tasks").command("list").option("--limit ", "Maximum rows"); list.addOption(list.createOption("--max ").hideHelp()); - const [units] = commandsJson(program).commands; + const [tasks] = commandsJson(program).commands; - assert.deepEqual(units.commands[0].options.map((option) => option.flags), ["--limit "]); + assert.deepEqual(tasks.commands[0].options.map((option) => option.flags), ["--limit "]); }); test("commandsJson rejects a verb outside the contract", () => { const program = createProgram({ name: "demo", version: "1.0.0", description: "Demo CLI" }); - program.command("units").command("destroy "); + program.command("tasks").command("destroy "); assert.throws(() => commandsJson(program), /Unsupported verb "destroy"/); }); +test("commandsJson accepts shared nouns under their canonical names", () => { + const program = createProgram({ name: "demo", version: "1.0.0", description: "Demo CLI" }); + program.command("courses").aliases(["units", "projects"]).command("list"); + program.command("grades").command("list [course]"); + + assert.deepEqual(commandsJson(program).commands.map((command) => command.name), ["courses", "grades"]); +}); + +test("commandsJson rejects a shared alias used as the command name", () => { + const program = createProgram({ name: "demo", version: "1.0.0", description: "Demo CLI" }); + program.command("units").alias("courses").command("list"); + + assert.throws(() => commandsJson(program), /Noun "units" is an alias; name the command "courses"/); +}); + +test("commandsJson rejects a contract verb a shared noun does not offer", () => { + const program = createProgram({ name: "demo", version: "1.0.0", description: "Demo CLI" }); + program.command("grades").command("set "); + + assert.throws(() => commandsJson(program), /Unsupported verb "set" on shared noun "grades"/); +}); + test("commandsJson rejects an unknown action in an exceptional group", () => { const program = createProgram({ name: "demo", version: "1.0.0", description: "Demo CLI" }); program.command("auth").command("bananas"); From 12906e1acb0585bc90c2e13d54a1d3aa084e6b11 Mon Sep 17 00:00:00 2001 From: bunizao <102936102+bunizao@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:43:51 +1100 Subject: [PATCH 2/6] feat(skills): render skills, MCP descriptions and plugins from one intent list Each CLI kept its own intent table, MCP description catalog and SKILL.md generator. An Intent now maps what a person says to the command and the MCP tool that answer it, and the kit renders everything from that list: - toolDescription(intent) for MCP tool descriptions - renderSkill for SKILL.md: intents, MCP tools, shared and CLI rules, setup, the command tree and exit codes - agentFiles for SKILL.md, a Claude Code plugin with its own marketplace, and Codex agents/openai.yaml; writeAgentFiles writes them or, with check, reports stale paths for CI - checkIntentCommands fails when an intent's command no longer parses against the program --- README.md | 49 ++++++++++- dist/index.d.ts | 2 + dist/index.js | 2 + dist/intents.d.ts | 31 +++++++ dist/intents.js | 70 ++++++++++++++++ dist/skills.d.ts | 63 +++++++++++++++ dist/skills.js | 134 ++++++++++++++++++++++++++++++ src/index.ts | 2 + src/intents.ts | 103 ++++++++++++++++++++++++ src/skills.ts | 188 +++++++++++++++++++++++++++++++++++++++++++ test/intents.test.js | 49 +++++++++++ test/skills.test.js | 86 ++++++++++++++++++++ 12 files changed, 776 insertions(+), 3 deletions(-) create mode 100644 dist/intents.d.ts create mode 100644 dist/intents.js create mode 100644 dist/skills.d.ts create mode 100644 dist/skills.js create mode 100644 src/intents.ts create mode 100644 src/skills.ts create mode 100644 test/intents.test.js create mode 100644 test/skills.test.js diff --git a/README.md b/README.md index 849e5df..d639117 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,10 @@ does not offer. The exit-code table, error vocabulary, verb set, and shared nouns are public versioned API. Changing one requires a major package release. +The conformance suite is isolated under `conformance/`. Until all three CLIs have published their +normalized releases, missing binaries are reported as skipped tests. Its CI workflow installs the +published packages before running the suite. + ## Shared nouns `SHARED_NOUNS` holds the nouns every CLI names the same way. A CLI that offers one uses its spec @@ -190,9 +194,48 @@ Each `grades` row carries `grade` and `max` as numbers or `null`, and a `status` `not_submitted`, or `unavailable` (the platform refused or failed for that row). Display strings such as a letter grade or a range are optional extra fields. -The conformance suite is isolated under `conformance/`. Until all three CLIs have published their -normalized releases, missing binaries are reported as skipped tests. Its CI workflow installs the -published packages before running the suite. +## Intents, skills and plugins + +An `Intent` maps what a person says to the command and the MCP tool that answer it. One list of +intents is the single source for the skill, the MCP tool descriptions and the agent packaging: + +```ts +const INTENTS: readonly Intent[] = [ + { + when: "my grades / what did I get", + what: "Graded items per course with grade, max and status.", + command: "example grades [COURSE]", + tool: "list_grades", + refs: "course code, name or id", + then: "show the item for feedback", + cost: "small", + }, +]; + +server.registerTool("list_grades", { description: toolDescription(INTENTS[0]) /* , schema */ }, handler); + +const files = agentFiles({ + skill: { name: "example-cli", description, title: "Example CLI", intents: INTENTS, program: commandsJson(program) }, + version, + repository: "https://github.com/example/example-cli", + author: { name: "example" }, + mcpServers: { example: { command: "npx", args: ["-y", "example-cli", "mcp"] } }, +}); +writeAgentFiles(process.cwd(), files); // { check: true } returns stale paths for CI +``` + +`agentFiles` returns, by path from the repository root: + +- `SKILL.md`, for `npx skills add owner/repo`. It holds the intent table, the MCP tools, the + shared rules plus the CLI's own, the setup, the full command tree and the exit codes. +- `.claude-plugin/plugin.json` and `.claude-plugin/marketplace.json`: the repository is a Claude + Code plugin whose one skill is `SKILL.md`, and its own marketplace. Install with + `claude plugin marketplace add owner/repo`, then `claude plugin install example-cli@example-cli`. +- `agents/openai.yaml`, when `openai` metadata is given. + +`checkIntentCommands(program, nouns, intents)` throws when an intent's command no longer parses: +a renamed noun or verb, a removed option, or the wrong binary name. Call it from the CLI's tests +so the skill and the MCP descriptions cannot drift from the command tree. ## Prompts and progress diff --git a/dist/index.d.ts b/dist/index.d.ts index c528a13..652130a 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -4,9 +4,11 @@ export * from "./color.js"; export * from "./confirm.js"; export * from "./errors.js"; export * from "./help.js"; +export * from "./intents.js"; export * from "./nouns.js"; export * from "./output.js"; export * from "./program.js"; export * from "./report.js"; +export * from "./skills.js"; export * from "./ui.js"; export * from "./theme.js"; diff --git a/dist/index.js b/dist/index.js index c528a13..652130a 100644 --- a/dist/index.js +++ b/dist/index.js @@ -4,9 +4,11 @@ export * from "./color.js"; export * from "./confirm.js"; export * from "./errors.js"; export * from "./help.js"; +export * from "./intents.js"; export * from "./nouns.js"; export * from "./output.js"; export * from "./program.js"; export * from "./report.js"; +export * from "./skills.js"; export * from "./ui.js"; export * from "./theme.js"; diff --git a/dist/intents.d.ts b/dist/intents.d.ts new file mode 100644 index 0000000..77d42c0 --- /dev/null +++ b/dist/intents.d.ts @@ -0,0 +1,31 @@ +import type { Command } from "commander"; +import { type NounSpec } from "./nouns.js"; +/** + * One thing a person asks for, mapped to the CLI command and the MCP tool that answer it. + * The skill, the MCP tool descriptions and the help text all render from these fields. + */ +export interface Intent { + /** What the person says, e.g. "my grades / what did I get". */ + readonly when: string; + /** What comes back, as a sentence. */ + readonly what: string; + /** The command as typed, with placeholders such as UNIT; omitted for an MCP-only tool. */ + readonly command?: string; + /** The MCP tool for the same request; omitted for a CLI-only command. */ + readonly tool?: string; + /** Where nearby requests go instead, e.g. "get_thread for thread bodies". */ + readonly instead?: string; + /** The references and arguments it accepts. */ + readonly refs?: string; + /** The usual next step. */ + readonly then?: string; + /** Rough response size. */ + readonly cost?: string; +} +/** The MCP tool description for an intent: prose, because a model reads it. */ +export declare function toolDescription(intent: Intent): string; +/** + * Throw when an intent's command no longer parses against the program: a renamed noun or verb, + * a wrong binary name, or an option that was removed. Run it in the CLI's tests. + */ +export declare function checkIntentCommands(program: Command, nouns: readonly NounSpec[], intents: readonly Intent[]): void; diff --git a/dist/intents.js b/dist/intents.js new file mode 100644 index 0000000..cb0b66a --- /dev/null +++ b/dist/intents.js @@ -0,0 +1,70 @@ +import { insertDefaultVerb } from "./nouns.js"; +/** The MCP tool description for an intent: prose, because a model reads it. */ +export function toolDescription(intent) { + return [ + sentence(intent.what), + sentence(`Use when the user says "${intent.when}"`), + intent.instead && sentence(`Instead: ${intent.instead}`), + intent.refs && sentence(`Input: ${intent.refs}`), + intent.then && sentence(`Then: ${intent.then}`), + intent.cost && sentence(`Cost: ${intent.cost}`), + ].filter(Boolean).join(" "); +} +/** + * Throw when an intent's command no longer parses against the program: a renamed noun or verb, + * a wrong binary name, or an option that was removed. Run it in the CLI's tests. + */ +export function checkIntentCommands(program, nouns, intents) { + const problems = intents.flatMap((intent) => (intent.command ? commandProblems(program, nouns, intent.command) : [])); + if (problems.length) + throw new Error(`Intent commands out of date:\n${problems.join("\n")}`); +} +const ALWAYS_ACCEPTED = new Set(["-h", "--help", "-V", "--version"]); +function commandProblems(program, nouns, text) { + const [binary, ...rest] = words(text); + if (binary !== program.name()) + return [`${text}: starts with "${binary}", not "${program.name()}"`]; + const argv = insertDefaultVerb(rest.map(unwrapOptionalFlag), nouns); + const path = [program]; + let unmatched; + for (const token of argv) { + if (token.startsWith("-")) + continue; + const current = path.at(-1); + const child = current.commands.find((command) => command.name() === token || command.aliases().includes(token)); + if (!child) { + unmatched = token; + break; + } + path.push(child); + } + const problems = []; + const resolved = path.at(-1); + // A group that takes no arguments of its own must be followed by one of its commands. + if (unmatched !== undefined && resolved.commands.length && !resolved.registeredArguments.length) { + problems.push(`${text}: "${unmatched}" is not a command of "${path.map((command) => command.name()).join(" ")}"`); + } + const options = new Set(path.flatMap((command) => command.options.flatMap(optionNames))); + for (const token of argv) { + const flag = token.split("=")[0] ?? token; + if (flag.startsWith("-") && !ALWAYS_ACCEPTED.has(flag) && !options.has(flag)) { + problems.push(`${text}: unknown option "${flag}"`); + } + } + return problems; +} +// Quoted strings and bracketed placeholders such as stay one word. +function words(text) { + return text.replace(/^`|`$/g, "").match(/"[^"]*"|'[^']*'|<[^>]*>|\[[^\]]*\]|\S+/g) ?? []; +} +// `[--final]` documents an optional flag; check it like `--final`. +function unwrapOptionalFlag(token) { + const inner = /^\[(-[^\s\]]+)/.exec(token); + return inner?.[1] ?? token; +} +function optionNames(option) { + return [option.long, option.short].filter((name) => Boolean(name)); +} +function sentence(text) { + return /[.!?]$/.test(text) ? text : `${text}.`; +} diff --git a/dist/skills.d.ts b/dist/skills.d.ts new file mode 100644 index 0000000..5abbff0 --- /dev/null +++ b/dist/skills.d.ts @@ -0,0 +1,63 @@ +import { type Intent } from "./intents.js"; +import type { ProgramDescription } from "./program.js"; +export interface SkillOptions { + /** The skill and plugin name, e.g. "edstem-cli". */ + readonly name: string; + /** When to use the skill; put the trigger first, hosts truncate long descriptions. */ + readonly description: string; + readonly title: string; + /** Markdown between the title and the intent table: the surfaces and the reference vocabulary. */ + readonly intro?: string; + readonly intents: readonly Intent[]; + /** Rules for this CLI only; the shared contract rules are always included. */ + readonly rules?: readonly string[]; + /** Markdown for the Setup section. */ + readonly setup?: string; + /** `commandsJson(program)` output. */ + readonly program: ProgramDescription; +} +/** A local stdio server, or a hosted Streamable HTTP server. */ +export type McpServer = { + readonly command: string; + readonly args?: readonly string[]; + readonly env?: Readonly>; +} | { + readonly type: "http"; + readonly url: string; + readonly headers?: Readonly>; +}; +export interface AgentFilesOptions { + readonly skill: SkillOptions; + readonly version: string; + /** The GitHub repository URL; it is also the plugin's homepage. */ + readonly repository: string; + readonly author: { + readonly name: string; + readonly url?: string; + }; + readonly license?: string; + readonly keywords?: readonly string[]; + /** MCP servers the Claude Code plugin starts, keyed by server name. */ + readonly mcpServers?: Readonly>; + /** Codex skill metadata, written to agents/openai.yaml. */ + readonly openai?: { + readonly displayName: string; + readonly shortDescription: string; + readonly defaultPrompt: string; + }; +} +/** SKILL.md for one CLI: what people say, what to run, the rules, and the full command tree. */ +export declare function renderSkill(options: SkillOptions): string; +/** + * Every agent packaging file for one CLI repository, keyed by path from the repository root: + * SKILL.md for `npx skills add`, the Claude Code plugin and its single-plugin marketplace, and + * agents/openai.yaml when Codex metadata is given. + */ +export declare function agentFiles(options: AgentFilesOptions): Record; +/** + * Write generated files under root. With `check`, write nothing and return the paths whose + * content differs, so CI can fail on a stale commit. + */ +export declare function writeAgentFiles(root: string, files: Readonly>, options?: { + check?: boolean; +}): string[]; diff --git a/dist/skills.js b/dist/skills.js new file mode 100644 index 0000000..cff9113 --- /dev/null +++ b/dist/skills.js @@ -0,0 +1,134 @@ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { stringify } from "yaml"; +import { EXIT_CODES } from "./errors.js"; +import { toolDescription } from "./intents.js"; +const SHARED_RULES = [ + "Run the narrowest command or tool that answers the request; one call per intent.", + "Piped output is JSON and a terminal gets a table. `--fields a,b` keeps only the listed top-level fields.", + "`read` prints Markdown and never changes anything upstream.", + "Run a mutating command only when the user asked for it. It needs a confirmation or `--yes`; `--dry-run` prints the plan first.", + "Never print, log or store tokens, cookies or session files.", +]; +/** SKILL.md for one CLI: what people say, what to run, the rules, and the full command tree. */ +export function renderSkill(options) { + const binary = options.program.name; + const commandIntents = options.intents.filter((intent) => intent.command); + const toolIntents = options.intents.filter((intent) => intent.tool); + const exitCodes = Object.entries(EXIT_CODES) + .map(([code, exit]) => `${exit} ${code}`) + .join(", "); + return [ + stringifyFrontmatter({ name: options.name, description: options.description }), + `# ${options.title}`, + options.intro, + "## What the user says, and what to run", + markdownTable(["The user says", "Run", "Then"], commandIntents.map((intent) => [intent.when, `\`${intent.command}\``, intent.then ?? ""])), + toolIntents.length + ? ["## MCP tools", markdownTable(["Tool", "Description"], toolIntents.map((intent) => [`\`${intent.tool}\``, toolDescription(intent)]))].join("\n\n") + : undefined, + "## Rules", + [...SHARED_RULES, ...(options.rules ?? []), `\`${binary} commands --json\` is the source of truth for the command tree.`] + .map((rule) => `- ${rule}`) + .join("\n"), + options.setup ? `## Setup\n\n${options.setup}` : undefined, + "## Commands", + markdownTable(["Command", "Description", "Arguments", "Options", "Mutating"], options.program.commands.flatMap((command) => commandRows(binary, command))), + "## Errors", + `Errors are printed once on stderr. Exit codes: 0 success, ${exitCodes}.`, + ].filter((section) => Boolean(section)).join("\n\n") + "\n"; +} +/** + * Every agent packaging file for one CLI repository, keyed by path from the repository root: + * SKILL.md for `npx skills add`, the Claude Code plugin and its single-plugin marketplace, and + * agents/openai.yaml when Codex metadata is given. + */ +export function agentFiles(options) { + const { skill } = options; + const files = { + "SKILL.md": renderSkill(skill), + ".claude-plugin/plugin.json": json({ + name: skill.name, + version: options.version, + description: skill.description, + author: options.author, + homepage: options.repository, + repository: options.repository, + ...(options.license ? { license: options.license } : {}), + ...(options.keywords?.length ? { keywords: options.keywords } : {}), + // The repository root is the plugin root, and its SKILL.md is the plugin's one skill. + skills: "./", + ...(options.mcpServers ? { mcpServers: options.mcpServers } : {}), + }), + ".claude-plugin/marketplace.json": json({ + name: skill.name, + description: skill.description, + owner: options.author, + plugins: [{ name: skill.name, source: "./", description: skill.description }], + }), + }; + if (options.openai) { + files["agents/openai.yaml"] = stringify({ + interface: { + display_name: options.openai.displayName, + short_description: options.openai.shortDescription, + default_prompt: options.openai.defaultPrompt, + }, + }); + } + return files; +} +/** + * Write generated files under root. With `check`, write nothing and return the paths whose + * content differs, so CI can fail on a stale commit. + */ +export function writeAgentFiles(root, files, options = {}) { + const stale = Object.entries(files) + .filter(([path, content]) => readOrEmpty(join(root, path)) !== content) + .map(([path]) => path); + if (options.check) + return stale; + for (const path of stale) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), files[path], "utf8"); + } + return stale; +} +function commandRows(binary, command, parents = []) { + const path = [...parents, command.name]; + const row = [ + `\`${[binary, ...path].join(" ")}\``, + command.description, + command.positionals.map((argument) => { + const name = argument.variadic ? `${argument.name}...` : argument.name; + return argument.required ? `<${name}>` : `[${name}]`; + }).join(" "), + command.options.map((option) => option.flags).join("
"), + command.mutating ? "yes" : "no", + ]; + return [row, ...command.commands.flatMap((child) => commandRows(binary, child, path))]; +} +function stringifyFrontmatter(fields) { + return `---\n${stringify(fields, { lineWidth: 0 })}---`; +} +function markdownTable(headers, rows) { + return [ + `| ${headers.join(" | ")} |`, + `| ${headers.map(() => "---").join(" | ")} |`, + ...rows.map((row) => `| ${row.map(escapeCell).join(" | ")} |`), + ].join("\n"); +} +function escapeCell(value) { + return value.replace(/\|/g, "\\|").replace(/\n/g, "
"); +} +function json(value) { + return `${JSON.stringify(value, null, 2)}\n`; +} +function readOrEmpty(path) { + try { + return readFileSync(path, "utf8"); + } + catch { + return ""; + } +} diff --git a/src/index.ts b/src/index.ts index c528a13..652130a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,9 +4,11 @@ export * from "./color.js"; export * from "./confirm.js"; export * from "./errors.js"; export * from "./help.js"; +export * from "./intents.js"; export * from "./nouns.js"; export * from "./output.js"; export * from "./program.js"; export * from "./report.js"; +export * from "./skills.js"; export * from "./ui.js"; export * from "./theme.js"; diff --git a/src/intents.ts b/src/intents.ts new file mode 100644 index 0000000..d2e89af --- /dev/null +++ b/src/intents.ts @@ -0,0 +1,103 @@ +import type { Command, Option } from "commander"; + +import { insertDefaultVerb, type NounSpec } from "./nouns.js"; + +/** + * One thing a person asks for, mapped to the CLI command and the MCP tool that answer it. + * The skill, the MCP tool descriptions and the help text all render from these fields. + */ +export interface Intent { + /** What the person says, e.g. "my grades / what did I get". */ + readonly when: string; + /** What comes back, as a sentence. */ + readonly what: string; + /** The command as typed, with placeholders such as UNIT; omitted for an MCP-only tool. */ + readonly command?: string; + /** The MCP tool for the same request; omitted for a CLI-only command. */ + readonly tool?: string; + /** Where nearby requests go instead, e.g. "get_thread for thread bodies". */ + readonly instead?: string; + /** The references and arguments it accepts. */ + readonly refs?: string; + /** The usual next step. */ + readonly then?: string; + /** Rough response size. */ + readonly cost?: string; +} + +/** The MCP tool description for an intent: prose, because a model reads it. */ +export function toolDescription(intent: Intent): string { + return [ + sentence(intent.what), + sentence(`Use when the user says "${intent.when}"`), + intent.instead && sentence(`Instead: ${intent.instead}`), + intent.refs && sentence(`Input: ${intent.refs}`), + intent.then && sentence(`Then: ${intent.then}`), + intent.cost && sentence(`Cost: ${intent.cost}`), + ].filter(Boolean).join(" "); +} + +/** + * Throw when an intent's command no longer parses against the program: a renamed noun or verb, + * a wrong binary name, or an option that was removed. Run it in the CLI's tests. + */ +export function checkIntentCommands(program: Command, nouns: readonly NounSpec[], intents: readonly Intent[]): void { + const problems = intents.flatMap((intent) => (intent.command ? commandProblems(program, nouns, intent.command) : [])); + if (problems.length) throw new Error(`Intent commands out of date:\n${problems.join("\n")}`); +} + +const ALWAYS_ACCEPTED = new Set(["-h", "--help", "-V", "--version"]); + +function commandProblems(program: Command, nouns: readonly NounSpec[], text: string): string[] { + const [binary, ...rest] = words(text); + if (binary !== program.name()) return [`${text}: starts with "${binary}", not "${program.name()}"`]; + + const argv = insertDefaultVerb(rest.map(unwrapOptionalFlag), nouns); + const path = [program]; + let unmatched: string | undefined; + for (const token of argv) { + if (token.startsWith("-")) continue; + const current = path.at(-1) as Command; + const child = current.commands.find((command) => command.name() === token || command.aliases().includes(token)); + if (!child) { + unmatched = token; + break; + } + path.push(child); + } + + const problems: string[] = []; + const resolved = path.at(-1) as Command; + // A group that takes no arguments of its own must be followed by one of its commands. + if (unmatched !== undefined && resolved.commands.length && !resolved.registeredArguments.length) { + problems.push(`${text}: "${unmatched}" is not a command of "${path.map((command) => command.name()).join(" ")}"`); + } + + const options = new Set(path.flatMap((command) => command.options.flatMap(optionNames))); + for (const token of argv) { + const flag = token.split("=")[0] ?? token; + if (flag.startsWith("-") && !ALWAYS_ACCEPTED.has(flag) && !options.has(flag)) { + problems.push(`${text}: unknown option "${flag}"`); + } + } + return problems; +} + +// Quoted strings and bracketed placeholders such as stay one word. +function words(text: string): string[] { + return text.replace(/^`|`$/g, "").match(/"[^"]*"|'[^']*'|<[^>]*>|\[[^\]]*\]|\S+/g) ?? []; +} + +// `[--final]` documents an optional flag; check it like `--final`. +function unwrapOptionalFlag(token: string): string { + const inner = /^\[(-[^\s\]]+)/.exec(token); + return inner?.[1] ?? token; +} + +function optionNames(option: Option): string[] { + return [option.long, option.short].filter((name): name is string => Boolean(name)); +} + +function sentence(text: string): string { + return /[.!?]$/.test(text) ? text : `${text}.`; +} diff --git a/src/skills.ts b/src/skills.ts new file mode 100644 index 0000000..3d3c1b5 --- /dev/null +++ b/src/skills.ts @@ -0,0 +1,188 @@ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +import { stringify } from "yaml"; + +import { EXIT_CODES } from "./errors.js"; +import { toolDescription, type Intent } from "./intents.js"; +import type { CommandDescription, ProgramDescription } from "./program.js"; + +export interface SkillOptions { + /** The skill and plugin name, e.g. "edstem-cli". */ + readonly name: string; + /** When to use the skill; put the trigger first, hosts truncate long descriptions. */ + readonly description: string; + readonly title: string; + /** Markdown between the title and the intent table: the surfaces and the reference vocabulary. */ + readonly intro?: string; + readonly intents: readonly Intent[]; + /** Rules for this CLI only; the shared contract rules are always included. */ + readonly rules?: readonly string[]; + /** Markdown for the Setup section. */ + readonly setup?: string; + /** `commandsJson(program)` output. */ + readonly program: ProgramDescription; +} + +/** A local stdio server, or a hosted Streamable HTTP server. */ +export type McpServer = + | { readonly command: string; readonly args?: readonly string[]; readonly env?: Readonly> } + | { readonly type: "http"; readonly url: string; readonly headers?: Readonly> }; + +export interface AgentFilesOptions { + readonly skill: SkillOptions; + readonly version: string; + /** The GitHub repository URL; it is also the plugin's homepage. */ + readonly repository: string; + readonly author: { readonly name: string; readonly url?: string }; + readonly license?: string; + readonly keywords?: readonly string[]; + /** MCP servers the Claude Code plugin starts, keyed by server name. */ + readonly mcpServers?: Readonly>; + /** Codex skill metadata, written to agents/openai.yaml. */ + readonly openai?: { readonly displayName: string; readonly shortDescription: string; readonly defaultPrompt: string }; +} + +const SHARED_RULES = [ + "Run the narrowest command or tool that answers the request; one call per intent.", + "Piped output is JSON and a terminal gets a table. `--fields a,b` keeps only the listed top-level fields.", + "`read` prints Markdown and never changes anything upstream.", + "Run a mutating command only when the user asked for it. It needs a confirmation or `--yes`; `--dry-run` prints the plan first.", + "Never print, log or store tokens, cookies or session files.", +]; + +/** SKILL.md for one CLI: what people say, what to run, the rules, and the full command tree. */ +export function renderSkill(options: SkillOptions): string { + const binary = options.program.name; + const commandIntents = options.intents.filter((intent) => intent.command); + const toolIntents = options.intents.filter((intent) => intent.tool); + const exitCodes = Object.entries(EXIT_CODES) + .map(([code, exit]) => `${exit} ${code}`) + .join(", "); + + return [ + stringifyFrontmatter({ name: options.name, description: options.description }), + `# ${options.title}`, + options.intro, + "## What the user says, and what to run", + markdownTable( + ["The user says", "Run", "Then"], + commandIntents.map((intent) => [intent.when, `\`${intent.command}\``, intent.then ?? ""]), + ), + toolIntents.length + ? ["## MCP tools", markdownTable(["Tool", "Description"], toolIntents.map((intent) => [`\`${intent.tool}\``, toolDescription(intent)]))].join("\n\n") + : undefined, + "## Rules", + [...SHARED_RULES, ...(options.rules ?? []), `\`${binary} commands --json\` is the source of truth for the command tree.`] + .map((rule) => `- ${rule}`) + .join("\n"), + options.setup ? `## Setup\n\n${options.setup}` : undefined, + "## Commands", + markdownTable( + ["Command", "Description", "Arguments", "Options", "Mutating"], + options.program.commands.flatMap((command) => commandRows(binary, command)), + ), + "## Errors", + `Errors are printed once on stderr. Exit codes: 0 success, ${exitCodes}.`, + ].filter((section): section is string => Boolean(section)).join("\n\n") + "\n"; +} + +/** + * Every agent packaging file for one CLI repository, keyed by path from the repository root: + * SKILL.md for `npx skills add`, the Claude Code plugin and its single-plugin marketplace, and + * agents/openai.yaml when Codex metadata is given. + */ +export function agentFiles(options: AgentFilesOptions): Record { + const { skill } = options; + const files: Record = { + "SKILL.md": renderSkill(skill), + ".claude-plugin/plugin.json": json({ + name: skill.name, + version: options.version, + description: skill.description, + author: options.author, + homepage: options.repository, + repository: options.repository, + ...(options.license ? { license: options.license } : {}), + ...(options.keywords?.length ? { keywords: options.keywords } : {}), + // The repository root is the plugin root, and its SKILL.md is the plugin's one skill. + skills: "./", + ...(options.mcpServers ? { mcpServers: options.mcpServers } : {}), + }), + ".claude-plugin/marketplace.json": json({ + name: skill.name, + description: skill.description, + owner: options.author, + plugins: [{ name: skill.name, source: "./", description: skill.description }], + }), + }; + if (options.openai) { + files["agents/openai.yaml"] = stringify({ + interface: { + display_name: options.openai.displayName, + short_description: options.openai.shortDescription, + default_prompt: options.openai.defaultPrompt, + }, + }); + } + return files; +} + +/** + * Write generated files under root. With `check`, write nothing and return the paths whose + * content differs, so CI can fail on a stale commit. + */ +export function writeAgentFiles(root: string, files: Readonly>, options: { check?: boolean } = {}): string[] { + const stale = Object.entries(files) + .filter(([path, content]) => readOrEmpty(join(root, path)) !== content) + .map(([path]) => path); + if (options.check) return stale; + for (const path of stale) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), files[path] as string, "utf8"); + } + return stale; +} + +function commandRows(binary: string, command: CommandDescription, parents: readonly string[] = []): string[][] { + const path = [...parents, command.name]; + const row = [ + `\`${[binary, ...path].join(" ")}\``, + command.description, + command.positionals.map((argument) => { + const name = argument.variadic ? `${argument.name}...` : argument.name; + return argument.required ? `<${name}>` : `[${name}]`; + }).join(" "), + command.options.map((option) => option.flags).join("
"), + command.mutating ? "yes" : "no", + ]; + return [row, ...command.commands.flatMap((child) => commandRows(binary, child, path))]; +} + +function stringifyFrontmatter(fields: Readonly>): string { + return `---\n${stringify(fields, { lineWidth: 0 })}---`; +} + +function markdownTable(headers: readonly string[], rows: readonly (readonly string[])[]): string { + return [ + `| ${headers.join(" | ")} |`, + `| ${headers.map(() => "---").join(" | ")} |`, + ...rows.map((row) => `| ${row.map(escapeCell).join(" | ")} |`), + ].join("\n"); +} + +function escapeCell(value: string): string { + return value.replace(/\|/g, "\\|").replace(/\n/g, "
"); +} + +function json(value: unknown): string { + return `${JSON.stringify(value, null, 2)}\n`; +} + +function readOrEmpty(path: string): string { + try { + return readFileSync(path, "utf8"); + } catch { + return ""; + } +} diff --git a/test/intents.test.js b/test/intents.test.js new file mode 100644 index 0000000..22fdb79 --- /dev/null +++ b/test/intents.test.js @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { SHARED_NOUNS, checkIntentCommands, createProgram, toolDescription } from "../dist/index.js"; + +function demo() { + const program = createProgram({ name: "demo", version: "1.0.0", description: "Demo CLI" }); + const courses = program.command("courses").aliases(["units", "projects"]); + courses.command("list").option("--archived", "Include archived courses"); + courses.command("show "); + program.command("grades").command("list [course]").option("--mode ", "Which rows"); + program.command("download ").alias("dl").option("--to ", "Destination"); + return program; +} + +const nouns = [SHARED_NOUNS.courses, { ...SHARED_NOUNS.grades, valueFlags: ["--mode"] }]; + +test("toolDescription reads as prose and skips missing fields", () => { + assert.equal( + toolDescription({ when: "my grades", what: "Graded items per course", refs: "course code or id", cost: "small" }), + 'Graded items per course. Use when the user says "my grades". Input: course code or id. Cost: small.', + ); +}); + +test("checkIntentCommands accepts commands that parse, with defaults, aliases and placeholders", () => { + assert.doesNotThrow(() => checkIntentCommands(demo(), nouns, [ + { when: "my courses", what: "Courses.", command: "demo courses --archived" }, + { when: "one course", what: "A course.", command: "`demo units UNIT`" }, + { when: "grades", what: "Grades.", command: "demo grades [UNIT] --mode all --json" }, + { when: "a file", what: "A file.", command: 'demo dl "UNIT TASK" --to DIR [--dry-run]' }, + { when: "MCP only", what: "A widget.", tool: "show_widget" }, + ])); +}); + +test("checkIntentCommands reports renamed commands, removed options and the wrong binary", () => { + assert.throws( + () => checkIntentCommands(demo(), nouns, [ + { when: "a", what: "A.", command: "demo lessons UNIT" }, + { when: "b", what: "B.", command: "demo courses list --state active" }, + { when: "c", what: "C.", command: "other courses" }, + ]), + (error) => { + assert.match(error.message, /demo lessons UNIT: "lessons" is not a command of "demo"/); + assert.match(error.message, /demo courses list --state active: unknown option "--state"/); + assert.match(error.message, /other courses: starts with "other", not "demo"/); + return true; + }, + ); +}); diff --git a/test/skills.test.js b/test/skills.test.js new file mode 100644 index 0000000..eafd572 --- /dev/null +++ b/test/skills.test.js @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { agentFiles, commandsJson, createProgram, mutating, renderSkill, writeAgentFiles } from "../dist/index.js"; + +function skillOptions() { + const program = createProgram({ name: "demo", version: "1.2.3", description: "Demo CLI" }); + program.command("courses").alias("units").command("list").description("List courses."); + mutating(program.command("tasks").command("submit ").description("Submit a task.")); + return { + name: "demo-cli", + description: "Read demo courses: grades, tasks and files.", + title: "Demo CLI", + intro: "COURSE is a code or id.", + intents: [ + { when: "my courses", what: "Courses.", command: "demo courses", tool: "list_courses", then: "pass a code as COURSE" }, + { when: "a widget", what: "A widget.", tool: "show_widget" }, + ], + rules: ["Ask before submitting."], + program: commandsJson(program), + }; +} + +test("renderSkill writes frontmatter, intents, tools, rules, commands and exit codes", () => { + const skill = renderSkill(skillOptions()); + + assert.match(skill, /^---\nname: demo-cli\ndescription: "Read demo courses: grades, tasks and files."\n---\n\n# Demo CLI\n\nCOURSE is a code or id.\n/); + assert.match(skill, /\| my courses \| `demo courses` \| pass a code as COURSE \|/); + assert.doesNotMatch(skill, /\| a widget \|/); + assert.match(skill, /\| `show_widget` \| A widget\. Use when the user says "a widget"\. \|/); + assert.match(skill, /- Ask before submitting\.\n- `demo commands --json` is the source of truth/); + assert.match(skill, /\| `demo tasks submit` \| Submit a task\. \| \| {2}\| yes \|/); + assert.match(skill, /Exit codes: 0 success, 2 usage, 3 auth/); + assert.ok(skill.endsWith("\n")); +}); + +test("agentFiles packages the skill as a Claude Code plugin, a marketplace and Codex metadata", () => { + const files = agentFiles({ + skill: skillOptions(), + version: "1.2.3", + repository: "https://github.com/example/demo-cli", + author: { name: "example" }, + license: "MIT", + mcpServers: { demo: { command: "npx", args: ["-y", "demo-cli", "mcp"] } }, + openai: { displayName: "Demo CLI", shortDescription: "Read demo courses", defaultPrompt: "Use $demo-cli." }, + }); + + assert.deepEqual(Object.keys(files), ["SKILL.md", ".claude-plugin/plugin.json", ".claude-plugin/marketplace.json", "agents/openai.yaml"]); + assert.deepEqual(JSON.parse(files[".claude-plugin/plugin.json"]), { + name: "demo-cli", + version: "1.2.3", + description: "Read demo courses: grades, tasks and files.", + author: { name: "example" }, + homepage: "https://github.com/example/demo-cli", + repository: "https://github.com/example/demo-cli", + license: "MIT", + skills: "./", + mcpServers: { demo: { command: "npx", args: ["-y", "demo-cli", "mcp"] } }, + }); + assert.deepEqual(JSON.parse(files[".claude-plugin/marketplace.json"]), { + name: "demo-cli", + description: "Read demo courses: grades, tasks and files.", + owner: { name: "example" }, + plugins: [{ name: "demo-cli", source: "./", description: "Read demo courses: grades, tasks and files." }], + }); + assert.equal( + files["agents/openai.yaml"], + "interface:\n display_name: Demo CLI\n short_description: Read demo courses\n default_prompt: Use $demo-cli.\n", + ); +}); + +test("writeAgentFiles writes stale files and reports them without writing under check", () => { + const root = mkdtempSync(join(tmpdir(), "cli-kit-skills-")); + try { + const files = { "SKILL.md": "one\n", ".claude-plugin/plugin.json": "{}\n" }; + assert.deepEqual(writeAgentFiles(root, files, { check: true }), ["SKILL.md", ".claude-plugin/plugin.json"]); + assert.deepEqual(writeAgentFiles(root, files), ["SKILL.md", ".claude-plugin/plugin.json"]); + assert.equal(readFileSync(join(root, ".claude-plugin/plugin.json"), "utf8"), "{}\n"); + assert.deepEqual(writeAgentFiles(root, files, { check: true }), []); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); From 1f10222a4972410e89e676ac46f6405ec20fe9a6 Mon Sep 17 00:00:00 2001 From: bunizao <102936102+bunizao@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:48:34 +1100 Subject: [PATCH 3/6] fix(nouns): keep shared names off other commands and colour every grade status --- dist/program.js | 9 ++++++--- dist/theme.js | 4 ++-- src/program.ts | 9 ++++++--- src/theme.ts | 4 ++-- test/contract.test.js | 4 ++++ test/program.test.js | 11 ++++++++++- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/dist/program.js b/dist/program.js index 4a0f08a..db4a292 100644 --- a/dist/program.js +++ b/dist/program.js @@ -55,9 +55,12 @@ function describe(command, noun) { if (noun && ACTION_GROUPS.has(noun) && !ACTIONS[noun]?.has(command.name())) { throw new Error(`Unsupported action "${command.name()}" on group "${noun}".`); } - const canonical = noun ? undefined : SHARED.find((spec) => spec.aliases?.includes(currentNoun)); - if (canonical) { - throw new Error(`Noun "${currentNoun}" is an alias; name the command "${canonical.name}".`); + // A shared noun's name and aliases belong to that noun alone, whether used as a name or an alias. + for (const word of noun ? [] : [currentNoun, ...command.aliases()]) { + const owner = SHARED.find((spec) => spec.name === word || spec.aliases?.includes(word)); + if (owner && owner.name !== currentNoun) { + throw new Error(`"${word}" belongs to the shared noun "${owner.name}"; name that command "${owner.name}".`); + } } if (verb && !VERB_SET.has(verb)) { throw new Error(`Unsupported verb "${verb}" on noun "${noun}".`); diff --git a/dist/theme.js b/dist/theme.js index dd53bf0..cf0b3fe 100644 --- a/dist/theme.js +++ b/dist/theme.js @@ -10,8 +10,8 @@ const TONE_STYLE = { // Checked in order: a negated word ("not graded") must win over the word it contains. const TONE_WORDS = [ ["muted", /^(?:-|—|none|n\/a|unknown)$|\b(?:not[ _](?:started|submitted|graded|attempted|available|yet)|no submission|un(?:attempted|graded|submitted)|archived|hidden|unlisted|inactive|closed|disabled)\b/iu], - ["danger", /\b(?:overdue|late|fail(?:ed|ure)?|error|missing|rejected|need(?:s)?[ _]help|blocked|expired|redo|resubmit|demonstrate|exceeded)\b/iu], - ["warning", /\b(?:pending|draft|working|in[ _]progress|due soon|today|tomorrow|awaiting|unanswered|scheduled|discuss|attempted|partial)\b/iu], + ["danger", /\b(?:overdue|late|fail(?:ed|ure)?|error|missing|rejected|need(?:s)?[ _]help|blocked|expired|redo|resubmit|demonstrate|exceeded|unavailable)\b/iu], + ["warning", /\b(?:pending|draft|working|in[ _]progress|due soon|today|tomorrow|awaiting|unanswered|scheduled|discuss|attempted|partial|unreleased)\b/iu], ["success", /\b(?:complete[d]?|done|submitted|graded|accepted|passed|resolved|answered|ready[ _]for[ _]feedback|active|signed in|ok|success(?:ful)?|enrolled)\b/iu], ["accent", /\b(?:announcement|pinned|staff|endorsed|official|urgent)\b/iu], ["info", /\b(?:question|post|note|general|info)\b/iu], diff --git a/src/program.ts b/src/program.ts index 2b35e35..580ebde 100644 --- a/src/program.ts +++ b/src/program.ts @@ -97,9 +97,12 @@ function describe(command: Command, noun?: string): CommandDescription { if (noun && ACTION_GROUPS.has(noun) && !ACTIONS[noun]?.has(command.name())) { throw new Error(`Unsupported action "${command.name()}" on group "${noun}".`); } - const canonical = noun ? undefined : SHARED.find((spec) => spec.aliases?.includes(currentNoun)); - if (canonical) { - throw new Error(`Noun "${currentNoun}" is an alias; name the command "${canonical.name}".`); + // A shared noun's name and aliases belong to that noun alone, whether used as a name or an alias. + for (const word of noun ? [] : [currentNoun, ...command.aliases()]) { + const owner = SHARED.find((spec) => spec.name === word || spec.aliases?.includes(word)); + if (owner && owner.name !== currentNoun) { + throw new Error(`"${word}" belongs to the shared noun "${owner.name}"; name that command "${owner.name}".`); + } } if (verb && !VERB_SET.has(verb)) { throw new Error(`Unsupported verb "${verb}" on noun "${noun}".`); diff --git a/src/theme.ts b/src/theme.ts index 2090c6c..9abc679 100644 --- a/src/theme.ts +++ b/src/theme.ts @@ -35,8 +35,8 @@ const TONE_STYLE: Readonly[0]>> = { // Checked in order: a negated word ("not graded") must win over the word it contains. const TONE_WORDS: readonly (readonly [Tone, RegExp])[] = [ ["muted", /^(?:-|—|none|n\/a|unknown)$|\b(?:not[ _](?:started|submitted|graded|attempted|available|yet)|no submission|un(?:attempted|graded|submitted)|archived|hidden|unlisted|inactive|closed|disabled)\b/iu], - ["danger", /\b(?:overdue|late|fail(?:ed|ure)?|error|missing|rejected|need(?:s)?[ _]help|blocked|expired|redo|resubmit|demonstrate|exceeded)\b/iu], - ["warning", /\b(?:pending|draft|working|in[ _]progress|due soon|today|tomorrow|awaiting|unanswered|scheduled|discuss|attempted|partial)\b/iu], + ["danger", /\b(?:overdue|late|fail(?:ed|ure)?|error|missing|rejected|need(?:s)?[ _]help|blocked|expired|redo|resubmit|demonstrate|exceeded|unavailable)\b/iu], + ["warning", /\b(?:pending|draft|working|in[ _]progress|due soon|today|tomorrow|awaiting|unanswered|scheduled|discuss|attempted|partial|unreleased)\b/iu], ["success", /\b(?:complete[d]?|done|submitted|graded|accepted|passed|resolved|answered|ready[ _]for[ _]feedback|active|signed in|ok|success(?:ful)?|enrolled)\b/iu], ["accent", /\b(?:announcement|pinned|staff|endorsed|official|urgent)\b/iu], ["info", /\b(?:question|post|note|general|info)\b/iu], diff --git a/test/contract.test.js b/test/contract.test.js index 609dcd5..ecebd6c 100644 --- a/test/contract.test.js +++ b/test/contract.test.js @@ -108,5 +108,9 @@ test("toneOf reads the shared status vocabulary and negations", () => { assert.equal(toneOf("announcement"), "accent"); assert.equal(toneOf("question"), "info"); assert.equal(toneOf("Unit A"), undefined); + assert.deepEqual( + ["graded", "ungraded", "unreleased", "not_submitted", "unavailable"].map((status) => toneOf(status)), + ["success", "muted", "warning", "muted", "danger"], + ); assert.equal(toneOf("Rediscuss", { rediscuss: "warning" }), "warning"); }); diff --git a/test/program.test.js b/test/program.test.js index 44da466..f72941f 100644 --- a/test/program.test.js +++ b/test/program.test.js @@ -91,7 +91,16 @@ test("commandsJson rejects a shared alias used as the command name", () => { const program = createProgram({ name: "demo", version: "1.0.0", description: "Demo CLI" }); program.command("units").alias("courses").command("list"); - assert.throws(() => commandsJson(program), /Noun "units" is an alias; name the command "courses"/); + assert.throws(() => commandsJson(program), /"units" belongs to the shared noun "courses"; name that command "courses"/); +}); + +test("commandsJson rejects a shared name or alias on another command", () => { + for (const alias of ["units", "grades"]) { + const program = createProgram({ name: "demo", version: "1.0.0", description: "Demo CLI" }); + program.command("tasks").alias(alias).command("list"); + + assert.throws(() => commandsJson(program), new RegExp(`"${alias}" belongs to the shared noun`)); + } }); test("commandsJson rejects a contract verb a shared noun does not offer", () => { From 66561f70d4d453ef85659ef76cb47c84fb109eb8 Mon Sep 17 00:00:00 2001 From: bunizao <102936102+bunizao@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:02:23 +1100 Subject: [PATCH 4/6] feat(skills): keep SKILL.md small and give Workers an intents entry point The command tree moves to references/commands.md, MCP tools are listed by when to use them, the read rule appears only for a CLI with a read verb, and agents/openai.yaml no longer wraps. @bunizao/cli-kit/intents exports toolDescription without commander or Node builtins. --- README.md | 12 +++++++-- dist/skills.d.ts | 14 +++++++--- dist/skills.js | 40 +++++++++++++++++++++------- package.json | 4 +++ src/skills.ts | 62 ++++++++++++++++++++++++++++++++------------ test/intents.test.js | 9 +++++++ test/skills.test.js | 36 ++++++++++++++++++------- 7 files changed, 137 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index d639117..13c52ed 100644 --- a/README.md +++ b/README.md @@ -226,13 +226,21 @@ writeAgentFiles(process.cwd(), files); // { check: true } returns stale paths fo `agentFiles` returns, by path from the repository root: -- `SKILL.md`, for `npx skills add owner/repo`. It holds the intent table, the MCP tools, the - shared rules plus the CLI's own, the setup, the full command tree and the exit codes. +- `SKILL.md`, for `npx skills add owner/repo`. An agent loads it whole, so it stays short: the + intent table, which MCP tool answers what, the shared rules plus the CLI's own, the setup and the + exit codes. Each tool's full description stays with the MCP server. +- `references/commands.md`: every command with its arguments, options and whether it mutates. + `SKILL.md` links to it. - `.claude-plugin/plugin.json` and `.claude-plugin/marketplace.json`: the repository is a Claude Code plugin whose one skill is `SKILL.md`, and its own marketplace. Install with `claude plugin marketplace add owner/repo`, then `claude plugin install example-cli@example-cli`. + A repository with a root `CLAUDE.md` gets one validator warning, since that file is not plugin + context; `claude plugin validate .` passes, `--strict` does not. - `agents/openai.yaml`, when `openai` metadata is given. +An MCP server that runs on Cloudflare Workers imports `toolDescription` from +`@bunizao/cli-kit/intents`, which pulls in neither commander nor Node builtins. + `checkIntentCommands(program, nouns, intents)` throws when an intent's command no longer parses: a renamed noun or verb, a removed option, or the wrong binary name. Call it from the CLI's tests so the skill and the MCP descriptions cannot drift from the command tree. diff --git a/dist/skills.d.ts b/dist/skills.d.ts index 5abbff0..63eee0a 100644 --- a/dist/skills.d.ts +++ b/dist/skills.d.ts @@ -1,4 +1,4 @@ -import { type Intent } from "./intents.js"; +import type { Intent } from "./intents.js"; import type { ProgramDescription } from "./program.js"; export interface SkillOptions { /** The skill and plugin name, e.g. "edstem-cli". */ @@ -46,11 +46,19 @@ export interface AgentFilesOptions { readonly defaultPrompt: string; }; } -/** SKILL.md for one CLI: what people say, what to run, the rules, and the full command tree. */ +/** Where agentFiles puts the full command tree; SKILL.md links to it so the skill stays small. */ +export declare const COMMAND_REFERENCE = "references/commands.md"; +/** + * SKILL.md for one CLI: what people say, what to run, the rules and the setup. An agent loads + * it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's full description + * stays with the server. + */ export declare function renderSkill(options: SkillOptions): string; +/** The full command tree as a Markdown table, written to COMMAND_REFERENCE. */ +export declare function renderCommandReference(program: ProgramDescription): string; /** * Every agent packaging file for one CLI repository, keyed by path from the repository root: - * SKILL.md for `npx skills add`, the Claude Code plugin and its single-plugin marketplace, and + * SKILL.md for `npx skills add` with its command reference, the Claude Code plugin and its single-plugin marketplace, and * agents/openai.yaml when Codex metadata is given. */ export declare function agentFiles(options: AgentFilesOptions): Record; diff --git a/dist/skills.js b/dist/skills.js index cff9113..42f090e 100644 --- a/dist/skills.js +++ b/dist/skills.js @@ -2,19 +2,27 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { stringify } from "yaml"; import { EXIT_CODES } from "./errors.js"; -import { toolDescription } from "./intents.js"; +/** Where agentFiles puts the full command tree; SKILL.md links to it so the skill stays small. */ +export const COMMAND_REFERENCE = "references/commands.md"; +const READ_RULE = "`read` prints Markdown and never changes anything upstream."; const SHARED_RULES = [ "Run the narrowest command or tool that answers the request; one call per intent.", "Piped output is JSON and a terminal gets a table. `--fields a,b` keeps only the listed top-level fields.", - "`read` prints Markdown and never changes anything upstream.", "Run a mutating command only when the user asked for it. It needs a confirmation or `--yes`; `--dry-run` prints the plan first.", "Never print, log or store tokens, cookies or session files.", ]; -/** SKILL.md for one CLI: what people say, what to run, the rules, and the full command tree. */ +/** + * SKILL.md for one CLI: what people say, what to run, the rules and the setup. An agent loads + * it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's full description + * stays with the server. + */ export function renderSkill(options) { const binary = options.program.name; const commandIntents = options.intents.filter((intent) => intent.command); const toolIntents = options.intents.filter((intent) => intent.tool); + const hasRead = options.program.commands.some(function reads(command) { + return command.verb === "read" || command.commands.some(reads); + }); const exitCodes = Object.entries(EXIT_CODES) .map(([code, exit]) => `${exit} ${code}`) .join(", "); @@ -25,28 +33,42 @@ export function renderSkill(options) { "## What the user says, and what to run", markdownTable(["The user says", "Run", "Then"], commandIntents.map((intent) => [intent.when, `\`${intent.command}\``, intent.then ?? ""])), toolIntents.length - ? ["## MCP tools", markdownTable(["Tool", "Description"], toolIntents.map((intent) => [`\`${intent.tool}\``, toolDescription(intent)]))].join("\n\n") + ? ["## MCP tools", markdownTable(["The user says", "Tool"], toolIntents.map((intent) => [intent.when, `\`${intent.tool}\``]))].join("\n\n") : undefined, "## Rules", - [...SHARED_RULES, ...(options.rules ?? []), `\`${binary} commands --json\` is the source of truth for the command tree.`] + [ + SHARED_RULES[0], + SHARED_RULES[1], + ...(hasRead ? [READ_RULE] : []), + ...SHARED_RULES.slice(2), + ...(options.rules ?? []), + `Every command, argument and option is in [${COMMAND_REFERENCE}](${COMMAND_REFERENCE}); \`${binary} commands --json\` is the source of truth.`, + ] .map((rule) => `- ${rule}`) .join("\n"), options.setup ? `## Setup\n\n${options.setup}` : undefined, - "## Commands", - markdownTable(["Command", "Description", "Arguments", "Options", "Mutating"], options.program.commands.flatMap((command) => commandRows(binary, command))), "## Errors", `Errors are printed once on stderr. Exit codes: 0 success, ${exitCodes}.`, ].filter((section) => Boolean(section)).join("\n\n") + "\n"; } +/** The full command tree as a Markdown table, written to COMMAND_REFERENCE. */ +export function renderCommandReference(program) { + return [ + `# ${program.name} commands`, + `Generated from \`${program.name} commands --json\` for version ${program.version}.`, + markdownTable(["Command", "Description", "Arguments", "Options", "Mutating"], program.commands.flatMap((command) => commandRows(program.name, command))), + ].join("\n\n") + "\n"; +} /** * Every agent packaging file for one CLI repository, keyed by path from the repository root: - * SKILL.md for `npx skills add`, the Claude Code plugin and its single-plugin marketplace, and + * SKILL.md for `npx skills add` with its command reference, the Claude Code plugin and its single-plugin marketplace, and * agents/openai.yaml when Codex metadata is given. */ export function agentFiles(options) { const { skill } = options; const files = { "SKILL.md": renderSkill(skill), + [COMMAND_REFERENCE]: renderCommandReference(skill.program), ".claude-plugin/plugin.json": json({ name: skill.name, version: options.version, @@ -74,7 +96,7 @@ export function agentFiles(options) { short_description: options.openai.shortDescription, default_prompt: options.openai.defaultPrompt, }, - }); + }, { lineWidth: 0 }); } return files; } diff --git a/package.json b/package.json index 75d63fe..4802953 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,10 @@ ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" + }, + "./intents": { + "types": "./dist/intents.d.ts", + "import": "./dist/intents.js" } }, "files": [ diff --git a/src/skills.ts b/src/skills.ts index 3d3c1b5..6254f42 100644 --- a/src/skills.ts +++ b/src/skills.ts @@ -4,7 +4,7 @@ import { dirname, join } from "node:path"; import { stringify } from "yaml"; import { EXIT_CODES } from "./errors.js"; -import { toolDescription, type Intent } from "./intents.js"; +import type { Intent } from "./intents.js"; import type { CommandDescription, ProgramDescription } from "./program.js"; export interface SkillOptions { @@ -43,19 +43,29 @@ export interface AgentFilesOptions { readonly openai?: { readonly displayName: string; readonly shortDescription: string; readonly defaultPrompt: string }; } +/** Where agentFiles puts the full command tree; SKILL.md links to it so the skill stays small. */ +export const COMMAND_REFERENCE = "references/commands.md"; + +const READ_RULE = "`read` prints Markdown and never changes anything upstream."; const SHARED_RULES = [ "Run the narrowest command or tool that answers the request; one call per intent.", "Piped output is JSON and a terminal gets a table. `--fields a,b` keeps only the listed top-level fields.", - "`read` prints Markdown and never changes anything upstream.", "Run a mutating command only when the user asked for it. It needs a confirmation or `--yes`; `--dry-run` prints the plan first.", "Never print, log or store tokens, cookies or session files.", ]; -/** SKILL.md for one CLI: what people say, what to run, the rules, and the full command tree. */ +/** + * SKILL.md for one CLI: what people say, what to run, the rules and the setup. An agent loads + * it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's full description + * stays with the server. + */ export function renderSkill(options: SkillOptions): string { const binary = options.program.name; const commandIntents = options.intents.filter((intent) => intent.command); const toolIntents = options.intents.filter((intent) => intent.tool); + const hasRead = options.program.commands.some(function reads(command): boolean { + return command.verb === "read" || command.commands.some(reads); + }); const exitCodes = Object.entries(EXIT_CODES) .map(([code, exit]) => `${exit} ${code}`) .join(", "); @@ -70,32 +80,47 @@ export function renderSkill(options: SkillOptions): string { commandIntents.map((intent) => [intent.when, `\`${intent.command}\``, intent.then ?? ""]), ), toolIntents.length - ? ["## MCP tools", markdownTable(["Tool", "Description"], toolIntents.map((intent) => [`\`${intent.tool}\``, toolDescription(intent)]))].join("\n\n") + ? ["## MCP tools", markdownTable(["The user says", "Tool"], toolIntents.map((intent) => [intent.when, `\`${intent.tool}\``]))].join("\n\n") : undefined, "## Rules", - [...SHARED_RULES, ...(options.rules ?? []), `\`${binary} commands --json\` is the source of truth for the command tree.`] + [ + SHARED_RULES[0], + SHARED_RULES[1], + ...(hasRead ? [READ_RULE] : []), + ...SHARED_RULES.slice(2), + ...(options.rules ?? []), + `Every command, argument and option is in [${COMMAND_REFERENCE}](${COMMAND_REFERENCE}); \`${binary} commands --json\` is the source of truth.`, + ] .map((rule) => `- ${rule}`) .join("\n"), options.setup ? `## Setup\n\n${options.setup}` : undefined, - "## Commands", - markdownTable( - ["Command", "Description", "Arguments", "Options", "Mutating"], - options.program.commands.flatMap((command) => commandRows(binary, command)), - ), "## Errors", `Errors are printed once on stderr. Exit codes: 0 success, ${exitCodes}.`, ].filter((section): section is string => Boolean(section)).join("\n\n") + "\n"; } +/** The full command tree as a Markdown table, written to COMMAND_REFERENCE. */ +export function renderCommandReference(program: ProgramDescription): string { + return [ + `# ${program.name} commands`, + `Generated from \`${program.name} commands --json\` for version ${program.version}.`, + markdownTable( + ["Command", "Description", "Arguments", "Options", "Mutating"], + program.commands.flatMap((command) => commandRows(program.name, command)), + ), + ].join("\n\n") + "\n"; +} + /** * Every agent packaging file for one CLI repository, keyed by path from the repository root: - * SKILL.md for `npx skills add`, the Claude Code plugin and its single-plugin marketplace, and + * SKILL.md for `npx skills add` with its command reference, the Claude Code plugin and its single-plugin marketplace, and * agents/openai.yaml when Codex metadata is given. */ export function agentFiles(options: AgentFilesOptions): Record { const { skill } = options; const files: Record = { "SKILL.md": renderSkill(skill), + [COMMAND_REFERENCE]: renderCommandReference(skill.program), ".claude-plugin/plugin.json": json({ name: skill.name, version: options.version, @@ -117,13 +142,16 @@ export function agentFiles(options: AgentFilesOptions): Record { }), }; if (options.openai) { - files["agents/openai.yaml"] = stringify({ - interface: { - display_name: options.openai.displayName, - short_description: options.openai.shortDescription, - default_prompt: options.openai.defaultPrompt, + files["agents/openai.yaml"] = stringify( + { + interface: { + display_name: options.openai.displayName, + short_description: options.openai.shortDescription, + default_prompt: options.openai.defaultPrompt, + }, }, - }); + { lineWidth: 0 }, + ); } return files; } diff --git a/test/intents.test.js b/test/intents.test.js index 22fdb79..c6310ac 100644 --- a/test/intents.test.js +++ b/test/intents.test.js @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import test from "node:test"; import { SHARED_NOUNS, checkIntentCommands, createProgram, toolDescription } from "../dist/index.js"; @@ -47,3 +48,11 @@ test("checkIntentCommands reports renamed commands, removed options and the wron }, ); }); + +test("the intents entry point imports nothing but nouns, so a Worker can bundle it", async () => { + const intents = await import("@bunizao/cli-kit/intents"); + assert.equal(typeof intents.toolDescription, "function"); + const imports = (file) => [...readFileSync(new URL(`../dist/${file}`, import.meta.url), "utf8").matchAll(/^import .* from "([^"]+)";$/gm)].map((match) => match[1]); + assert.deepEqual(imports("intents.js"), ["./nouns.js"]); + assert.deepEqual(imports("nouns.js"), []); +}); diff --git a/test/skills.test.js b/test/skills.test.js index eafd572..4ee33d3 100644 --- a/test/skills.test.js +++ b/test/skills.test.js @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; -import { agentFiles, commandsJson, createProgram, mutating, renderSkill, writeAgentFiles } from "../dist/index.js"; +import { COMMAND_REFERENCE, agentFiles, commandsJson, createProgram, mutating, renderCommandReference, renderSkill, writeAgentFiles } from "../dist/index.js"; function skillOptions() { const program = createProgram({ name: "demo", version: "1.2.3", description: "Demo CLI" }); @@ -24,19 +24,36 @@ function skillOptions() { }; } -test("renderSkill writes frontmatter, intents, tools, rules, commands and exit codes", () => { +test("renderSkill writes frontmatter, intents, tools, rules and exit codes, and links the command tree", () => { const skill = renderSkill(skillOptions()); assert.match(skill, /^---\nname: demo-cli\ndescription: "Read demo courses: grades, tasks and files."\n---\n\n# Demo CLI\n\nCOURSE is a code or id.\n/); assert.match(skill, /\| my courses \| `demo courses` \| pass a code as COURSE \|/); - assert.doesNotMatch(skill, /\| a widget \|/); - assert.match(skill, /\| `show_widget` \| A widget\. Use when the user says "a widget"\. \|/); - assert.match(skill, /- Ask before submitting\.\n- `demo commands --json` is the source of truth/); - assert.match(skill, /\| `demo tasks submit` \| Submit a task\. \| \| {2}\| yes \|/); + // MCP tools are listed by when to use them; the full description stays with the server. + assert.match(skill, /## MCP tools\n\n\| The user says \| Tool \|\n\| --- \| --- \|\n\| my courses \| `list_courses` \|\n\| a widget \| `show_widget` \|/); + assert.doesNotMatch(skill, /Use when the user says/); + assert.match(skill, /- Ask before submitting\.\n- Every command, argument and option is in \[references\/commands\.md\]\(references\/commands\.md\); `demo commands --json` is the source of truth\./); + assert.doesNotMatch(skill, /`demo tasks submit`/); assert.match(skill, /Exit codes: 0 success, 2 usage, 3 auth/); assert.ok(skill.endsWith("\n")); }); +test("renderSkill states the read rule only for a CLI with a read verb", () => { + assert.doesNotMatch(renderSkill(skillOptions()), /`read` prints Markdown/); + + const program = createProgram({ name: "demo", version: "1.2.3", description: "Demo CLI" }); + program.command("tasks").command("read "); + assert.match(renderSkill({ ...skillOptions(), program: commandsJson(program) }), /- `read` prints Markdown and never changes anything upstream\./); +}); + +test("renderCommandReference lists every command with its arguments, options and mutations", () => { + const reference = renderCommandReference(skillOptions().program); + + assert.match(reference, /^# demo commands\n\nGenerated from `demo commands --json` for version 1\.2\.3\./); + assert.match(reference, /\| `demo courses list` \| List courses\. \| {2}\| {2}\| no \|/); + assert.match(reference, /\| `demo tasks submit` \| Submit a task\. \| \| {2}\| yes \|/); +}); + test("agentFiles packages the skill as a Claude Code plugin, a marketplace and Codex metadata", () => { const files = agentFiles({ skill: skillOptions(), @@ -45,10 +62,10 @@ test("agentFiles packages the skill as a Claude Code plugin, a marketplace and C author: { name: "example" }, license: "MIT", mcpServers: { demo: { command: "npx", args: ["-y", "demo-cli", "mcp"] } }, - openai: { displayName: "Demo CLI", shortDescription: "Read demo courses", defaultPrompt: "Use $demo-cli." }, + openai: { displayName: "Demo CLI", shortDescription: "Read demo courses", defaultPrompt: `Use $demo-cli ${"to read courses ".repeat(6)}` }, }); - assert.deepEqual(Object.keys(files), ["SKILL.md", ".claude-plugin/plugin.json", ".claude-plugin/marketplace.json", "agents/openai.yaml"]); + assert.deepEqual(Object.keys(files), ["SKILL.md", COMMAND_REFERENCE, ".claude-plugin/plugin.json", ".claude-plugin/marketplace.json", "agents/openai.yaml"]); assert.deepEqual(JSON.parse(files[".claude-plugin/plugin.json"]), { name: "demo-cli", version: "1.2.3", @@ -68,7 +85,8 @@ test("agentFiles packages the skill as a Claude Code plugin, a marketplace and C }); assert.equal( files["agents/openai.yaml"], - "interface:\n display_name: Demo CLI\n short_description: Read demo courses\n default_prompt: Use $demo-cli.\n", + // One line per field, however long the prompt. + `interface:\n display_name: Demo CLI\n short_description: Read demo courses\n default_prompt: "Use $demo-cli ${"to read courses ".repeat(6)}"\n`, ); }); From 71e647103a768491fc2ad7ea163c42a2ea1545b8 Mon Sep 17 00:00:00 2001 From: bunizao <102936102+bunizao@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:41:14 +1100 Subject: [PATCH 5/6] feat(skills)!: trigger descriptions, one intent table, checklists and a skills// folder SkillOptions gains useWhen, mcpServer and workflows. The frontmatter description reads ". Use when ." and renderSkill rejects names and descriptions outside the Agent Skills limits or written in the first or second person. The CLI and MCP columns share one table, tools are qualified as server:tool, and a CLI with a mutating command gets a shared dry-run, confirm, apply, read-back checklist. Rules say why; exit codes move to references/commands.md. agentFiles now writes the skill to skills// with its references and agents/openai.yaml, takes extra reference pages, and drops the plugin's skills field since Claude Code loads skills/ by default. --- README.md | 33 +++++-- dist/skills.d.ts | 39 +++++++-- dist/skills.js | 179 ++++++++++++++++++++++++++------------ src/skills.ts | 206 +++++++++++++++++++++++++++++++------------- test/skills.test.js | 78 ++++++++++++----- 5 files changed, 380 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index 13c52ed..930ae28 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,16 @@ const INTENTS: readonly Intent[] = [ server.registerTool("list_grades", { description: toolDescription(INTENTS[0]) /* , schema */ }, handler); const files = agentFiles({ - skill: { name: "example-cli", description, title: "Example CLI", intents: INTENTS, program: commandsJson(program) }, + skill: { + name: "example-cli", + description: "Reads Example courses, grades and deadlines", + useWhen: 'the user names Example or asks "what\'s due", "my grades"', + title: "Example CLI", + intents: INTENTS, + mcpServer: "example", + workflows: [{ title: "Submit an assignment", steps: ["...", "Check the submission status"] }], + program: commandsJson(program), + }, version, repository: "https://github.com/example/example-cli", author: { name: "example" }, @@ -226,17 +235,25 @@ writeAgentFiles(process.cwd(), files); // { check: true } returns stale paths fo `agentFiles` returns, by path from the repository root: -- `SKILL.md`, for `npx skills add owner/repo`. An agent loads it whole, so it stays short: the - intent table, which MCP tool answers what, the shared rules plus the CLI's own, the setup and the - exit codes. Each tool's full description stays with the MCP server. -- `references/commands.md`: every command with its arguments, options and whether it mutates. - `SKILL.md` links to it. +- `skills//SKILL.md`, which `npx skills add owner/repo` and Claude Code both find. An agent + loads it whole, so it stays short: one table of what the user says against the CLI command and + the `server:tool` MCP tool, the workflows as copyable checklists, the rules and the setup. A CLI + with a mutating command gets the shared dry-run, confirm, apply, read-back workflow first. Each + tool's full description stays with the MCP server. +- The frontmatter description is `description` then "Use when `useWhen`". `renderSkill` throws + when the name or description breaks the Agent Skills limits (lowercase hyphenated name, 1024 + characters, no `<` or `>`) or uses "I" or "you" outside quotes. +- `skills//references/commands.md`: every command with its arguments, options and whether + it mutates, plus the exit codes. Pages in `references` are written next to it. +- `skills//agents/openai.yaml`, when `openai` metadata is given. - `.claude-plugin/plugin.json` and `.claude-plugin/marketplace.json`: the repository is a Claude - Code plugin whose one skill is `SKILL.md`, and its own marketplace. Install with + Code plugin, which loads `skills/` by default, and its own marketplace. Install with `claude plugin marketplace add owner/repo`, then `claude plugin install example-cli@example-cli`. A repository with a root `CLAUDE.md` gets one validator warning, since that file is not plugin context; `claude plugin validate .` passes, `--strict` does not. -- `agents/openai.yaml`, when `openai` metadata is given. + +`writeAgentFiles` never deletes, so a CLI moving to this layout removes its old root `SKILL.md`, +`references/` and `agents/` itself. An MCP server that runs on Cloudflare Workers imports `toolDescription` from `@bunizao/cli-kit/intents`, which pulls in neither commander nor Node builtins. diff --git a/dist/skills.d.ts b/dist/skills.d.ts index 63eee0a..55b5db3 100644 --- a/dist/skills.d.ts +++ b/dist/skills.d.ts @@ -3,12 +3,21 @@ import type { ProgramDescription } from "./program.js"; export interface SkillOptions { /** The skill and plugin name, e.g. "edstem-cli". */ readonly name: string; - /** When to use the skill; put the trigger first, hosts truncate long descriptions. */ + /** What the skill does, in the third person. It is also the plugin's description. */ readonly description: string; + /** + * When an agent should load the skill, rendered after the description as "Use when …". Name the + * site and quote what people say; agents undertrigger, so list more phrases rather than fewer. + */ + readonly useWhen: string; readonly title: string; /** Markdown between the title and the intent table: the surfaces and the reference vocabulary. */ readonly intro?: string; readonly intents: readonly Intent[]; + /** The MCP server name tools are qualified with: "moodle" renders `moodle:due`. Required when an intent has a tool. */ + readonly mcpServer?: string; + /** Multi-step tasks rendered as checklists, after the shared one for mutating commands. */ + readonly workflows?: readonly Workflow[]; /** Rules for this CLI only; the shared contract rules are always included. */ readonly rules?: readonly string[]; /** Markdown for the Setup section. */ @@ -16,6 +25,12 @@ export interface SkillOptions { /** `commandsJson(program)` output. */ readonly program: ProgramDescription; } +export interface Workflow { + /** An imperative title, e.g. "Submit an assignment". */ + readonly title: string; + /** In order; end with a step that checks the result. */ + readonly steps: readonly string[]; +} /** A local stdio server, or a hosted Streamable HTTP server. */ export type McpServer = { readonly command: string; @@ -39,27 +54,33 @@ export interface AgentFilesOptions { readonly keywords?: readonly string[]; /** MCP servers the Claude Code plugin starts, keyed by server name. */ readonly mcpServers?: Readonly>; - /** Codex skill metadata, written to agents/openai.yaml. */ + /** Codex skill metadata, written to agents/openai.yaml in the skill folder. */ readonly openai?: { readonly displayName: string; readonly shortDescription: string; readonly defaultPrompt: string; }; + /** Hand-written reference pages by file name, written next to the command reference. */ + readonly references?: Readonly>; } -/** Where agentFiles puts the full command tree; SKILL.md links to it so the skill stays small. */ +/** Where agentFiles puts the full command tree, relative to the skill folder; SKILL.md links to it. */ export declare const COMMAND_REFERENCE = "references/commands.md"; +/** The frontmatter description: what the skill does, then when to use it. */ +export declare function skillDescription(options: Pick): string; /** - * SKILL.md for one CLI: what people say, what to run, the rules and the setup. An agent loads - * it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's full description - * stays with the server. + * SKILL.md for one CLI: what people say, what to run, the workflows, the rules and the setup. + * An agent loads it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's + * full description stays with the server. Throws when the name or description breaks the + * Agent Skills limits. */ export declare function renderSkill(options: SkillOptions): string; -/** The full command tree as a Markdown table, written to COMMAND_REFERENCE. */ +/** The full command tree and the exit codes, written to COMMAND_REFERENCE. */ export declare function renderCommandReference(program: ProgramDescription): string; /** * Every agent packaging file for one CLI repository, keyed by path from the repository root: - * SKILL.md for `npx skills add` with its command reference, the Claude Code plugin and its single-plugin marketplace, and - * agents/openai.yaml when Codex metadata is given. + * the skill folder skills// (SKILL.md, its references and Codex's agents/openai.yaml), which + * `npx skills add` and Claude Code both find, plus the Claude Code plugin and its single-plugin + * marketplace. */ export declare function agentFiles(options: AgentFilesOptions): Record; /** diff --git a/dist/skills.js b/dist/skills.js index 42f090e..852c192 100644 --- a/dist/skills.js +++ b/dist/skills.js @@ -2,95 +2,100 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { stringify } from "yaml"; import { EXIT_CODES } from "./errors.js"; -/** Where agentFiles puts the full command tree; SKILL.md links to it so the skill stays small. */ +/** Where agentFiles puts the full command tree, relative to the skill folder; SKILL.md links to it. */ export const COMMAND_REFERENCE = "references/commands.md"; -const READ_RULE = "`read` prints Markdown and never changes anything upstream."; -const SHARED_RULES = [ - "Run the narrowest command or tool that answers the request; one call per intent.", - "Piped output is JSON and a terminal gets a table. `--fields a,b` keeps only the listed top-level fields.", - "Run a mutating command only when the user asked for it. It needs a confirmation or `--yes`; `--dry-run` prints the plan first.", - "Never print, log or store tokens, cookies or session files.", -]; +const RULES = { + narrow: "Start from the table row that matches the request: the narrowest command or tool answers sooner and costs fewer tokens than a broad listing.", + output: "Piped output is JSON. `--fields a,b` keeps only those top-level fields, which keeps long lists cheap.", + read: "`read` prints Markdown and changes nothing upstream, so it is safe to run unasked.", + mutate: "Change something upstream only when the user asked for that change: other people see it, and most changes cannot be undone.", + secrets: "Keep tokens, cookies and session files out of replies, logs and files; each one is a live sign-in.", + errors: "A failure prints one error with a `hint` on stderr; follow the hint before retrying.", +}; +const MUTATION_WORKFLOW = { + title: "Change something upstream", + steps: [ + "Check the user asked for this change, on these targets", + "Run the command with `--dry-run` and show the plan", + "Get the user's yes to that plan", + "Run it again with `--yes`", + "Read the result back with the matching `list` or `show` and report what changed", + ], +}; +/** The frontmatter description: what the skill does, then when to use it. */ +export function skillDescription(options) { + return `${sentence(options.description)} Use when ${sentence(options.useWhen)}`; +} /** - * SKILL.md for one CLI: what people say, what to run, the rules and the setup. An agent loads - * it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's full description - * stays with the server. + * SKILL.md for one CLI: what people say, what to run, the workflows, the rules and the setup. + * An agent loads it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's + * full description stays with the server. Throws when the name or description breaks the + * Agent Skills limits. */ export function renderSkill(options) { + const description = skillDescription(options); + const problems = frontmatterProblems(options.name, description); + if (problems.length) + throw new Error(`Skill "${options.name}" frontmatter:\n${problems.join("\n")}`); const binary = options.program.name; - const commandIntents = options.intents.filter((intent) => intent.command); - const toolIntents = options.intents.filter((intent) => intent.tool); - const hasRead = options.program.commands.some(function reads(command) { - return command.verb === "read" || command.commands.some(reads); - }); - const exitCodes = Object.entries(EXIT_CODES) - .map(([code, exit]) => `${exit} ${code}`) - .join(", "); + const hasRead = someCommand(options.program, (command) => command.verb === "read"); + const hasMutation = someCommand(options.program, (command) => command.mutating); + const workflows = [...(hasMutation ? [MUTATION_WORKFLOW] : []), ...(options.workflows ?? [])]; return [ - stringifyFrontmatter({ name: options.name, description: options.description }), + stringifyFrontmatter({ name: options.name, description }), `# ${options.title}`, options.intro, "## What the user says, and what to run", - markdownTable(["The user says", "Run", "Then"], commandIntents.map((intent) => [intent.when, `\`${intent.command}\``, intent.then ?? ""])), - toolIntents.length - ? ["## MCP tools", markdownTable(["The user says", "Tool"], toolIntents.map((intent) => [intent.when, `\`${intent.tool}\``]))].join("\n\n") + intentTable(options), + workflows.length + ? ["## Workflows", "Copy the checklist into your reply and tick each step as it is done; the last step is what shows the change landed.", ...workflows.map(renderWorkflow)].join("\n\n") : undefined, "## Rules", [ - SHARED_RULES[0], - SHARED_RULES[1], - ...(hasRead ? [READ_RULE] : []), - ...SHARED_RULES.slice(2), + RULES.narrow, + RULES.output, + ...(hasRead ? [RULES.read] : []), + ...(hasMutation ? [RULES.mutate] : []), + RULES.secrets, + RULES.errors, ...(options.rules ?? []), - `Every command, argument and option is in [${COMMAND_REFERENCE}](${COMMAND_REFERENCE}); \`${binary} commands --json\` is the source of truth.`, + `Every command, argument, option and exit code is in [${COMMAND_REFERENCE}](${COMMAND_REFERENCE}); \`${binary} commands --json\` is the source of truth.`, ] .map((rule) => `- ${rule}`) .join("\n"), options.setup ? `## Setup\n\n${options.setup}` : undefined, - "## Errors", - `Errors are printed once on stderr. Exit codes: 0 success, ${exitCodes}.`, ].filter((section) => Boolean(section)).join("\n\n") + "\n"; } -/** The full command tree as a Markdown table, written to COMMAND_REFERENCE. */ +/** The full command tree and the exit codes, written to COMMAND_REFERENCE. */ export function renderCommandReference(program) { + const exitCodes = Object.entries(EXIT_CODES).map(([code, exit]) => [String(exit), code]); return [ `# ${program.name} commands`, `Generated from \`${program.name} commands --json\` for version ${program.version}.`, markdownTable(["Command", "Description", "Arguments", "Options", "Mutating"], program.commands.flatMap((command) => commandRows(program.name, command))), + "## Exit codes", + "With `--json`, a failure is one `{ ok: false, error: { code, message, hint } }` object on stderr.", + markdownTable(["Exit", "Error code"], [["0", "success"], ...exitCodes]), ].join("\n\n") + "\n"; } /** * Every agent packaging file for one CLI repository, keyed by path from the repository root: - * SKILL.md for `npx skills add` with its command reference, the Claude Code plugin and its single-plugin marketplace, and - * agents/openai.yaml when Codex metadata is given. + * the skill folder skills// (SKILL.md, its references and Codex's agents/openai.yaml), which + * `npx skills add` and Claude Code both find, plus the Claude Code plugin and its single-plugin + * marketplace. */ export function agentFiles(options) { const { skill } = options; + const folder = `skills/${skill.name}`; const files = { - "SKILL.md": renderSkill(skill), - [COMMAND_REFERENCE]: renderCommandReference(skill.program), - ".claude-plugin/plugin.json": json({ - name: skill.name, - version: options.version, - description: skill.description, - author: options.author, - homepage: options.repository, - repository: options.repository, - ...(options.license ? { license: options.license } : {}), - ...(options.keywords?.length ? { keywords: options.keywords } : {}), - // The repository root is the plugin root, and its SKILL.md is the plugin's one skill. - skills: "./", - ...(options.mcpServers ? { mcpServers: options.mcpServers } : {}), - }), - ".claude-plugin/marketplace.json": json({ - name: skill.name, - description: skill.description, - owner: options.author, - plugins: [{ name: skill.name, source: "./", description: skill.description }], - }), + [`${folder}/SKILL.md`]: renderSkill(skill), + [`${folder}/${COMMAND_REFERENCE}`]: renderCommandReference(skill.program), }; + for (const [name, content] of Object.entries(options.references ?? {})) { + files[`${folder}/${dirname(COMMAND_REFERENCE)}/${name}`] = content; + } if (options.openai) { - files["agents/openai.yaml"] = stringify({ + files[`${folder}/agents/openai.yaml`] = stringify({ interface: { display_name: options.openai.displayName, short_description: options.openai.shortDescription, @@ -98,6 +103,24 @@ export function agentFiles(options) { }, }, { lineWidth: 0 }); } + // The repository root is the plugin root; Claude Code loads skills/ from it by default. + files[".claude-plugin/plugin.json"] = json({ + name: skill.name, + version: options.version, + description: skill.description, + author: options.author, + homepage: options.repository, + repository: options.repository, + ...(options.license ? { license: options.license } : {}), + ...(options.keywords?.length ? { keywords: options.keywords } : {}), + ...(options.mcpServers ? { mcpServers: options.mcpServers } : {}), + }); + files[".claude-plugin/marketplace.json"] = json({ + name: skill.name, + description: skill.description, + owner: options.author, + plugins: [{ name: skill.name, source: "./", description: skill.description }], + }); return files; } /** @@ -116,6 +139,48 @@ export function writeAgentFiles(root, files, options = {}) { } return stale; } +// The limits from the Agent Skills spec, plus the third person: the description is read as +// part of the system prompt, where "I" and "you" point at the wrong speaker. +function frontmatterProblems(name, description) { + const problems = []; + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(name) || name.length > 64) + problems.push(`name must be lowercase words joined by hyphens, at most 64 characters`); + if (/anthropic|claude/.test(name)) + problems.push(`name must not contain "anthropic" or "claude"`); + if (description.length > 1024) + problems.push(`description is ${description.length} characters; the limit is 1024`); + if (/[<>]/.test(description)) + problems.push("description must not contain < or >"); + // Quoted phrases are what the user says, so "my grades" is fine there. + const unquoted = description.replace(/"[^"]*"|“[^”]*”/g, ""); + const person = /\b(?:I|me|my|we|our|you|your)\b/i.exec(unquoted); + if (person) + problems.push(`description must be in the third person outside quotes; found "${person[0]}"`); + return problems; +} +function intentTable(options) { + const { intents, mcpServer } = options; + if (!mcpServer && intents.some((intent) => intent.tool)) { + throw new Error(`Skill "${options.name}" has MCP tools but no mcpServer to qualify them with.`); + } + const columns = [["The user says", (intent) => intent.when]]; + if (intents.some((intent) => intent.command)) + columns.push(["CLI", (intent) => (intent.command ? `\`${intent.command}\`` : "")]); + if (intents.some((intent) => intent.tool)) + columns.push(["MCP tool", (intent) => (intent.tool ? `\`${mcpServer}:${intent.tool}\`` : "")]); + if (intents.some((intent) => intent.then)) + columns.push(["Then", (intent) => intent.then ?? ""]); + return markdownTable(columns.map(([header]) => header), intents.map((intent) => columns.map(([, cell]) => cell(intent)))); +} +// A fenced block, so the agent copies it verbatim and ticks it in its reply. +function renderWorkflow(workflow) { + return [`### ${workflow.title}`, ["```text", ...workflow.steps.map((step) => `- [ ] ${step}`), "```"].join("\n")].join("\n\n"); +} +function someCommand(program, test) { + return program.commands.some(function visit(command) { + return test(command) || command.commands.some(visit); + }); +} function commandRows(binary, command, parents = []) { const path = [...parents, command.name]; const row = [ @@ -143,6 +208,10 @@ function markdownTable(headers, rows) { function escapeCell(value) { return value.replace(/\|/g, "\\|").replace(/\n/g, "
"); } +function sentence(text) { + const trimmed = text.trim(); + return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`; +} function json(value) { return `${JSON.stringify(value, null, 2)}\n`; } diff --git a/src/skills.ts b/src/skills.ts index 6254f42..09d3592 100644 --- a/src/skills.ts +++ b/src/skills.ts @@ -10,12 +10,21 @@ import type { CommandDescription, ProgramDescription } from "./program.js"; export interface SkillOptions { /** The skill and plugin name, e.g. "edstem-cli". */ readonly name: string; - /** When to use the skill; put the trigger first, hosts truncate long descriptions. */ + /** What the skill does, in the third person. It is also the plugin's description. */ readonly description: string; + /** + * When an agent should load the skill, rendered after the description as "Use when …". Name the + * site and quote what people say; agents undertrigger, so list more phrases rather than fewer. + */ + readonly useWhen: string; readonly title: string; /** Markdown between the title and the intent table: the surfaces and the reference vocabulary. */ readonly intro?: string; readonly intents: readonly Intent[]; + /** The MCP server name tools are qualified with: "moodle" renders `moodle:due`. Required when an intent has a tool. */ + readonly mcpServer?: string; + /** Multi-step tasks rendered as checklists, after the shared one for mutating commands. */ + readonly workflows?: readonly Workflow[]; /** Rules for this CLI only; the shared contract rules are always included. */ readonly rules?: readonly string[]; /** Markdown for the Setup section. */ @@ -24,6 +33,13 @@ export interface SkillOptions { readonly program: ProgramDescription; } +export interface Workflow { + /** An imperative title, e.g. "Submit an assignment". */ + readonly title: string; + /** In order; end with a step that checks the result. */ + readonly steps: readonly string[]; +} + /** A local stdio server, or a hosted Streamable HTTP server. */ export type McpServer = | { readonly command: string; readonly args?: readonly string[]; readonly env?: Readonly> } @@ -39,68 +55,85 @@ export interface AgentFilesOptions { readonly keywords?: readonly string[]; /** MCP servers the Claude Code plugin starts, keyed by server name. */ readonly mcpServers?: Readonly>; - /** Codex skill metadata, written to agents/openai.yaml. */ + /** Codex skill metadata, written to agents/openai.yaml in the skill folder. */ readonly openai?: { readonly displayName: string; readonly shortDescription: string; readonly defaultPrompt: string }; + /** Hand-written reference pages by file name, written next to the command reference. */ + readonly references?: Readonly>; } -/** Where agentFiles puts the full command tree; SKILL.md links to it so the skill stays small. */ +/** Where agentFiles puts the full command tree, relative to the skill folder; SKILL.md links to it. */ export const COMMAND_REFERENCE = "references/commands.md"; -const READ_RULE = "`read` prints Markdown and never changes anything upstream."; -const SHARED_RULES = [ - "Run the narrowest command or tool that answers the request; one call per intent.", - "Piped output is JSON and a terminal gets a table. `--fields a,b` keeps only the listed top-level fields.", - "Run a mutating command only when the user asked for it. It needs a confirmation or `--yes`; `--dry-run` prints the plan first.", - "Never print, log or store tokens, cookies or session files.", -]; +const RULES = { + narrow: "Start from the table row that matches the request: the narrowest command or tool answers sooner and costs fewer tokens than a broad listing.", + output: "Piped output is JSON. `--fields a,b` keeps only those top-level fields, which keeps long lists cheap.", + read: "`read` prints Markdown and changes nothing upstream, so it is safe to run unasked.", + mutate: "Change something upstream only when the user asked for that change: other people see it, and most changes cannot be undone.", + secrets: "Keep tokens, cookies and session files out of replies, logs and files; each one is a live sign-in.", + errors: "A failure prints one error with a `hint` on stderr; follow the hint before retrying.", +}; + +const MUTATION_WORKFLOW: Workflow = { + title: "Change something upstream", + steps: [ + "Check the user asked for this change, on these targets", + "Run the command with `--dry-run` and show the plan", + "Get the user's yes to that plan", + "Run it again with `--yes`", + "Read the result back with the matching `list` or `show` and report what changed", + ], +}; + +/** The frontmatter description: what the skill does, then when to use it. */ +export function skillDescription(options: Pick): string { + return `${sentence(options.description)} Use when ${sentence(options.useWhen)}`; +} /** - * SKILL.md for one CLI: what people say, what to run, the rules and the setup. An agent loads - * it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's full description - * stays with the server. + * SKILL.md for one CLI: what people say, what to run, the workflows, the rules and the setup. + * An agent loads it whole, so the command tree lives in COMMAND_REFERENCE and each MCP tool's + * full description stays with the server. Throws when the name or description breaks the + * Agent Skills limits. */ export function renderSkill(options: SkillOptions): string { + const description = skillDescription(options); + const problems = frontmatterProblems(options.name, description); + if (problems.length) throw new Error(`Skill "${options.name}" frontmatter:\n${problems.join("\n")}`); + const binary = options.program.name; - const commandIntents = options.intents.filter((intent) => intent.command); - const toolIntents = options.intents.filter((intent) => intent.tool); - const hasRead = options.program.commands.some(function reads(command): boolean { - return command.verb === "read" || command.commands.some(reads); - }); - const exitCodes = Object.entries(EXIT_CODES) - .map(([code, exit]) => `${exit} ${code}`) - .join(", "); + const hasRead = someCommand(options.program, (command) => command.verb === "read"); + const hasMutation = someCommand(options.program, (command) => command.mutating); + const workflows = [...(hasMutation ? [MUTATION_WORKFLOW] : []), ...(options.workflows ?? [])]; return [ - stringifyFrontmatter({ name: options.name, description: options.description }), + stringifyFrontmatter({ name: options.name, description }), `# ${options.title}`, options.intro, "## What the user says, and what to run", - markdownTable( - ["The user says", "Run", "Then"], - commandIntents.map((intent) => [intent.when, `\`${intent.command}\``, intent.then ?? ""]), - ), - toolIntents.length - ? ["## MCP tools", markdownTable(["The user says", "Tool"], toolIntents.map((intent) => [intent.when, `\`${intent.tool}\``]))].join("\n\n") + intentTable(options), + workflows.length + ? ["## Workflows", "Copy the checklist into your reply and tick each step as it is done; the last step is what shows the change landed.", ...workflows.map(renderWorkflow)].join("\n\n") : undefined, "## Rules", [ - SHARED_RULES[0], - SHARED_RULES[1], - ...(hasRead ? [READ_RULE] : []), - ...SHARED_RULES.slice(2), + RULES.narrow, + RULES.output, + ...(hasRead ? [RULES.read] : []), + ...(hasMutation ? [RULES.mutate] : []), + RULES.secrets, + RULES.errors, ...(options.rules ?? []), - `Every command, argument and option is in [${COMMAND_REFERENCE}](${COMMAND_REFERENCE}); \`${binary} commands --json\` is the source of truth.`, + `Every command, argument, option and exit code is in [${COMMAND_REFERENCE}](${COMMAND_REFERENCE}); \`${binary} commands --json\` is the source of truth.`, ] .map((rule) => `- ${rule}`) .join("\n"), options.setup ? `## Setup\n\n${options.setup}` : undefined, - "## Errors", - `Errors are printed once on stderr. Exit codes: 0 success, ${exitCodes}.`, ].filter((section): section is string => Boolean(section)).join("\n\n") + "\n"; } -/** The full command tree as a Markdown table, written to COMMAND_REFERENCE. */ +/** The full command tree and the exit codes, written to COMMAND_REFERENCE. */ export function renderCommandReference(program: ProgramDescription): string { + const exitCodes = Object.entries(EXIT_CODES).map(([code, exit]) => [String(exit), code]); return [ `# ${program.name} commands`, `Generated from \`${program.name} commands --json\` for version ${program.version}.`, @@ -108,41 +141,30 @@ export function renderCommandReference(program: ProgramDescription): string { ["Command", "Description", "Arguments", "Options", "Mutating"], program.commands.flatMap((command) => commandRows(program.name, command)), ), + "## Exit codes", + "With `--json`, a failure is one `{ ok: false, error: { code, message, hint } }` object on stderr.", + markdownTable(["Exit", "Error code"], [["0", "success"], ...exitCodes]), ].join("\n\n") + "\n"; } /** * Every agent packaging file for one CLI repository, keyed by path from the repository root: - * SKILL.md for `npx skills add` with its command reference, the Claude Code plugin and its single-plugin marketplace, and - * agents/openai.yaml when Codex metadata is given. + * the skill folder skills// (SKILL.md, its references and Codex's agents/openai.yaml), which + * `npx skills add` and Claude Code both find, plus the Claude Code plugin and its single-plugin + * marketplace. */ export function agentFiles(options: AgentFilesOptions): Record { const { skill } = options; + const folder = `skills/${skill.name}`; const files: Record = { - "SKILL.md": renderSkill(skill), - [COMMAND_REFERENCE]: renderCommandReference(skill.program), - ".claude-plugin/plugin.json": json({ - name: skill.name, - version: options.version, - description: skill.description, - author: options.author, - homepage: options.repository, - repository: options.repository, - ...(options.license ? { license: options.license } : {}), - ...(options.keywords?.length ? { keywords: options.keywords } : {}), - // The repository root is the plugin root, and its SKILL.md is the plugin's one skill. - skills: "./", - ...(options.mcpServers ? { mcpServers: options.mcpServers } : {}), - }), - ".claude-plugin/marketplace.json": json({ - name: skill.name, - description: skill.description, - owner: options.author, - plugins: [{ name: skill.name, source: "./", description: skill.description }], - }), + [`${folder}/SKILL.md`]: renderSkill(skill), + [`${folder}/${COMMAND_REFERENCE}`]: renderCommandReference(skill.program), }; + for (const [name, content] of Object.entries(options.references ?? {})) { + files[`${folder}/${dirname(COMMAND_REFERENCE)}/${name}`] = content; + } if (options.openai) { - files["agents/openai.yaml"] = stringify( + files[`${folder}/agents/openai.yaml`] = stringify( { interface: { display_name: options.openai.displayName, @@ -153,6 +175,24 @@ export function agentFiles(options: AgentFilesOptions): Record { { lineWidth: 0 }, ); } + // The repository root is the plugin root; Claude Code loads skills/ from it by default. + files[".claude-plugin/plugin.json"] = json({ + name: skill.name, + version: options.version, + description: skill.description, + author: options.author, + homepage: options.repository, + repository: options.repository, + ...(options.license ? { license: options.license } : {}), + ...(options.keywords?.length ? { keywords: options.keywords } : {}), + ...(options.mcpServers ? { mcpServers: options.mcpServers } : {}), + }); + files[".claude-plugin/marketplace.json"] = json({ + name: skill.name, + description: skill.description, + owner: options.author, + plugins: [{ name: skill.name, source: "./", description: skill.description }], + }); return files; } @@ -172,6 +212,47 @@ export function writeAgentFiles(root: string, files: Readonly 64) problems.push(`name must be lowercase words joined by hyphens, at most 64 characters`); + if (/anthropic|claude/.test(name)) problems.push(`name must not contain "anthropic" or "claude"`); + if (description.length > 1024) problems.push(`description is ${description.length} characters; the limit is 1024`); + if (/[<>]/.test(description)) problems.push("description must not contain < or >"); + // Quoted phrases are what the user says, so "my grades" is fine there. + const unquoted = description.replace(/"[^"]*"|“[^”]*”/g, ""); + const person = /\b(?:I|me|my|we|our|you|your)\b/i.exec(unquoted); + if (person) problems.push(`description must be in the third person outside quotes; found "${person[0]}"`); + return problems; +} + +function intentTable(options: SkillOptions): string { + const { intents, mcpServer } = options; + if (!mcpServer && intents.some((intent) => intent.tool)) { + throw new Error(`Skill "${options.name}" has MCP tools but no mcpServer to qualify them with.`); + } + const columns: [string, (intent: Intent) => string][] = [["The user says", (intent) => intent.when]]; + if (intents.some((intent) => intent.command)) columns.push(["CLI", (intent) => (intent.command ? `\`${intent.command}\`` : "")]); + if (intents.some((intent) => intent.tool)) columns.push(["MCP tool", (intent) => (intent.tool ? `\`${mcpServer}:${intent.tool}\`` : "")]); + if (intents.some((intent) => intent.then)) columns.push(["Then", (intent) => intent.then ?? ""]); + return markdownTable( + columns.map(([header]) => header), + intents.map((intent) => columns.map(([, cell]) => cell(intent))), + ); +} + +// A fenced block, so the agent copies it verbatim and ticks it in its reply. +function renderWorkflow(workflow: Workflow): string { + return [`### ${workflow.title}`, ["```text", ...workflow.steps.map((step) => `- [ ] ${step}`), "```"].join("\n")].join("\n\n"); +} + +function someCommand(program: ProgramDescription, test: (command: CommandDescription) => boolean): boolean { + return program.commands.some(function visit(command): boolean { + return test(command) || command.commands.some(visit); + }); +} + function commandRows(binary: string, command: CommandDescription, parents: readonly string[] = []): string[][] { const path = [...parents, command.name]; const row = [ @@ -203,6 +284,11 @@ function escapeCell(value: string): string { return value.replace(/\|/g, "\\|").replace(/\n/g, "
"); } +function sentence(text: string): string { + const trimmed = text.trim(); + return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`; +} + function json(value: unknown): string { return `${JSON.stringify(value, null, 2)}\n`; } diff --git a/test/skills.test.js b/test/skills.test.js index 4ee33d3..1ade6cd 100644 --- a/test/skills.test.js +++ b/test/skills.test.js @@ -12,49 +12,72 @@ function skillOptions() { mutating(program.command("tasks").command("submit ").description("Submit a task.")); return { name: "demo-cli", - description: "Read demo courses: grades, tasks and files.", + description: "Reads demo courses: grades, tasks and files", + useWhen: 'the user names Demo or asks "what\'s due", "my grades"', title: "Demo CLI", intro: "COURSE is a code or id.", intents: [ { when: "my courses", what: "Courses.", command: "demo courses", tool: "list_courses", then: "pass a code as COURSE" }, { when: "a widget", what: "A widget.", tool: "show_widget" }, ], + mcpServer: "demo", + workflows: [{ title: "Submit a task", steps: ["Find the task with `demo tasks list`", "Check it shows as submitted"] }], rules: ["Ask before submitting."], program: commandsJson(program), }; } -test("renderSkill writes frontmatter, intents, tools, rules and exit codes, and links the command tree", () => { +test("renderSkill writes a does-and-when description, one intent table, workflows and rules", () => { const skill = renderSkill(skillOptions()); - assert.match(skill, /^---\nname: demo-cli\ndescription: "Read demo courses: grades, tasks and files."\n---\n\n# Demo CLI\n\nCOURSE is a code or id.\n/); - assert.match(skill, /\| my courses \| `demo courses` \| pass a code as COURSE \|/); - // MCP tools are listed by when to use them; the full description stays with the server. - assert.match(skill, /## MCP tools\n\n\| The user says \| Tool \|\n\| --- \| --- \|\n\| my courses \| `list_courses` \|\n\| a widget \| `show_widget` \|/); - assert.doesNotMatch(skill, /Use when the user says/); - assert.match(skill, /- Ask before submitting\.\n- Every command, argument and option is in \[references\/commands\.md\]\(references\/commands\.md\); `demo commands --json` is the source of truth\./); - assert.doesNotMatch(skill, /`demo tasks submit`/); - assert.match(skill, /Exit codes: 0 success, 2 usage, 3 auth/); - assert.ok(skill.endsWith("\n")); + assert.ok(skill.startsWith([ + "---", + "name: demo-cli", + `description: "Reads demo courses: grades, tasks and files. Use when the user names Demo or asks \\"what's due\\", \\"my grades\\"."`, + "---\n\n# Demo CLI\n\nCOURSE is a code or id.\n", + ].join("\n"))); + // One row per request: the CLI and the qualified MCP tool side by side. + assert.match(skill, /\| The user says \| CLI \| MCP tool \| Then \|\n\| --- \| --- \| --- \| --- \|\n\| my courses \| `demo courses` \| `demo:list_courses` \| pass a code as COURSE \|\n\| a widget \| {2}\| `demo:show_widget` \| {2}\|/); + // The shared mutation workflow comes first, as a copyable checklist ending in a check. + assert.match(skill, /### Change something upstream\n\n```text\n- \[ \] Check the user asked[^\n]*\n(?:- \[ \] [^\n]*\n)*- \[ \] Read the result back[^\n]*\n```\n\n### Submit a task\n\n```text\n- \[ \] Find the task with `demo tasks list`\n- \[ \] Check it shows as submitted\n```/); + assert.match(skill, /- Change something upstream only when the user asked/); + assert.match(skill, /- Ask before submitting\.\n- Every command, argument, option and exit code is in \[references\/commands\.md\]\(references\/commands\.md\); `demo commands --json` is the source of truth\.\n$/); + assert.doesNotMatch(skill, /`demo tasks submit`|Exit codes|Use when the user says/); }); -test("renderSkill states the read rule only for a CLI with a read verb", () => { - assert.doesNotMatch(renderSkill(skillOptions()), /`read` prints Markdown/); - +test("renderSkill leaves out what the CLI cannot do", () => { const program = createProgram({ name: "demo", version: "1.2.3", description: "Demo CLI" }); program.command("tasks").command("read "); - assert.match(renderSkill({ ...skillOptions(), program: commandsJson(program) }), /- `read` prints Markdown and never changes anything upstream\./); + const skill = renderSkill({ ...skillOptions(), intents: [{ when: "a task", what: "A task.", command: "demo tasks read 1" }], workflows: [], program: commandsJson(program) }); + + assert.match(skill, /\| The user says \| CLI \|\n/); + assert.match(skill, /- `read` prints Markdown and changes nothing upstream/); + assert.doesNotMatch(skill, /## Workflows|Change something upstream/); + assert.doesNotMatch(renderSkill(skillOptions()), /`read` prints Markdown/); +}); + +test("renderSkill rejects frontmatter that breaks the Agent Skills limits", () => { + const cases = [ + [{ name: "Demo_CLI" }, /name must be lowercase words joined by hyphens/], + [{ name: "claude-demo" }, /must not contain "anthropic" or "claude"/], + [{ useWhen: "x".repeat(1000) }, /description is \d+ characters; the limit is 1024/], + [{ description: "Reads courses" }, /must not contain < or >/], + [{ useWhen: "you ask about Demo" }, /third person outside quotes; found "you"/], + [{ mcpServer: undefined }, /has MCP tools but no mcpServer/], + ]; + for (const [override, message] of cases) assert.throws(() => renderSkill({ ...skillOptions(), ...override }), message); }); -test("renderCommandReference lists every command with its arguments, options and mutations", () => { +test("renderCommandReference lists every command, its mutations and the exit codes", () => { const reference = renderCommandReference(skillOptions().program); assert.match(reference, /^# demo commands\n\nGenerated from `demo commands --json` for version 1\.2\.3\./); assert.match(reference, /\| `demo courses list` \| List courses\. \| {2}\| {2}\| no \|/); assert.match(reference, /\| `demo tasks submit` \| Submit a task\. \| \| {2}\| yes \|/); + assert.match(reference, /## Exit codes\n\n[^\n]*\n\n\| Exit \| Error code \|\n\| --- \| --- \|\n\| 0 \| success \|\n\| 2 \| usage \|\n\| 3 \| auth \|/); }); -test("agentFiles packages the skill as a Claude Code plugin, a marketplace and Codex metadata", () => { +test("agentFiles puts the skill in its own folder next to the Claude Code plugin and marketplace", () => { const files = agentFiles({ skill: skillOptions(), version: "1.2.3", @@ -63,28 +86,37 @@ test("agentFiles packages the skill as a Claude Code plugin, a marketplace and C license: "MIT", mcpServers: { demo: { command: "npx", args: ["-y", "demo-cli", "mcp"] } }, openai: { displayName: "Demo CLI", shortDescription: "Read demo courses", defaultPrompt: `Use $demo-cli ${"to read courses ".repeat(6)}` }, + references: { "setup.md": "# Setup\n" }, }); - assert.deepEqual(Object.keys(files), ["SKILL.md", COMMAND_REFERENCE, ".claude-plugin/plugin.json", ".claude-plugin/marketplace.json", "agents/openai.yaml"]); + assert.deepEqual(Object.keys(files), [ + "skills/demo-cli/SKILL.md", + `skills/demo-cli/${COMMAND_REFERENCE}`, + "skills/demo-cli/references/setup.md", + "skills/demo-cli/agents/openai.yaml", + ".claude-plugin/plugin.json", + ".claude-plugin/marketplace.json", + ]); + assert.equal(files["skills/demo-cli/references/setup.md"], "# Setup\n"); + // No skills field: Claude Code loads skills/ from the plugin root by default. assert.deepEqual(JSON.parse(files[".claude-plugin/plugin.json"]), { name: "demo-cli", version: "1.2.3", - description: "Read demo courses: grades, tasks and files.", + description: "Reads demo courses: grades, tasks and files", author: { name: "example" }, homepage: "https://github.com/example/demo-cli", repository: "https://github.com/example/demo-cli", license: "MIT", - skills: "./", mcpServers: { demo: { command: "npx", args: ["-y", "demo-cli", "mcp"] } }, }); assert.deepEqual(JSON.parse(files[".claude-plugin/marketplace.json"]), { name: "demo-cli", - description: "Read demo courses: grades, tasks and files.", + description: "Reads demo courses: grades, tasks and files", owner: { name: "example" }, - plugins: [{ name: "demo-cli", source: "./", description: "Read demo courses: grades, tasks and files." }], + plugins: [{ name: "demo-cli", source: "./", description: "Reads demo courses: grades, tasks and files" }], }); assert.equal( - files["agents/openai.yaml"], + files["skills/demo-cli/agents/openai.yaml"], // One line per field, however long the prompt. `interface:\n display_name: Demo CLI\n short_description: Read demo courses\n default_prompt: "Use $demo-cli ${"to read courses ".repeat(6)}"\n`, ); From 3bf51af6717c6e7d530945d8da33a52c50b16d76 Mon Sep 17 00:00:00 2001 From: bunizao <102936102+bunizao@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:06:13 +1100 Subject: [PATCH 6/6] feat(evals): add a Codex trigger eval for sibling skills evals/trigger.mjs puts every sibling skill in one Codex workspace and records which skills// Codex opens first for each request. Runs are killed at their first command and the CLI binaries are failing stubs, so nothing reaches a real account. The 29-request pool covers Moodle, OnTrack and Ed requests plus near misses. --- .gitignore | 1 + README.md | 26 +++++++ evals/trigger-pool.json | 118 ++++++++++++++++++++++++++++++ evals/trigger.mjs | 149 ++++++++++++++++++++++++++++++++++++++ test/trigger-eval.test.js | 36 +++++++++ 5 files changed, 330 insertions(+) create mode 100644 evals/trigger-pool.json create mode 100644 evals/trigger.mjs create mode 100644 test/trigger-eval.test.js diff --git a/.gitignore b/.gitignore index 0d234d4..494f000 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ conformance/node_modules/ *.tgz .DS_Store .playwright-mcp/ +trigger-results.json diff --git a/README.md b/README.md index 930ae28..1715e36 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,32 @@ An MCP server that runs on Cloudflare Workers imports `toolDescription` from a renamed noun or verb, a removed option, or the wrong binary name. Call it from the CLI's tests so the skill and the MCP descriptions cannot drift from the command tree. +### Trigger evals + +`evals/trigger.mjs` checks whether the descriptions route requests to the right skill. It puts +every sibling skill in one Codex workspace, so they compete as they do for a student who has all +three installed. A skill that steals its sibling's requests looks like the sibling undertriggering +when either is tested alone. For each query it records which `skills//` Codex opens first. + +```bash +node evals/trigger.mjs \ + --skill ../moodle-cli/skills/moodle-cli \ + --skill ../ontrack-cli/skills/ontrack-cli \ + --skill ../edstem-cli/skills/edstem-cli \ + --runs 3 +``` + +- The folder name is the skill name. `--pool` defaults to `evals/trigger-pool.json`: 29 + requests, each with the skill it should reach or `none`. `--model` defaults to Codex's configured + model. `--out` keeps every run's pick and first command. +- Each Codex run is killed at its first command, and each skill's binary (`moodle`, `ontrack`, + `edstem`) is a failing stub first on `PATH`, so nothing reaches a real account. `HOME` is a + temporary folder that hides the user's own skills; `CODEX_HOME` keeps the sign-in. +- A run costs about 16k input tokens, so the default 29 × 3 is about 1.4M. Results move by about + one run per query between passes, so compare changes across three runs, not one. +- This measures Codex, not Claude. Claude Code reads the same folder layout and frontmatter, but + its routing can differ. + ## Prompts and progress `createUi` gives every CLI the same interactive surface, drawn with `@clack/prompts` on diff --git a/evals/trigger-pool.json b/evals/trigger-pool.json new file mode 100644 index 0000000..2ce6136 --- /dev/null +++ b/evals/trigger-pool.json @@ -0,0 +1,118 @@ +[ + { + "query": "whats due for me this week across all my units? i keep forgetting the quiz deadlines", + "expected": "moodle-cli" + }, + { + "query": "can you grab the week 8 lecture slides for algorithms and put them in ~/uni/algorithms", + "expected": "moodle-cli" + }, + { + "query": "did my marks for assignment 2 in databases come back yet", + "expected": "moodle-cli" + }, + { + "query": "any new announcements in my units since monday? the lecturer said something about the exam room changing", + "expected": "moodle-cli" + }, + { + "query": "submit ~/Downloads/a2_report.pdf to the assignment 2 dropbox for discrete maths, save it as a draft only, don't finalise", + "expected": "moodle-cli" + }, + { + "query": "i think my tutor messaged me on moodle yesterday, can you check", + "expected": "moodle-cli" + }, + { + "query": "what did i get wrong on the week 5 quiz? i want to see the correct answers and feedback", + "expected": "moodle-cli" + }, + { + "query": "show me the rubric feedback the marker left on my academic writing essay", + "expected": "moodle-cli" + }, + { + "query": "is the intro programming workshop page for week 6 up yet", + "expected": "moodle-cli" + }, + { + "query": "pull my grades for this semester and work out my running average in academic writing", + "expected": "moodle-cli" + }, + { + "query": "which of my ontrack tasks are overdue for intro programming", + "expected": "ontrack-cli" + }, + { + "query": "what does task 3.1P actually ask for, i havent read the sheet", + "expected": "ontrack-cli" + }, + { + "query": "did my tutor reply to my 2.1P submission? they said it needed fixes", + "expected": "ontrack-cli" + }, + { + "query": "mark 4.2C as working on it", + "expected": "ontrack-cli" + }, + { + "query": "upload ~/code/task5/main.py and the screenshot to 5.1P and send it for feedback", + "expected": "ontrack-cli" + }, + { + "query": "how far am i from a distinction in OO design, which credit tasks are left", + "expected": "ontrack-cli" + }, + { + "query": "tell my tutor on 3.3D that i've fixed the loop bug and resubmitted", + "expected": "ontrack-cli" + }, + { + "query": "download all the task resources for data structures so i can work offline on the train", + "expected": "ontrack-cli" + }, + { + "query": "my target grade is HD, which tasks haven't been signed off yet", + "expected": "ontrack-cli" + }, + { + "query": "doubtfire says resubmit on 6.1P, what exactly did the tutor say", + "expected": "ontrack-cli" + }, + { + "query": "has anyone on the ed forum asked about the ambiguity in the assignment 1 spec for algorithms", + "expected": "edstem-cli" + }, + { + "query": "catch me up on what i missed in the databases discussion board this week", + "expected": "edstem-cli" + }, + { + "query": "open the week 4 lesson on ed and tell me which slides i still haven't finished", + "expected": "edstem-cli" + }, + { + "query": "search ed for posts about the segfault in the A2 starter code", + "expected": "edstem-cli" + }, + { + "query": "explain how moodle plugins are written in PHP, I want to build one for my club's website", + "expected": "none" + }, + { + "query": "draft an email to my lecturer asking for a one week extension on assignment 2, I was sick", + "expected": "none" + }, + { + "query": "i got 72, 65 and 80 on three assessments worth 20/30/50 percent, what's my weighted mark", + "expected": "none" + }, + { + "query": "set up a local doubtfire dev instance with docker so I can contribute to the open source project", + "expected": "none" + }, + { + "query": "make me a study timetable for exam period, I have 4 exams starting june 3", + "expected": "none" + } +] diff --git a/evals/trigger.mjs b/evals/trigger.mjs new file mode 100644 index 0000000..f8a5094 --- /dev/null +++ b/evals/trigger.mjs @@ -0,0 +1,149 @@ +#!/usr/bin/env node +// Trigger eval for sibling skills on Codex: for each query, which skill's SKILL.md does Codex +// open first? All skills compete at once, because a skill that steals its sibling's requests +// looks like the sibling undertriggering when each is tested alone. +// +// node evals/trigger.mjs --skill ../moodle-cli/skills/moodle-cli --skill ../ontrack-cli/skills/ontrack-cli \ +// --skill ../edstem-cli/skills/edstem-cli [--pool evals/trigger-pool.json] [--model ] [--runs 3] +// +// Codex is killed as soon as it starts its first command, and each skill's binary is a stub +// first on PATH, so a run reads at most one file and changes nothing. +import { spawn } from "node:child_process"; +import { chmodSync, cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { parseArgs } from "node:util"; +import { fileURLToPath } from "node:url"; + +/** Per query, the most-picked skill against the expected one; per skill, skill-creator's binary view. */ +export function score(results, skills) { + const byQuery = new Map(); + for (const result of results) { + const entry = byQuery.get(result.query) ?? { query: result.query, expected: result.expected, picks: {} }; + entry.picks[result.picked] = (entry.picks[result.picked] ?? 0) + 1; + byQuery.set(result.query, entry); + } + const queries = [...byQuery.values()].map((entry) => { + const [top] = Object.entries(entry.picks).sort((a, b) => b[1] - a[1])[0]; + // Opening an unrelated file, or nothing, is a correct "none". + const ok = top === entry.expected || (entry.expected === "none" && !skills.includes(top)); + return { ...entry, top, ok }; + }); + const perSkill = Object.fromEntries(skills.map((skill) => { + const counts = { shouldTrigger: [0, 0], shouldNot: [0, 0] }; + for (const entry of queries) { + const runs = Object.values(entry.picks).reduce((sum, n) => sum + n, 0); + const fired = (entry.picks[skill] ?? 0) / runs >= 0.5; + const bucket = entry.expected === skill ? counts.shouldTrigger : counts.shouldNot; + bucket[0] += entry.expected === skill ? Number(fired) : Number(!fired); + bucket[1] += 1; + } + return [skill, counts]; + })); + return { queries, perSkill, routed: queries.filter((entry) => entry.ok).length }; +} + +function runOnce({ query, workspace, home, stubs, model, timeoutMs, skills }) { + const args = ["exec", "--json", "--skip-git-repo-check", "-s", "read-only", + "-c", "notify=[]", "-c", "mcp_servers={}", "-c", "plugins={}", "-c", "features.memories=false", "-c", "features.apps=false", + ...(model ? ["-m", model] : []), query]; + const child = spawn("codex", args, { + cwd: workspace, + // A throwaway HOME hides user-level skills; CODEX_HOME keeps the user's sign-in and provider. + env: { ...process.env, HOME: home, CODEX_HOME: process.env.CODEX_HOME ?? join(homedir(), ".codex"), PATH: `${stubs}:${process.env.PATH}` }, + stdio: ["ignore", "pipe", "ignore"], + }); + return new Promise((done) => { + let settled = false; + const finish = (picked, command = "") => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.kill(); + done({ picked, command: command.slice(0, 200) }); + }; + const timer = setTimeout(() => finish("timeout"), timeoutMs); + createInterface({ input: child.stdout }).on("line", (line) => { + let event; + try { + event = JSON.parse(line); + } catch { + return; + } + if (event.type === "item.started" && event.item?.type === "command_execution") { + const command = event.item.command ?? ""; + finish(skills.find((skill) => command.includes(`skills/${skill}/`)) ?? "other", command); + } else if (event.type === "turn.completed") { + finish("none"); + } else if (event.type === "turn.failed") { + finish("error", event.error?.message ?? ""); + } + }); + child.on("close", () => finish("none")); + }); +} + +async function main() { + const { values } = parseArgs({ + options: { + skill: { type: "string", multiple: true }, + pool: { type: "string", default: fileURLToPath(new URL("trigger-pool.json", import.meta.url)) }, + model: { type: "string" }, + runs: { type: "string", default: "3" }, + workers: { type: "string", default: "6" }, + timeout: { type: "string", default: "120" }, + out: { type: "string", default: "trigger-results.json" }, + }, + }); + const skillDirs = (values.skill ?? []).map((dir) => resolve(dir)); + if (!skillDirs.length) throw new Error("Pass each competing skill folder with --skill."); + const skills = skillDirs.map((dir) => basename(dir)); + const pool = JSON.parse(readFileSync(values.pool, "utf8")); + + const root = mkdtempSync(join(tmpdir(), "trigger-eval-")); + const workspace = join(root, "ws"); + const home = join(root, "home"); + const stubs = join(root, "stubs"); + for (const [index, dir] of skillDirs.entries()) cpSync(dir, join(workspace, ".agents", "skills", skills[index]), { recursive: true }); + mkdirSync(home, { recursive: true }); + mkdirSync(stubs, { recursive: true }); + for (const skill of skills) { + const stub = join(stubs, skill.replace(/-cli$/, "")); + writeFileSync(stub, "#!/bin/sh\necho 'stub: trigger eval' >&2\nexit 1\n"); + chmodSync(stub, 0o755); + } + + const jobs = pool.flatMap((item) => Array.from({ length: Number(values.runs) }, () => item)); + const results = []; + let next = 0; + const worker = async () => { + while (next < jobs.length) { + const item = jobs[next++]; + const result = await runOnce({ query: item.query, workspace, home, stubs, model: values.model, timeoutMs: Number(values.timeout) * 1000, skills }); + results.push({ ...item, ...result }); + process.stderr.write(`\r${results.length}/${jobs.length}`); + } + }; + try { + await Promise.all(Array.from({ length: Number(values.workers) }, worker)); + } finally { + rmSync(root, { recursive: true, force: true }); + } + writeFileSync(values.out, `${JSON.stringify(results, null, 2)}\n`); + + const { queries, perSkill, routed } = score(results, skills); + process.stderr.write("\n"); + for (const entry of queries.filter((q) => !q.ok)) console.log(`MISS ${entry.expected.padEnd(12)} ${JSON.stringify(entry.picks)} ${entry.query}`); + console.log(`${routed}/${queries.length} queries routed to the expected skill (majority of runs)`); + for (const [skill, { shouldTrigger, shouldNot }] of Object.entries(perSkill)) { + console.log(`${skill}: should-trigger ${shouldTrigger[0]}/${shouldTrigger[1]}, should-not ${shouldNot[0]}/${shouldNot[1]}`); + } +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/test/trigger-eval.test.js b/test/trigger-eval.test.js new file mode 100644 index 0000000..c637b01 --- /dev/null +++ b/test/trigger-eval.test.js @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +import { score } from "../evals/trigger.mjs"; + +const SKILLS = ["moodle-cli", "ontrack-cli"]; +const run = (query, expected, picked) => ({ query, expected, picked }); + +test("trigger eval scores the majority pick per query and each skill's binary view", () => { + const { queries, perSkill, routed } = score([ + run("what's due", "moodle-cli", "moodle-cli"), + run("what's due", "moodle-cli", "moodle-cli"), + run("what's due", "moodle-cli", "ontrack-cli"), + run("overdue tasks", "ontrack-cli", "moodle-cli"), + run("overdue tasks", "ontrack-cli", "moodle-cli"), + run("overdue tasks", "ontrack-cli", "ontrack-cli"), + run("study timetable", "none", "other"), + run("study timetable", "none", "none"), + ], SKILLS); + + assert.deepEqual(queries.map(({ top, ok }) => [top, ok]), [["moodle-cli", true], ["moodle-cli", false], ["other", true]]); + assert.equal(routed, 2); + assert.deepEqual(perSkill["moodle-cli"], { shouldTrigger: [1, 1], shouldNot: [1, 2] }); + assert.deepEqual(perSkill["ontrack-cli"], { shouldTrigger: [0, 1], shouldNot: [2, 2] }); +}); + +test("trigger pool names no institution's unit codes", () => { + const pool = JSON.parse(readFileSync(new URL("../evals/trigger-pool.json", import.meta.url), "utf8")); + assert.ok(pool.length > 0); + for (const { query, expected } of pool) { + assert.equal(typeof query, "string"); + assert.equal(typeof expected, "string"); + assert.doesNotMatch(query, /\b[A-Z]{3}\d{4}\b/); + } +});