From 90389cd11f50f517a4ec055da6ce18cf60c86b05 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Sun, 8 Nov 2026 12:18:18 +0800 Subject: [PATCH] feat(demo): add an installed skill-library demo (--kind skill_library) Closes #878 agentdescent demo --kind skill_library builds a two-skill */SKILL.md tree and runs it offline from any cwd. The default demo smoke note and README / quickstart / usage now point at that command instead of python -m examples.skill_library_evolution, which only works from a source checkout. The examples module stays as a thin wrapper around the shared fixture. --- README.md | 4 +- agentdescent/cli.py | 60 +++++- agentdescent/demo.py | 269 +++++++++++++++++++++++++- changelog.d/878-demo-skill-library.md | 6 + docs/quickstart-skill.md | 11 +- docs/usage.md | 6 +- examples/skill_library_evolution.py | 171 ++-------------- tests/test_demo.py | 111 ++++++++++- 8 files changed, 454 insertions(+), 184 deletions(-) create mode 100644 changelog.d/878-demo-skill-library.md diff --git a/README.md b/README.md index 7ffb1c7e..96849e62 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ print(result.outcomes()) # why it went that way On 40 HotpotQA items (12 held out) this took held-out exact match from **0.167 to 0.583**, learning *"Respond with only the requested answer, omitting any extra explanation or restatement."* That evolves an **instruction** (`SingleSlot`), not -an Agent Skill folder — for a library run `python -m examples.skill_library_evolution --dry-run`. +an Agent Skill folder — for a library run `agentdescent demo --kind skill_library`. Pass `agent=LLMAgent(claude_code())` (or any [agent or LLM](https://thinkflowlab.github.io/agentdescent/agents/)) instead of `run`/`propose` to evolve against a tool-using CLI agent. @@ -203,7 +203,7 @@ for you to check first. `agent_dir`, `agent_code`, `policy_slot` — or **`plugin`**: the host plugins themselves are evolvable, each rollout in an isolated copy of the host with hooks and permissions frozen and a recursion guard. Flagship library demo: -`python -m examples.skill_library_evolution --dry-run`. +`agentdescent demo --kind skill_library`. [The plugin, step by step →](https://thinkflowlab.github.io/agentdescent/plugin-quickstart/) · [set it up and test it →](https://thinkflowlab.github.io/agentdescent/testing-guide/) diff --git a/agentdescent/cli.py b/agentdescent/cli.py index 0e487bd0..6ed9a02c 100644 --- a/agentdescent/cli.py +++ b/agentdescent/cli.py @@ -681,10 +681,17 @@ def cmd_demo(a: argparse.Namespace) -> int: f"{list(demo_mod.OFFLINE_ALGORITHMS)}. Run it for real with " f"`agentdescent algorithms run {algo} ` " f"(`agentdescent algorithms show {algo}` for its parameters)") + try: + demo_mod.resolve_demo_kind(algo, getattr(a, "kind", None)) + except ValueError as e: + raise SpecError(str(e)) from None root = os.path.abspath(os.path.expanduser(a.dir)) if a.dir else tempfile.mkdtemp( prefix="agentdescent-demo-") os.makedirs(root, exist_ok=True) - spec_dict = demo_mod.build(root, algorithm=algo) + try: + spec_dict = demo_mod.build(root, algorithm=algo, kind=getattr(a, "kind", None)) + except ValueError as e: + raise SpecError(str(e)) from None kind = spec_dict["kind"] as_json = getattr(a, "json", False) @@ -695,6 +702,15 @@ def cmd_demo(a: argparse.Namespace) -> int: print(f" prompt.txt -> {open(prompt_path).read().strip()!r} " f"(wrong: `id` is a row number)") print(" cases.jsonl -> 12 prompts with embedded CSVs and known totals") + elif kind == "skill_library": + csv_rules = os.path.join(root, "csv-total", "references", "rules.md") + date_rules = os.path.join(root, "date-parse", "references", "rules.md") + print(f"A two-skill library (csv-total + date-parse), in {root}") + print(f" csv-total/references/rules.md -> " + f"{open(csv_rules).read().strip()!r} (wrong: `id` is a row number)") + print(f" date-parse/references/rules.md -> " + f"{open(date_rules).read().strip()!r} (wrong: date tasks ask for year)") + print(" cases.jsonl -> 12 tasks (CSV totals + date years)") else: skill = os.path.join(root, demo_mod.DEMO_SKILL) rules = os.path.join(skill, "references", "rules.md") @@ -703,8 +719,9 @@ def cmd_demo(a: argparse.Namespace) -> int: f"{open(rules).read().strip()!r} (wrong: `id` is a row number)") print(" cases.jsonl -> 12 CSVs with known totals") print(" the agent is a local program, so this costs nothing and needs no key") - print(" note: this is a smoke trajectory; for multi-round library edits see") - print(" python -m examples.skill_library_evolution (#847)") + if kind != "skill_library": + print(" note: this is a smoke trajectory; for multi-round library edits see") + print(" agentdescent demo --kind skill_library") if algo: print(f" algorithm: {algo} (`agentdescent algorithms show {algo}`)") print() @@ -737,6 +754,14 @@ def cmd_demo(a: argparse.Namespace) -> int: "agentdescent doctor", "agentdescent plan spec.json", ] + elif kind == "skill_library": + next_steps = [ + f"agentdescent show {rd.run_id}", + f"agentdescent apply {rd.run_id} --dry-run", + "agentdescent init --kind skill_library --data --out spec.json", + "agentdescent doctor", + "agentdescent plan spec.json", + ] else: next_steps = [ f"agentdescent show {rd.run_id}", @@ -799,6 +824,19 @@ def cmd_demo(a: argparse.Namespace) -> int: agentdescent init --kind text --data --out spec.json agentdescent doctor + agentdescent plan spec.json""") + elif kind == "skill_library": + print(f""" +The library on disk is untouched -- an evolved artifact only lands when you say so: + + agentdescent show {rd.run_id} the diff + agentdescent apply {rd.run_id} --dry-run what it would write + +Your own run is the same spec with three things changed -- your library, your +cases.jsonl, and a real agent instead of the offline one: + + agentdescent init --kind skill_library --data --out spec.json + agentdescent doctor # what is missing before a real run agentdescent plan spec.json""") else: print(f""" @@ -1239,10 +1277,10 @@ def write_starter_spec( if spec["kind"] == "skill_library": # #864 / #846: demo is skill_dir smoke; name the library card path. report.append(("next", - "(`agentdescent demo --algorithm archive_selection` " - "is an offline skill_dir smoke; for this library " - "Path A card use --algorithm archive_selection — " - "see docs/quickstart-skill.md)")) + "(`agentdescent demo --kind skill_library` is " + "the offline library demo; `demo --algorithm " + "archive_selection` is a one-skill smoke of the " + "same card — see docs/quickstart-skill.md)")) else: report.append(("next", "(`agentdescent demo` runs a complete example, " "offline, to copy from)")) @@ -4608,10 +4646,14 @@ def build_parser() -> argparse.ArgumentParser: s = sub.add_parser("demo", help="build a complete example and run it, offline") s.add_argument("--dir", help="where to build it (default: a temp directory)") + s.add_argument("--kind", choices=("skill_dir", "text", "skill_library"), + help="demo fixture (default: skill_dir, or text when " + "--algorithm is a text-only card)") s.add_argument("--algorithm", metavar="NAME", help="run the demo with this named algorithm offline " - "(skill_dir: archive_selection; text: gepa, dgm, ace, " - "skillopt, rrsi; evoskill is refused — needs a real model). " + "(skill_dir / skill_library: archive_selection; text: " + "gepa, dgm, ace, skillopt, rrsi; evoskill is refused — " + "needs a real model). " "Currently: " + ", ".join(_demo_offline_algorithms())) s.set_defaults(fn=cmd_demo) diff --git a/agentdescent/demo.py b/agentdescent/demo.py index 657bca80..eeb2d5f1 100644 --- a/agentdescent/demo.py +++ b/agentdescent/demo.py @@ -4,8 +4,8 @@ worker agent on ``PATH``, a provider key, and a dataset with known answers -- so the first thing they see is an error about a missing file. This module is the answer to *"show me it working"*: one command builds a real skill -directory (or a text prompt), a real dataset, and runs the real loop against -a deterministic offline agent. +directory, a multi-file skill library, or a text prompt), a real dataset, and +runs the real loop against a deterministic offline agent. Nothing here is a mock of the framework. Staging, the ledger, the merge and the held-out gate all run exactly as they do against Claude Code. Only the @@ -26,24 +26,35 @@ import random import re import sys +import tempfile from typing import Any, Dict, List, Optional from .agents import Completion, cli_agent +from .skillspec import validate_skill_tree __all__ = [ "DEMO_SKILL", + "LIBRARY_SKILL_NAMES", "OFFLINE_ALGORITHMS", "OFFLINE_SKIP", "OFFLINE_TEXT_ALGORITHMS", "build", + "demo_library_spec", "demo_spec", "demo_text_spec", - "learned_summary", + "learned_summary", + "library_rows", + "library_validator", + "make_library_dir", "offline_agent", + "offline_library_agent", + "offline_library_reflector", "offline_reflector", "offline_text_agent", "offline_text_reflector", + "resolve_demo_kind", "skip_reason", + "write_library", ] #: The demo's skill directory name, and the artifact id the run reports. @@ -75,7 +86,7 @@ OFFLINE_TEXT_ALGORITHMS = ("gepa", "dgm", "ace", "skillopt", "rrsi") #: Wrong on purpose: `id` is a row number. The smoke demo fixes it in one -#: commit; multi-round library evolution lives in examples.skill_library_evolution (#847). +#: commit; multi-round library evolution is ``demo --kind skill_library`` (#878). _RULES_MD = "COLUMN: id\n" _TEXT_SEED = "COLUMN: id\n" @@ -121,7 +132,7 @@ def offline_agent() -> Completion: def offline_reflector(prompt: str) -> str: - """Propose the COLUMN fix as a tree edit (smoke demo; see skill_library for multi-round).""" + """Propose the COLUMN fix as a tree edit (smoke demo; see --kind skill_library).""" match = re.search( r"--- references/rules\.md ---\n(.*?)(?:\n--- |\nTASK|\Z)", prompt, re.S) body = (match.group(1).strip() if match else "") @@ -155,7 +166,7 @@ def offline_text_reflector(prompt: str) -> str: """Text / SkillOpt offline reflector: one-shot COLUMN fix (#845). The skill_dir demo is a one-commit smoke; multi-round teaching is - examples.skill_library_evolution (#847). Text cards stay one-shot. + ``demo --kind skill_library`` (#878). Text cards stay one-shot. """ if '"edits"' in prompt or "Return ONLY a JSON object" in prompt: return json.dumps({ @@ -170,6 +181,216 @@ def offline_text_reflector(prompt: str) -> str: + +LIBRARY_SKILL_NAMES = ("csv-total", "date-parse") + +_CSV_LIBRARY_SKILL = """\ +--- +name: csv-total +description: Total one column of data.csv. Use when asked for a CSV column sum. +--- +# csv-total +STATUS: draft +Read `references/rules.md` for the column; reply with only the sum. +""" + +_DATE_LIBRARY_SKILL = """\ +--- +name: date-parse +description: Extract a calendar field from date.txt. Use when asked for year/month/day. +--- +# date-parse +STATUS: draft +Read `references/rules.md` for the field; parse `date.txt` (YYYY-MM-DD); reply with only that value. +""" + +_LIBRARY_AGENT_SRC = r""" +import csv, os, re, sys +skills = os.path.join('.claude', 'skills') + +def read_rules(name): + p = os.path.join(skills, name, 'references', 'rules.md') + return open(p).read() if os.path.exists(p) else '' + +if os.path.exists('data.csv'): + rules = read_rules('csv-total') + if not rules: + print('MISSING_SKILL'); sys.exit(0) + m = re.search(r'COLUMN:\s*(\w+)', rules) + column = m.group(1) if m else 'id' + rows = list(csv.DictReader(open('data.csv'))) + print(sum(int(r[column]) for r in rows if r.get(column, '').strip().isdigit())) +elif os.path.exists('date.txt'): + rules = read_rules('date-parse') + if not rules: + print('MISSING_SKILL'); sys.exit(0) + m = re.search(r'FIELD:\s*(\w+)', rules) + field = (m.group(1) if m else 'month').lower() + y, mo, d = open('date.txt').read().strip().split('-') + print({'year': str(int(y)), 'month': str(int(mo)), 'day': str(int(d))}.get(field, 'UNKNOWN')) +else: + print('NO_FIXTURE') +""" + + +def write_library(root: str) -> None: + """Write the two-skill fixture (wrong COLUMN / FIELD on purpose) under ``root``.""" + for name, skill, rules in ( + ("csv-total", _CSV_LIBRARY_SKILL, "COLUMN: id\n"), + ("date-parse", _DATE_LIBRARY_SKILL, "FIELD: month\n"), + ): + path = os.path.join(root, name, "references") + os.makedirs(path, exist_ok=True) + with open(os.path.join(root, name, "SKILL.md"), "w", encoding="utf-8") as fh: + fh.write(skill) + with open(os.path.join(path, "rules.md"), "w", encoding="utf-8") as fh: + fh.write(rules) + + +def make_library_dir() -> str: + """Temp directory with the two-skill fixture (used by the source-checkout example).""" + root = tempfile.mkdtemp(prefix="agentdescent-skill-library-") + write_library(root) + return root + + +def library_validator(state): + """Per-skill Agent Skills frontmatter gate, matching the checkout example.""" + names = {p.split("/", 1)[0] for p in state if p and "/" in p} + for name in sorted(names): + skill_state = {} + for p, body in state.items(): + if p == f"{name}/SKILL.md": + skill_state["SKILL.md"] = body + elif p.startswith(f"{name}/"): + skill_state[p[len(name) + 1:]] = body + if "SKILL.md" not in skill_state: + continue + errors = validate_skill_tree(skill_state, root_name=name) + if errors: + return f"skill-library-{name}-{errors[0]}" + return None + + +def library_rows(n: int = 16, seed: int = 0) -> List[Dict[str, Any]]: + """Mixed CSV-total and date-year tasks for the library fixture. + + CSV and date rows are interleaved so a prefix/suffix held-out split + still contains both kinds; a single-skill commit then moves reward. + """ + rng = random.Random(seed) + csv_rows: List[Dict[str, Any]] = [] + for _ in range(n // 2): + amounts = [rng.randint(1, 99) for _ in range(rng.randint(3, 6))] + csv_body = "id,amount,note\n" + "\n".join( + f"{j + 1},{a},row-{j}" for j, a in enumerate(amounts)) + csv_rows.append({ + "kind": "csv", + "prompt": "What is the total of the requested column in data.csv?", + "gold": str(sum(amounts)), + "fixtures": {"data.csv": csv_body}, + }) + rng2 = random.Random(seed + 17) + date_rows: List[Dict[str, Any]] = [] + for _ in range(n - n // 2): + y, m, d = rng2.randint(1990, 2025), rng2.randint(1, 12), rng2.randint(1, 28) + date_rows.append({ + "kind": "date", + "prompt": "What year is recorded in date.txt?", + "gold": str(y), + "fixtures": {"date.txt": f"{y:04d}-{m:02d}-{d:02d}\n"}, + }) + rows: List[Dict[str, Any]] = [] + for i in range(max(len(csv_rows), len(date_rows))): + if i < len(csv_rows): + rows.append(csv_rows[i]) + if i < len(date_rows): + rows.append(date_rows[i]) + return rows + + +def offline_library_agent() -> Completion: + """Workspace agent that reads ``*/SKILL.md`` off the library layout.""" + return cli_agent([sys.executable, "-c", _LIBRARY_AGENT_SRC]) + + +def _library_skill_is_draft(prompt: str, name: str) -> bool: + marker = f"--- {name}/SKILL.md ---" + if marker not in prompt: + return False + section = prompt.split(marker, 1)[1].split("\n--- ", 1)[0] + return "STATUS: draft" in section + + +def _library_skill_shown(prompt: str, name: str) -> bool: + return f"--- {name}/SKILL.md ---" in prompt + + +def offline_library_reflector(prompt: str) -> str: + """Pair each SKILL.md STATUS flip with that skill's rule fix. + + Path A inlines ``*/SKILL.md`` but not nested ``references/rules.md``, so + ``COLUMN: amount`` never appears in the prompt. Using draft-vs-ready as + the per-skill signal keeps each 2-file commit reward-improving (#878). + """ + edits = [] + csv_draft = _library_skill_is_draft(prompt, "csv-total") + date_draft = _library_skill_is_draft(prompt, "date-parse") + if csv_draft or (not _library_skill_shown(prompt, "csv-total") + and "COLUMN: amount" not in prompt): + if csv_draft: + edits.append({ + "path": "csv-total/SKILL.md", + "content": _CSV_LIBRARY_SKILL.replace( + "STATUS: draft", "STATUS: ready"), + }) + edits.append({"path": "csv-total/references/rules.md", + "content": "COLUMN: amount\n"}) + if date_draft or (not _library_skill_shown(prompt, "date-parse") + and "FIELD: year" not in prompt): + if date_draft: + edits.append({ + "path": "date-parse/SKILL.md", + "content": _DATE_LIBRARY_SKILL.replace( + "STATUS: draft", "STATUS: ready"), + }) + edits.append({"path": "date-parse/references/rules.md", + "content": "FIELD: year\n"}) + if not edits: + edits.append({"path": "csv-total/references/rules.md", + "content": "COLUMN: amount\n"}) + return "" + json.dumps({ + "rationale": "mark skills ready; sum amount; date tasks ask for year", + "edits": edits[:2], + }) + "" + + +def resolve_demo_kind(algorithm: Optional[str] = None, + kind: Optional[str] = None) -> str: + """Pick the demo fixture from an explicit ``--kind`` and/or ``--algorithm``.""" + if kind is None: + return "text" if algorithm in OFFLINE_TEXT_ALGORITHMS else "skill_dir" + if kind not in ("text", "skill_dir", "skill_library"): + raise ValueError( + f"demo --kind must be one of ['skill_dir', 'text', 'skill_library'], " + f"got {kind!r}") + if kind == "skill_library" and algorithm in OFFLINE_TEXT_ALGORITHMS: + raise ValueError( + f"demo --kind skill_library does not run the text-only card " + f"{algorithm!r}; use archive_selection or omit --algorithm") + if kind == "skill_dir" and algorithm in OFFLINE_TEXT_ALGORITHMS: + raise ValueError( + f"demo --kind skill_dir does not run the text-only card " + f"{algorithm!r}; omit --kind (the demo picks the text fixture) " + f"or pass --kind text") + if kind == "text" and algorithm and algorithm not in OFFLINE_TEXT_ALGORITHMS: + raise ValueError( + f"demo --kind text: {algorithm!r} runs on a skill_dir / " + f"skill_library fixture; omit --kind or pass --kind skill_dir " + f"or --kind skill_library") + return kind + + def _rows(n: int = 12, seed: int = 0) -> List[Dict[str, Any]]: """One CSV per task, staged into the workspace beside the skill.""" rng = random.Random(seed) @@ -230,15 +451,37 @@ def demo_text_spec(root: str) -> Dict[str, Any]: } -def build(root: str, algorithm: Optional[str] = None) -> Dict[str, Any]: - """Write the demo's skill/prompt, dataset and spec under ``root``.""" - use_text = algorithm in OFFLINE_TEXT_ALGORITHMS - if use_text: +def demo_library_spec(root: str) -> Dict[str, Any]: + """The skill_library spec for ``demo --kind skill_library`` (#878).""" + return { + "version": 1, + "kind": "skill_library", + "target": root, + "data": {"path": os.path.join(root, "cases.jsonl"), + "prompt": "prompt", "gold": "gold"}, + "score": "exact", + "agent": {"ref": "agentdescent.demo:offline_library_agent"}, + "reflect": {"ref": "agentdescent.demo:offline_library_reflector", "call": False}, + "prompt_template": "{prompt}", + "evolve": {"rounds": 4, "n_workers": 2, "seed": 0}, + } + + +def build(root: str, algorithm: Optional[str] = None, + kind: Optional[str] = None) -> Dict[str, Any]: + """Write the demo's skill/library/prompt, dataset and spec under ``root``.""" + resolved = resolve_demo_kind(algorithm, kind) + if resolved == "text": with open(os.path.join(root, "prompt.txt"), "w", encoding="utf-8") as fh: fh.write(_TEXT_SEED) with open(os.path.join(root, "cases.jsonl"), "w", encoding="utf-8") as fh: fh.write("\n".join(json.dumps(r) for r in _text_rows()) + "\n") spec = demo_text_spec(root) + elif resolved == "skill_library": + write_library(root) + with open(os.path.join(root, "cases.jsonl"), "w", encoding="utf-8") as fh: + fh.write("\n".join(json.dumps(r) for r in library_rows(n=12)) + "\n") + spec = demo_library_spec(root) else: skill = os.path.join(root, DEMO_SKILL) os.makedirs(os.path.join(skill, "references"), exist_ok=True) @@ -269,4 +512,10 @@ def learned_summary(state: Dict[str, Any], *, kind: str) -> str: if bullets: return f"playbook -> {bullets[-1].strip()!r}" return f"state -> {state!r}" + if kind == "skill_library": + parts = [] + for name in LIBRARY_SKILL_NAMES: + rules = state.get(f"{name}/references/rules.md", "").strip() + parts.append(f"{name}/rules.md -> {rules!r}") + return "; ".join(parts) return f"rules.md -> {state.get('references/rules.md', '').strip()!r}" diff --git a/changelog.d/878-demo-skill-library.md b/changelog.d/878-demo-skill-library.md new file mode 100644 index 00000000..14215905 --- /dev/null +++ b/changelog.d/878-demo-skill-library.md @@ -0,0 +1,6 @@ +Added +`agentdescent demo --kind skill_library` runs the offline multi-file +`*/SKILL.md` library demo from an install (no source checkout, no key). +The default `demo` smoke note and README / quickstart-skill / usage now +point at that command; `python -m examples.skill_library_evolution` stays +as the source-checkout CLI (`--dry-run`, `--agent`, `--install-to`) (#878). diff --git a/docs/quickstart-skill.md b/docs/quickstart-skill.md index 42317711..50c46ff7 100644 --- a/docs/quickstart-skill.md +++ b/docs/quickstart-skill.md @@ -7,7 +7,7 @@ description: Instruction / one Agent Skill / skill library — pick the artifact Pick the artifact: **instruction** (`SingleSlot` / `AppendRules`, this page / [`skill_evolution`](skill-evolution.md)), **one Agent Skill** ([`skill_dir_evolution`](quickstart-directory.md)), or a **skill library** -(`python -m examples.skill_library_evolution --dry-run`, Path A `--kind skill_library`, +(`agentdescent demo --kind skill_library`, Path A `init --kind skill_library`, [`evoskill`](algo-evoskill.md) or [`archive_selection`](algorithms.md)). A `SingleSlot` result is not an Agent Skill folder. @@ -24,9 +24,10 @@ agentdescent plan spec.json # agentdescent evolve spec.json ``` -Offline smoke of the same *card* (skill_dir fixture, not a library tree): -`agentdescent demo --algorithm archive_selection`. Multi-round library teaching -still lives in `python -m examples.skill_library_evolution --dry-run`. +Offline library teaching (two `*/SKILL.md`, no key): +`agentdescent demo --kind skill_library`. The same *card* on the one-skill +smoke is `agentdescent demo --algorithm archive_selection`; both work with +`--algorithm archive_selection`. --- @@ -140,7 +141,7 @@ custom optimizer (`aggregator_factory=`), a multi-step agent as `run=` — is ## Next -* **Want a real Agent Skill / library?** Path A: `init --kind skill_library --algorithm archive_selection` (above), or `python -m examples.skill_library_evolution --dry-run` / [`evoskill`](algo-evoskill.md); then `agentdescent apply --install-to ~/.claude/skills`, or [evolve a directory](quickstart-directory.md). +* **Want a real Agent Skill / library?** Offline: `agentdescent demo --kind skill_library`. Path A: `init --kind skill_library --algorithm archive_selection` (above), or [`evoskill`](algo-evoskill.md); then `agentdescent apply --install-to ~/.claude/skills`, or [evolve a directory](quickstart-directory.md). * **Want the knobs?** [The `evolve` method](evolution.md). * **Want to know why it works?** [Concepts](concepts.md), then [the aggregator](aggregator.md). diff --git a/docs/usage.md b/docs/usage.md index e7711d3b..6cbc0dff 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -54,8 +54,10 @@ python -m examples.rq2_staleness ### Evolving a directory (a skill folder a real agent reads) ```bash -python -m examples.skill_dir_evolution -python -m examples.skill_library_evolution --dry-run +agentdescent demo +agentdescent demo --kind skill_library +python -m examples.skill_dir_evolution # source checkout only +python -m examples.skill_library_evolution --dry-run # source checkout only ``` Materialises a candidate skill directory into a throwaway workspace at diff --git a/examples/skill_library_evolution.py b/examples/skill_library_evolution.py index 4f2f698d..19f59b03 100644 --- a/examples/skill_library_evolution.py +++ b/examples/skill_library_evolution.py @@ -2,12 +2,12 @@ Evolve a multi-file Agent Skill library an agent reads off disk. Flagship skill path: ≥2 ``*/SKILL.md`` with YAML frontmatter, ``layout="skill_library"``. -Offline agent fails if it does not open those files. Contrast -``examples.skill_evolution`` (AppendRules playbook), ``examples.skill_dir_evolution`` -(one skill), ``examples.evoskill`` (paper port). Seed is wrong: ``COLUMN: id`` / -``FIELD: month``. +The installed command is ``agentdescent demo --kind skill_library`` (any cwd, no +key). This module is the source-checkout CLI: ``--dry-run``, ``--agent +claude-code`` / ``codex``, and ``--install-to``. - python -m examples.skill_library_evolution --dry-run + agentdescent demo --kind skill_library + python -m examples.skill_library_evolution --dry-run # source checkout python -m examples.skill_library_evolution python -m examples.skill_library_evolution --install-to ~/.claude/skills """ @@ -15,99 +15,27 @@ from __future__ import annotations import argparse -import json -import os -import random import shutil -import sys -import tempfile -from typing import Dict, List, Mapping, Optional +from typing import Dict from agentdescent import ( AggregatorConfig, FileTree, Policies, evolve, load_tree, scorer, tree_reflector, tree_runner, ) -from agentdescent.agents import claude_code, cli_agent, codex, openai_compatible +from agentdescent.agents import claude_code, codex, openai_compatible +from agentdescent.demo import ( + LIBRARY_SKILL_NAMES as SKILL_NAMES, + library_rows as make_rows, + library_validator, + make_library_dir, + offline_library_agent as offline_agent, + offline_library_reflector as _offline_library_reflector, +) from agentdescent.governance import SKILL_BLAST_RADIUS -from agentdescent.skillspec import validate_skill_tree - -SKILL_NAMES = ("csv-total", "date-parse") - -_CSV_SKILL = """\ ---- -name: csv-total -description: Total one column of data.csv. Use when asked for a CSV column sum. ---- -# csv-total -STATUS: draft -Read `references/rules.md` for the column; reply with only the sum. -""" - -_DATE_SKILL = """\ ---- -name: date-parse -description: Extract a calendar field from date.txt. Use when asked for year/month/day. ---- -# date-parse -STATUS: draft -Read `references/rules.md` for the field; parse `date.txt` (YYYY-MM-DD); reply with only that value. -""" -def make_library_dir() -> str: - root = tempfile.mkdtemp(prefix="agentdescent-skill-library-") - for name, skill, rules in ( - ("csv-total", _CSV_SKILL, "COLUMN: id\n"), - ("date-parse", _DATE_SKILL, "FIELD: month\n"), - ): - path = os.path.join(root, name, "references") - os.makedirs(path) - open(os.path.join(root, name, "SKILL.md"), "w").write(skill) - open(os.path.join(path, "rules.md"), "w").write(rules) - return root - - -def library_validator(state: Mapping[str, Optional[str]]) -> Optional[str]: - names = {p.split("/", 1)[0] for p in state if p and "/" in p} - for name in sorted(names): - skill_state = {} - for p, body in state.items(): - if p == f"{name}/SKILL.md": - skill_state["SKILL.md"] = body - elif p.startswith(f"{name}/"): - skill_state[p[len(name) + 1:]] = body - if "SKILL.md" not in skill_state: - continue - errors = validate_skill_tree(skill_state, root_name=name) - if errors: - return f"skill-library-{name}-{errors[0]}" - return None - - -def make_rows(n: int = 16, seed: int = 0) -> List[dict]: - rng = random.Random(seed) - rows = [] - for i in range(n // 2): - amounts = [rng.randint(1, 99) for _ in range(rng.randint(3, 6))] - csv = "id,amount,note\n" + "\n".join( - f"{j + 1},{a},row-{j}" for j, a in enumerate(amounts)) - rows.append({ - "kind": "csv", - "prompt": "What is the total of the requested column in data.csv?", - "gold": str(sum(amounts)), - "fixtures": {"data.csv": csv}, - }) - rng2 = random.Random(seed + 17) - for i in range(n - n // 2): - y, m, d = rng2.randint(1990, 2025), rng2.randint(1, 12), rng2.randint(1, 28) - rows.append({ - "kind": "date", - "prompt": "What year is recorded in date.txt?", - "gold": str(y), - "fixtures": {"date.txt": f"{y:04d}-{m:02d}-{d:02d}\n"}, - }) - rng.shuffle(rows) - return rows +def offline_reflector(): + return _offline_library_reflector def to_tasks(rows): @@ -117,71 +45,6 @@ def to_tasks(rows): for i, r in enumerate(rows)] -_OFFLINE_AGENT = r""" -import csv, os, re, sys -skills = os.path.join('.claude', 'skills') - -def read_rules(name): - p = os.path.join(skills, name, 'references', 'rules.md') - return open(p).read() if os.path.exists(p) else '' - -if os.path.exists('data.csv'): - rules = read_rules('csv-total') - if not rules: - print('MISSING_SKILL'); sys.exit(0) - m = re.search(r'COLUMN:\s*(\w+)', rules) - column = m.group(1) if m else 'id' - rows = list(csv.DictReader(open('data.csv'))) - print(sum(int(r[column]) for r in rows if r.get(column, '').strip().isdigit())) -elif os.path.exists('date.txt'): - rules = read_rules('date-parse') - if not rules: - print('MISSING_SKILL'); sys.exit(0) - m = re.search(r'FIELD:\s*(\w+)', rules) - field = (m.group(1) if m else 'month').lower() - y, mo, d = open('date.txt').read().strip().split('-') - print({'year': str(int(y)), 'month': str(int(mo)), 'day': str(int(d))}.get(field, 'UNKNOWN')) -else: - print('NO_FIXTURE') -""" - - -def offline_agent(): - return cli_agent([sys.executable, "-c", _OFFLINE_AGENT]) - - -def offline_reflector(): - def complete(prompt: str) -> str: - edits = [] - # Per-skill STATUS flip from the inlined --- /SKILL.md --- sections (#847). - for name in SKILL_NAMES: - marker = f"--- {name}/SKILL.md ---" - if marker not in prompt: - continue - section = prompt.split(marker, 1)[1].split("\n--- ", 1)[0] - if "STATUS: draft" not in section: - continue - body = _CSV_SKILL if name == "csv-total" else _DATE_SKILL - edits.append({ - "path": f"{name}/SKILL.md", - "content": body.replace("STATUS: draft", "STATUS: ready"), - }) - if "COLUMN: amount" not in prompt: - edits.append({"path": "csv-total/references/rules.md", - "content": "COLUMN: amount\n"}) - if "FIELD: year" not in prompt: - edits.append({"path": "date-parse/references/rules.md", - "content": "FIELD: year\n"}) - if not edits: - edits.append({"path": "csv-total/references/rules.md", - "content": "COLUMN: amount\n"}) - return "" + json.dumps({ - "rationale": "mark skills ready; sum amount; date tasks ask for year", - "edits": edits[:2], - }) + "" - return complete - - def estimate_calls(rounds: int, workers: int, tasks: int) -> Dict[str, int]: held = max(1, int(tasks * 0.3)) return { diff --git a/tests/test_demo.py b/tests/test_demo.py index feb556ac..14c2c9a1 100644 --- a/tests/test_demo.py +++ b/tests/test_demo.py @@ -259,7 +259,7 @@ def test_skip_reason_only_for_listed_skips(): def test_demo_labels_itself_smoke_and_points_at_library_example(tmp_path, monkeypatch): - """#847: demo stays a one-commit smoke; multi-round lives in skill_library.""" + """#878: default demo is a one-commit smoke; library path is the installed CLI.""" monkeypatch.setenv("AGENTDESCENT_HOME", str(tmp_path / "home")) buf = io.StringIO() with redirect_stdout(buf): @@ -268,4 +268,111 @@ def test_demo_labels_itself_smoke_and_points_at_library_example(tmp_path, monkey out = buf.getvalue() assert code == 0, out assert "smoke trajectory" in out - assert "skill_library_evolution" in out + assert "demo --kind skill_library" in out + assert "skill_library_evolution" not in out + + +def test_build_library_writes_two_valid_skills(tmp_path): + spec = demo.build(str(tmp_path), kind="skill_library") + assert spec["kind"] == "skill_library" + assert spec["target"] == str(tmp_path) + assert spec["agent"]["ref"].startswith("agentdescent.") + assert spec["reflect"]["ref"].startswith("agentdescent.") + assert (tmp_path / "csv-total" / "SKILL.md").exists() + assert (tmp_path / "date-parse" / "SKILL.md").exists() + assert (tmp_path / "csv-total" / "references" / "rules.md").read_text().strip() == "COLUMN: id" + assert (tmp_path / "date-parse" / "references" / "rules.md").read_text().strip() == "FIELD: month" + rows = [json.loads(l) for l in (tmp_path / "cases.jsonl").read_text().splitlines()] + assert len(rows) == 12 + assert {r["kind"] for r in rows} == {"csv", "date"} + + +def test_demo_kind_skill_library_runs_and_edits_a_skill_body(tmp_path, monkeypatch): + """#878: installed CLI evolves a ≥2-skill library and commits a SKILL.md edit.""" + monkeypatch.setenv("AGENTDESCENT_HOME", str(tmp_path / "home")) + buf = io.StringIO() + with redirect_stdout(buf): + code = cli.main(["--store", str(tmp_path / "runs"), "demo", + "--kind", "skill_library", + "--dir", str(tmp_path / "work")]) + out = buf.getvalue() + assert code == 0, out + assert "held-out reward: 0.000 -> 1.000" in out, out + assert "COLUMN: amount" in out and "FIELD: year" in out, out + assert "init --kind skill_library" in out + assert "python -m examples" not in out + rounds = [line for line in out.splitlines() if line.strip().startswith("round ")] + assert len(rounds) >= 2, out + runs = runstore.list_runs(store=str(tmp_path / "runs")) + assert len(runs) == 1 and runs[0].state == "done" + # disk stays at the seed; the body edit lives in the run result + assert "STATUS: draft" in (tmp_path / "work" / "csv-total" / "SKILL.md").read_text() + assert "csv-total/rules.md" in out and "date-parse/rules.md" in out + result = runstore.get(runs[0].run_id, store=str(tmp_path / "runs")).result() + assert result is not None + assert "STATUS: ready" in result.state["csv-total/SKILL.md"] + assert "COLUMN: amount" in result.state["csv-total/references/rules.md"] + assert "FIELD: year" in result.state["date-parse/references/rules.md"] + + +def test_demo_kind_skill_library_json(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTDESCENT_HOME", str(tmp_path / "home")) + buf = io.StringIO() + with redirect_stdout(buf): + code = cli.main(["--store", str(tmp_path / "runs"), "demo", + "--kind", "skill_library", "--json", + "--dir", str(tmp_path / "work")]) + out = buf.getvalue() + assert code == 0, out + payload = json.loads(out) + assert payload["ok"] is True + assert payload["kind"] == "skill_library" + assert payload["initial_reward"] == 0.0 + assert payload["final_reward"] == 1.0 + assert payload["outcomes"].get("committed", 0) >= 1 + assert "COLUMN: amount" in payload["learned"] + assert "FIELD: year" in payload["learned"] + assert any("skill_library" in step for step in payload["next"]) + assert "A two-skill library" not in out + + +def test_demo_kind_skill_library_with_archive_selection(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTDESCENT_HOME", str(tmp_path / "home")) + buf = io.StringIO() + with redirect_stdout(buf): + code = cli.main(["--store", str(tmp_path / "runs"), "demo", + "--kind", "skill_library", + "--algorithm", "archive_selection", + "--dir", str(tmp_path / "work")]) + out = buf.getvalue() + assert code == 0, out + assert "algorithm: archive_selection" in out + assert "held-out reward: 0.000 -> 1.000" in out, out + spec = json.loads((tmp_path / "work" / "spec.json").read_text()) + assert spec["kind"] == "skill_library" and spec["algorithm"] == "archive_selection" + + +def test_demo_kind_skill_library_refuses_text_only_card(tmp_path, capsys): + code = cli.main(["--store", str(tmp_path / "runs"), "demo", + "--kind", "skill_library", "--algorithm", "gepa", + "--dir", str(tmp_path / "work")]) + err = capsys.readouterr().err + assert code == 2 + assert "skill_library" in err and "gepa" in err + assert not (tmp_path / "work").exists() + + +def test_docs_lead_with_installed_library_demo(): + """#878: README / quickstart / usage name the wheel command first.""" + files = [ + os.path.join(ROOT, "README.md"), + os.path.join(ROOT, "docs", "quickstart-skill.md"), + os.path.join(ROOT, "docs", "usage.md"), + ] + for path in files: + text = open(path, encoding="utf-8").read() + assert "demo --kind skill_library" in text, path + for line in text.splitlines(): + if "python -m examples.skill_library_evolution" in line: + lowered = line.lower() + assert "checkout" in lowered, f"{path}: {line!r} should say it needs a source checkout"