From 927c149c05fbf2109a69b3ceeb394751bc9cde7e Mon Sep 17 00:00:00 2001 From: growlf Date: Sun, 25 Oct 2026 13:32:53 -0700 Subject: [PATCH 1/4] test(skills): guard the research skill and /research command (#452) Fails while the skill does not exist. The guards exist because both halves degrade silently: a command naming a deleted skill still runs and answers from memory, and a skill missing a frontmatter field is simply never advertised. --- tests/test_research_skill.py | 96 ++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/test_research_skill.py diff --git a/tests/test_research_skill.py b/tests/test_research_skill.py new file mode 100644 index 0000000..d5f116c --- /dev/null +++ b/tests/test_research_skill.py @@ -0,0 +1,96 @@ +"""The research skill and its /research command must stay real and discoverable. + +Two ways this pair degrades silently, and both are worse than having no skill: + +- the command survives while the skill it names is renamed or deleted, so the + agent is told to load a procedure that does not exist and falls back to + answering from memory — the exact defect the skill exists to prevent; +- the skill's frontmatter drifts off the canonical four fields, so discovery + stops advertising it and nobody notices, because a missing skill looks + identical to a skill that was never needed. + +The definition-of-done guard already covers AGENTS.md registration for new +files; it only fires on the commit that adds them, so registration is re-checked +here too. +""" + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SKILL = ROOT / ".opencode" / "skills" / "research" / "SKILL.md" +COMMAND = ROOT / ".opencode" / "command" / "research.md" +AGENTS = ROOT / "AGENTS.md" + +CANONICAL_FIELDS = ["name", "description", "mode", "triggers"] +MAX_CONTENT_LINES = 60 + + +def _split_frontmatter(path): + """(field names, frontmatter text, body) for a Markdown file.""" + text = path.read_text() + lines = text.splitlines() + assert lines and lines[0].strip() == "---", f"{path}: no frontmatter" + try: + end = lines.index("---", 1) + except ValueError: # pragma: no cover - malformed file + raise AssertionError(f"{path}: frontmatter is never closed") + fields = [ + line.split(":", 1)[0].strip() + for line in lines[1:end] + if line and not line.startswith((" ", "\t")) and ":" in line + ] + return fields, "\n".join(lines[1:end]), "\n".join(lines[end + 1:]) + + +def test_skill_exists_with_the_canonical_four_fields(): + assert SKILL.is_file(), "the research skill is missing" + fields, frontmatter, _ = _split_frontmatter(SKILL) + assert fields == CANONICAL_FIELDS, ( + "skills carry exactly name/description/mode/triggers — a fifth field or " + f"a reordering breaks discovery; found {fields}" + ) + assert re.search(r"^name:\s*research\s*$", frontmatter, re.M) + assert re.search(r"^mode:\s*skill\s*$", frontmatter, re.M) + + +def test_skill_is_short_enough_to_load_every_time(): + """A skill costs tokens on every matching message; a long one is skipped.""" + _, _, body = _split_frontmatter(SKILL) + content_lines = [line for line in body.splitlines() if line.strip()] + assert len(content_lines) <= MAX_CONTENT_LINES, ( + f"{len(content_lines)} content lines exceeds the {MAX_CONTENT_LINES} " + "ceiling — move procedure detail to docs/ and reference it" + ) + + +def test_command_loads_the_skill_and_takes_arguments(): + assert COMMAND.is_file(), "the /research command is missing" + _, frontmatter, body = _split_frontmatter(COMMAND) + assert re.search(r"^description:\s*\S", frontmatter, re.M), ( + "the command needs a description or it reads as an unlabelled prompt" + ) + assert "`research`" in body, ( + "the command must name the `research` skill it loads, so renaming the " + "skill fails here instead of silently degrading to an unsourced answer" + ) + assert "$ARGUMENTS" in body, "the command must pass the operator's topic through" + + +def test_skill_is_registered_in_agents_md(): + assert "`research`" in AGENTS.read_text(), ( + "AGENTS.md is the discovery surface agents read first; an unregistered " + "skill is one nobody loads — bin/definition-of-done-guard.py rejects " + "this on the commit that adds it" + ) + + +def test_skill_carries_the_no_guess_rule(): + """The one sentence that makes the pair worth having.""" + _, _, body = _split_frontmatter(SKILL) + lowered = body.lower() + assert "guess" in lowered, "the no-guess rule is the point of the skill" + assert "unverified" in lowered, ( + "the skill must make 'unverified' an acceptable answer, or the agent " + "has no honest output for a question it cannot source" + ) From 6c9bf034f8e6c91cc5587cb830d7708b01b459a4 Mon Sep 17 00:00:00 2001 From: growlf Date: Sun, 25 Oct 2026 13:36:14 -0700 Subject: [PATCH 2/4] feat(skills): add research skill and /research command (#452) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers a factual question by walking an evidence ladder — repo, then live system state, then vendor docs, then secondary — and reports an evidence table with confidence and an explicit unverified list. Memory becomes a lead instead of a finding: no claim reaches the operator unless a source was read this session. websearch stays the tool doc for the web rung; this is the procedure that says when to reach for it. --- .claude/skills/research | 1 + .opencode/command/research.md | 25 +++++++++++ .opencode/skills/research/SKILL.md | 71 ++++++++++++++++++++++++++++++ AGENTS.md | 2 +- 4 files changed, 98 insertions(+), 1 deletion(-) create mode 120000 .claude/skills/research create mode 100644 .opencode/command/research.md create mode 100644 .opencode/skills/research/SKILL.md diff --git a/.claude/skills/research b/.claude/skills/research new file mode 120000 index 0000000..8e17e3e --- /dev/null +++ b/.claude/skills/research @@ -0,0 +1 @@ +../../.opencode/skills/research \ No newline at end of file diff --git a/.opencode/command/research.md b/.opencode/command/research.md new file mode 100644 index 0000000..0fbb8c8 --- /dev/null +++ b/.opencode/command/research.md @@ -0,0 +1,25 @@ +--- +description: Research a topic from primary sources — per-claim citations, confidence, and explicit unknowns instead of guesses +--- + +# /research — evidence-first answer + +## Description + +Research a topic to a sourced answer. Every load-bearing claim carries a source read in this session; confidence and unverified gaps are stated plainly. + +## Triggers + +`/research`, `research `, `find out whether`, `verify that`, `is it true that`, `look up` + +## Procedure + +Load the `research` skill and run it against: $ARGUMENTS + +1. Restate the question and the decision it feeds. +2. Walk the evidence ladder — repo/code, then live tools, then vendor docs, then + secondary sources. +3. Cross-check load-bearing claims, and date anything that drifts. +4. Return the answer with the evidence table, confidence, and the unverified list. + +If `$ARGUMENTS` is empty, ask what to research. Never answer from memory. diff --git a/.opencode/skills/research/SKILL.md b/.opencode/skills/research/SKILL.md new file mode 100644 index 0000000..37e720e --- /dev/null +++ b/.opencode/skills/research/SKILL.md @@ -0,0 +1,71 @@ +--- +name: research +description: Evidence-first procedure for factual questions — route through repo, live tools, and web, then report sources and confidence instead of guesses +mode: skill +triggers: research,verify claim,fact check,look up,find out,investigate,evidence,is it true +--- + +# research + +> Load this skill when: asked to research, verify, or answer a factual question +> whose answer is not already in context — and always when a wrong answer would +> cost something. + +## The rule + +**No claim reaches the operator unless a source was read in this session.** + +Memory is a lead, never a finding. It tells you where to look; it never tells +you what is true. "Unverified" and "I don't know" are complete answers. A +confident guess is not, because the operator cannot tell it apart from a sourced +one. + +## Evidence ladder — cheapest authoritative rung first + +| Rung | Source | Reach for | +|---|---|---| +| 1 | This repo and its code | `grep`, `read`, run the tool or test | +| 2 | Live system state | `proxmox_*`, `technitium_*`, `mikromcp` via `bin/mcp-call.py`, `gh`, `vikunja` | +| 3 | Vendor docs, API reference, upstream source | `websearch` skill → `donsetch` | +| 4 | Blog, news, forum, another model's output | corroboration only — never load-bearing | + +Stop at the first rung that answers the question. Rung 1 is cheaper than rung 3 +every time: look before you search. + +## Procedure + +1. **Restate** the question and the decision it feeds, in one line. If the + answer changes nothing, say so instead of researching it. +2. **Decompose** into the claims the answer depends on — you will source each. +3. **Walk the ladder** per claim, loading the domain skill that governs how that + source is read (`django-expert`, `proxmox-cli`, `routeros`, `gh`, …). +4. **Cross-check** load-bearing claims: two independent sources, or one primary + source plus a reproduction. Two pages quoting one press release are one + source. +5. **Date it.** Record version and date for anything that drifts — APIs, prices, + defaults, package behaviour. An undated fact is a guess with a citation. + +## Output + + Answer: + + | Claim | Source | How verified | + |---|---|---| + | … | url / file:line / tool call | read / reproduced / cross-checked | + + Confidence: high | medium | low — + Unverified: + +Name the gaps. Silence about a gap reads as certainty. + +## Do NOT + +- Answer from memory because the question "feels" settled. +- Let one blog post carry a factual answer. +- Repeat a failed search verbatim — reformulate twice, then report the dead end. +- Pad a thin evidence base with confident prose. + +## Related + +`websearch` (the web rung's tool) · `knowledge-base` (write up what you learned) · +`gh`, `github-cli` (issue and PR evidence) diff --git a/AGENTS.md b/AGENTS.md index 7ebbe2c..f6c5141 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,7 +154,7 @@ python3 bin/automation-ladder.py sync-agents # then restart the agent session hook is the tighter follow-up. ## Skills (load with: opencode tool skill use ) -`startsession` | `lifecycle` | `git` | `security` | `backup` | `infra` | `check-connectivity` | `templates` | `tools` | `endsession` | `idea-triage` | `idea-cmd` | `baseline` | `gh` | `helpdesk-ticket` | `frappe-access` | `dogfood-cycle` | `release` | `zabbix` | `cleanup` | `handoff` | `ticket-triage` | `gitlab-pages-dns` | `grind` | `knowledge-base` | `routeros` | `github-cli` | `triage` | `bwunlock` | `websearch` | `technitium-dns` | `proxmox-cli` | `vikunja-ticket` | `vikunja-user` | `mail-auth-check` | `shared-browser` +`startsession` | `lifecycle` | `git` | `security` | `backup` | `infra` | `check-connectivity` | `templates` | `tools` | `endsession` | `idea-triage` | `idea-cmd` | `baseline` | `gh` | `helpdesk-ticket` | `frappe-access` | `dogfood-cycle` | `release` | `zabbix` | `cleanup` | `handoff` | `ticket-triage` | `gitlab-pages-dns` | `grind` | `knowledge-base` | `routeros` | `github-cli` | `triage` | `bwunlock` | `research` | `websearch` | `technitium-dns` | `proxmox-cli` | `vikunja-ticket` | `vikunja-user` | `mail-auth-check` | `shared-browser` Load the relevant skill before working in its domain. From 4be10e598845d4dd904a20eac56658946053c346 Mon Sep 17 00:00:00 2001 From: growlf Date: Sun, 25 Oct 2026 14:08:43 -0700 Subject: [PATCH 3/4] fix(skills): make /research self-contained (#452) The command now carries the mandate and the three-part output contract, so compliance no longer depends on the model choosing to load the skill first. Verified through the command path (POST /api/session/{id}/command): the template reaches the model with the topic substituted, and the local model returns the answer, an evidence table, a confidence line, and an unverified line. Correction: an earlier draft of this message claimed the model skipped the skill and ignored the format when driven by 'opencode run'. That command does not expand slash commands at all, so the template never reached the model. The failure was in the test harness, not the model. --- .opencode/command/research.md | 33 ++++++++++++++++++------------ .opencode/skills/research/SKILL.md | 21 ++++++++++++------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/.opencode/command/research.md b/.opencode/command/research.md index 0fbb8c8..94b7ab5 100644 --- a/.opencode/command/research.md +++ b/.opencode/command/research.md @@ -2,24 +2,31 @@ description: Research a topic from primary sources — per-claim citations, confidence, and explicit unknowns instead of guesses --- -# /research — evidence-first answer +# /research -## Description +Topic: $ARGUMENTS -Research a topic to a sourced answer. Every load-bearing claim carries a source read in this session; confidence and unverified gaps are stated plainly. +**Mandate:** use all available and appropriate skills, tools, web research, and +other resources to establish what is factually true. No claim reaches the +operator unless a source was read in this session. A guess is a defect; +"unverified" is an acceptable answer. -## Triggers - -`/research`, `research `, `find out whether`, `verify that`, `is it true that`, `look up` - -## Procedure - -Load the `research` skill and run it against: $ARGUMENTS +Load the `research` skill (skill tool, id `research`) for the evidence ladder, +then: 1. Restate the question and the decision it feeds. -2. Walk the evidence ladder — repo/code, then live tools, then vendor docs, then +2. Walk the ladder — repo/code first, then live tools, then vendor docs, then secondary sources. 3. Cross-check load-bearing claims, and date anything that drifts. -4. Return the answer with the evidence table, confidence, and the unverified list. +4. Answer in exactly three parts. Prose alone is a failed run: + + **Answer:** one paragraph, direct. + + | Claim | Source | How verified | + |---|---|---| + | … | url / `file:line` / tool call | read / reproduced / cross-checked | + + **Confidence:** high · medium · low — why. + **Unverified:** what you could not confirm, or "none". -If `$ARGUMENTS` is empty, ask what to research. Never answer from memory. +If the topic above is empty, ask what to research. Never answer from memory. diff --git a/.opencode/skills/research/SKILL.md b/.opencode/skills/research/SKILL.md index 37e720e..d635675 100644 --- a/.opencode/skills/research/SKILL.md +++ b/.opencode/skills/research/SKILL.md @@ -20,6 +20,13 @@ you what is true. "Unverified" and "I don't know" are complete answers. A confident guess is not, because the operator cannot tell it apart from a sourced one. +Every answer carries three parts — prose alone is an **incomplete answer**, +because the operator cannot audit it: + +1. the answer, direct; +2. an evidence table, one row per load-bearing claim; +3. confidence, plus an explicit list of what is still unverified. + ## Evidence ladder — cheapest authoritative rung first | Rung | Source | Reach for | @@ -45,16 +52,16 @@ every time: look before you search. 5. **Date it.** Record version and date for anything that drifts — APIs, prices, defaults, package behaviour. An undated fact is a guess with a citation. -## Output +## Output — return it literally, not as prose - Answer: +**Answer:** one paragraph, direct. - | Claim | Source | How verified | - |---|---|---| - | … | url / file:line / tool call | read / reproduced / cross-checked | +| Claim | Source | How verified | +|---|---|---| +| … | url / `file:line` / tool call | read / reproduced / cross-checked | - Confidence: high | medium | low — - Unverified: +**Confidence:** high · medium · low — why. +**Unverified:** what you could not confirm, and what would confirm it (or "none"). Name the gaps. Silence about a gap reads as certainty. From d18d9a27e36cfdf51f624e9fe7c4eddcd44c827e Mon Sep 17 00:00:00 2001 From: growlf Date: Sun, 25 Oct 2026 14:30:43 -0700 Subject: [PATCH 4/4] test(skills): guard the command's three-part output contract (#452) The command body is the only text guaranteed to reach the model on either path, and the review found the format contract unguarded: dropping the table or the confidence line would leave every test green while the deliverable disappeared. --- tests/test_research_skill.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_research_skill.py b/tests/test_research_skill.py index d5f116c..04b88a2 100644 --- a/tests/test_research_skill.py +++ b/tests/test_research_skill.py @@ -77,6 +77,27 @@ def test_command_loads_the_skill_and_takes_arguments(): assert "$ARGUMENTS" in body, "the command must pass the operator's topic through" +def test_command_carries_the_output_contract(): + """The command body is the only text guaranteed to reach the model. + + Verified 2026-10-25 by capturing the provider request: `opencode run` does + not expand slash commands, so on that path the literal text is the whole + prompt; on the command path the body is the whole prompt. Either way, if the + body loses the format contract, nothing else supplies it — the skill is only + read when the model chooses to load it. + """ + _, _, body = _split_frontmatter(COMMAND) + for required in ( + "| Claim | Source | How verified |", + "**Confidence:**", + "**Unverified:**", + ): + assert required in body, ( + f"the command no longer states {required!r}; the three-part answer " + "is the deliverable, not a suggestion" + ) + + def test_skill_is_registered_in_agents_md(): assert "`research`" in AGENTS.read_text(), ( "AGENTS.md is the discovery surface agents read first; an unregistered "