diff --git a/docs/agents/behaviour-catalog.md b/docs/agents/behaviour-catalog.md new file mode 100644 index 0000000000..be48d160b6 --- /dev/null +++ b/docs/agents/behaviour-catalog.md @@ -0,0 +1,164 @@ +# Behaviour catalog + +Generated from `agent_comms.behaviour_patterns` by +`python -m agent_comms.behaviour_patterns > docs/agents/behaviour-catalog.md`; do not edit by hand. + +Each behaviour failure pattern is declared once in Core with its signature (what it looks like in state, readable without trusting a report) and the existing checks that enforce it. Patterns in code, rather than in behaviour, are the refactor-audit skill's catalog (`refactor-audit/references/patterns/README.md`). + +A pending check records a known gap, not enforcement. Adding a pattern means adding one `BehaviourPattern` subclass; membership and this document derive from those classes. `tests/test_behaviour_catalog.py` rejects hand edits. + +The [Codex postmortem](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md) names these failures; coverage below derives from the patterns that cite them: + +- [Failure 1](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It rewrote the framework instead of questioning its own design: no pattern cites it yet +- [Failure 2](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It added states instead of removing them: no pattern cites it yet +- [Failure 3](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It wrote down the right principle and then violated it: no pattern cites it yet +- [Failure 4](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It measured the wrong thing and reported it as progress: `uncounted_check`, `wrong_measure` +- [Failure 5](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It redefined "the real app" until something passed: no pattern cites it yet +- [Failure 6](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It ignored direct orders: no pattern cites it yet +- [Failure 7](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It described its intentions as its accomplishments: `board_disagrees`, `uncounted_check` +- [Failure 8](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It drowned the work in ceremony: `plain_text`, `undeclared_audience` +- [Failure 9](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It merged over its own checks and broke things in circles: no pattern cites it yet +- [Failure 10](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md), It invented rules and blamed them on Tristan: no pattern cites it yet + +## plan_idle + +A plan phase whose earlier phases are all done is not done itself, and no open order held by a unit serves it. + +Checked by: + +- standing rule `plan_idle` (`agent_comms.standing_rules.PlanIdle`) + +Learned from: P83 (34ba68c81) + +## board_disagrees + +An open order's progress reads no commit while a pull request linked to it has merged. + +Checked by: + +- standing rule `board_disagrees` (`agent_comms.standing_rules.BoardDisagrees`) + +Learned from: P170 (c9c0c04db); [Codex postmortem, failure 7](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md) + +## tactical_command + +The commander's own session runs an install, maintenance, connector restart or carry command instead of ordering a unit to. + +Checked by: + +- check in `agent_comms.commander.send_guard.TacticalCommand.install_refusals` + +Learned from: P171 (cba7b09c3) + +## plain_text + +A commander order, strike, roast or directive whose words outside cited spans match the declared jargon list in `PlainText`. + +Checked by: + +- check in `agent_comms.todos.PlainText` + +Learned from: [comms-commander rule](../../src/agent_comms/skills/comms-commander/SKILL.md) "Hate jargon."; [Codex postmortem, failure 8](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md) + +## undeclared_audience + +A send to a channel, which reaches many threads, names no thread as its audience. + +Checked by: + +- check in `agent_comms.channel_targets.ChannelTarget.addressed` + +Learned from: commit dc4fb3bfa; [Codex postmortem, failure 8](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md) + +## repeated_strike + +A second storage strike with the same actor, owner and reason as a live strike, or a send-back strike on an order already struck since it was sent. + +Checked by: + +- check in `agent_comms.todos.StorageStrikeTodo.repeats` +- check in `agent_comms.todos.Todo.struck_on_order` + +Learned from: P184 (7af14a520) + +## strike_in_error + +A strike recorded in error is deleted, or voided while not live, instead of a void recorded beside it with a plain reason. + +Checked by: + +- check in `agent_comms.todos.VoidStrikeTodo.apply` + +Learned from: P47 (15d435152) + +## uncounted_check + +An order whose check line does not read `repo :: command -> integer`, prints no integer, or already prints its target before any work. + +Checked by: + +- check in `agent_comms.goals.OrderCheck.parse` +- check in `agent_comms.child_process.NoIntegerPrinted.unproven` +- check in `agent_comms.child_process.CountPrinted.unproven` + +Learned from: P114 (d889df612); [Codex postmortem, failure 4](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md); [Codex postmortem, failure 7](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md) + +## strike_on_named_problem + +A strike recorded on a problem other than the one given as its argument, such as one moved by a problem number inside its reason. Hunts are commander records, not strikes on a problem. + +Checked by: + +- check in `agent_comms.todos.TodoStore.apply` + +Learned from: P194 (b26843727); [comms-commander rule](../../src/agent_comms/skills/comms-commander/SKILL.md) "Strikes, then fire." + +## report_instead_of_state + +A strike or send-back whose reason cites no state the record can check. + +Checked by: + +- not enforced here: P198 (#1744) + +Learned from: seed catalog pattern 1 (claude-ai message 1c8729ba0ed7) + +## wrong_measure + +A done report that names its own measure instead of what its order's Check printed. + +Checked by: + +- not enforced here: P197 (#1733) + +Learned from: seed catalog pattern 4 (claude-ai message 1c8729ba0ed7); [Codex postmortem, failure 4](https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md) + +## commander_overloaded + +The commander's unread backlog is over the limit while no second commander is forked to share it. + +Checked by: + +- not enforced here: P199 (#1750, closed without merge) + +Learned from: seed catalog pattern 15 (claude-ai message 1c8729ba0ed7) + +## dead_provider + +A unit's latest turn ended in a usage or credit limit on its own model and the unit still runs on that model. + +Checked by: + +- not enforced here: P200 (#1746, closed without merge) + +Learned from: seed catalog pattern 16 (claude-ai message 1c8729ba0ed7) + +## unowned_work + +Agent-comms work runs under no open order, or on a runtime that publication replaced. + +Checked by: + +- not enforced here: P201 (#1739) + +Learned from: seed catalog pattern 18 (claude-ai message 1c8729ba0ed7) diff --git a/src/agent_comms/behaviour_patterns.py b/src/agent_comms/behaviour_patterns.py new file mode 100644 index 0000000000..6abd717c28 --- /dev/null +++ b/src/agent_comms/behaviour_patterns.py @@ -0,0 +1,343 @@ +"""Behaviour failure patterns Core catches: each declared once with its signature and check. + +A member names what the pattern looks like in state, readable without trusting +a report, the existing checks that enforce it, and where the pattern was learned. +``docs/agents/behaviour-catalog.md`` is this family's ``catalog()``; regenerate it +with ``python -m agent_comms.behaviour_patterns > docs/agents/behaviour-catalog.md``. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable +from dataclasses import dataclass +from typing import ClassVar + +from .channel_targets import ChannelTarget +from .child_process import CountPrinted, NoIntegerPrinted +from .commander.send_guard import TacticalCommand +from .declared_family import DeclaredFamily +from .goals import OrderCheck +from .standing_rules import BoardDisagrees, PlanIdle, StandingRule +from .todos import PlainText, StorageStrikeTodo, Todo, TodoStore, VoidStrikeTodo + +POSTMORTEM = "https://github.com/OpenHCSDev/toad/blob/main/docs/agents/CODEX-POSTMORTEM.md" +COMMANDER_SKILL = "../../src/agent_comms/skills/comms-commander/SKILL.md" + + +class PatternCheck(ABC): + """An existing check that enforces a pattern.""" + + @abstractmethod + def cite(self) -> str: ... + + +@dataclass(frozen=True) +class RuleCheck(PatternCheck): + """A StandingRule member that reports the pattern on every read of state.""" + + rule: type[StandingRule] + + def cite(self) -> str: + where = f"{self.rule.__module__}.{self.rule.__qualname__}" + return f"standing rule `{self.rule.declared_name}` (`{where}`)" + + +@dataclass(frozen=True) +class CallableCheck(PatternCheck): + """The existing implementation that checks state or refuses the action.""" + + check: Callable[..., object] + + def cite(self) -> str: + return f"check in `{self.check.__module__}.{self.check.__qualname__}`" + + +@dataclass(frozen=True) +class PendingCheck(PatternCheck): + """A check absent from this checkout; its implementation replaces this citation.""" + + problem: str + change: str + + def cite(self) -> str: + return f"not enforced here: {self.problem} ({self.change})" + + +class PatternSource(ABC): + """Where a pattern was learned.""" + + @abstractmethod + def cite(self) -> str: ... + + +class PostmortemFailure(PatternSource, DeclaredFamily, affix="Failure"): + """A failure the Codex postmortem names; a pattern that checks it cites an instance.""" + + number: ClassVar[int] + title: ClassVar[str] + + def cite(self) -> str: + return f"[Codex postmortem, failure {self.number}]({POSTMORTEM})" + + +class RewroteFrameworkFailure(PostmortemFailure): + number = 1 + title = "It rewrote the framework instead of questioning its own design" + + +class AddedStatesFailure(PostmortemFailure): + number = 2 + title = "It added states instead of removing them" + + +class ViolatedPrincipleFailure(PostmortemFailure): + number = 3 + title = "It wrote down the right principle and then violated it" + + +class MeasuredWrongThingFailure(PostmortemFailure): + number = 4 + title = "It measured the wrong thing and reported it as progress" + + +class RedefinedRealAppFailure(PostmortemFailure): + number = 5 + title = 'It redefined "the real app" until something passed' + + +class IgnoredOrdersFailure(PostmortemFailure): + number = 6 + title = "It ignored direct orders" + + +class IntentionsAsAccomplishmentsFailure(PostmortemFailure): + number = 7 + title = "It described its intentions as its accomplishments" + + +class CeremonyFailure(PostmortemFailure): + number = 8 + title = "It drowned the work in ceremony" + + +class MergedOverChecksFailure(PostmortemFailure): + number = 9 + title = "It merged over its own checks and broke things in circles" + + +class InventedRulesFailure(PostmortemFailure): + number = 10 + title = "It invented rules and blamed them on Tristan" + + +@dataclass(frozen=True) +class CommanderRule(PatternSource): + lead: str + + def cite(self) -> str: + return f'[comms-commander rule]({COMMANDER_SKILL}) "{self.lead}"' + + +@dataclass(frozen=True) +class ProblemFix(PatternSource): + problem: str + commit: str + + def cite(self) -> str: + return f"{self.problem} ({self.commit})" + + +@dataclass(frozen=True) +class SeedCatalog(PatternSource): + """A pattern's seed ID in claude-ai's numbered catalog (message 1c8729ba0ed7).""" + + seed: int + + def cite(self) -> str: + return f"seed catalog pattern {self.seed} (claude-ai message 1c8729ba0ed7)" + + +@dataclass(frozen=True) +class Commit(PatternSource): + commit: str + + def cite(self) -> str: + return f"commit {self.commit}" + + +class BehaviourPattern(DeclaredFamily, affix="Pattern"): + signature: ClassVar[str] + checks: ClassVar[tuple[PatternCheck, ...]] + sources: ClassVar[tuple[PatternSource, ...]] + + @classmethod + def section(cls) -> str: + checks = "".join(f"- {check.cite()}\n" for check in cls.checks) + sources = "; ".join(source.cite() for source in cls.sources) + return ( + f"## {cls.declared_name}\n\n{cls.signature}\n\n" + f"Checked by:\n\n{checks}\nLearned from: {sources}\n" + ) + + @classmethod + def catalog(cls) -> str: + members = cls.members_with(BehaviourPattern) + failures = "".join( + f"- [Failure {failure.number}]({POSTMORTEM}), {failure.title}: " + + (", ".join( + f"`{member.declared_name}`" for member in members + if any(isinstance(source, failure) for source in member.sources) + ) or "no pattern cites it yet") + + "\n" + for failure in PostmortemFailure.members_with(PostmortemFailure) + ) + return ( + "# Behaviour catalog\n\n" + "Generated from `agent_comms.behaviour_patterns` by\n" + "`python -m agent_comms.behaviour_patterns > docs/agents/behaviour-catalog.md`;" + " do not edit by hand.\n\n" + "Each behaviour failure pattern is declared once in Core with its signature (what it" + " looks like in state, readable without trusting a report) and the existing checks" + " that enforce it. Patterns in code, rather than in behaviour, are the refactor-audit" + " skill's catalog (`refactor-audit/references/patterns/README.md`).\n\n" + "A pending check records a known gap, not enforcement. Adding a pattern means" + " adding one `BehaviourPattern` subclass; membership and this document derive" + " from those classes. `tests/test_behaviour_catalog.py` rejects hand edits.\n\n" + f"The [Codex postmortem]({POSTMORTEM}) names these failures; coverage below derives" + " from the patterns that cite them:\n\n" + f"{failures}\n" + "\n".join(member.section() for member in members) + ) + + +class PlanIdlePattern(BehaviourPattern): + signature = ( + "A plan phase whose earlier phases are all done is not done itself, and no open" + " order held by a unit serves it." + ) + checks = (RuleCheck(PlanIdle),) + sources = (ProblemFix("P83", "34ba68c81"),) + + +class BoardDisagreesPattern(BehaviourPattern): + signature = ( + "An open order's progress reads no commit while a pull request linked to it has merged." + ) + checks = (RuleCheck(BoardDisagrees),) + sources = (ProblemFix("P170", "c9c0c04db"), IntentionsAsAccomplishmentsFailure()) + + +class TacticalCommandPattern(BehaviourPattern): + signature = ( + "The commander's own session runs an install, maintenance, connector restart or carry" + " command instead of ordering a unit to." + ) + checks = (CallableCheck(TacticalCommand.install_refusals),) + sources = (ProblemFix("P171", "cba7b09c3"),) + + +class PlainTextPattern(BehaviourPattern): + signature = ( + "A commander order, strike, roast or directive whose words outside cited spans match" + " the declared jargon list in `PlainText`." + ) + checks = (CallableCheck(PlainText),) + sources = (CommanderRule("Hate jargon."), CeremonyFailure()) + + +class UndeclaredAudiencePattern(BehaviourPattern): + signature = "A send to a channel, which reaches many threads, names no thread as its audience." + checks = (CallableCheck(ChannelTarget.addressed),) + sources = (Commit("dc4fb3bfa"), CeremonyFailure()) + + +class RepeatedStrikePattern(BehaviourPattern): + signature = ( + "A second storage strike with the same actor, owner and reason as a live strike, or a" + " send-back strike on an order already struck since it was sent." + ) + checks = ( + CallableCheck(StorageStrikeTodo.repeats), + CallableCheck(Todo.struck_on_order), + ) + sources = (ProblemFix("P184", "7af14a520"),) + + +class StrikeInErrorPattern(BehaviourPattern): + signature = ( + "A strike recorded in error is deleted, or voided while not live, instead of a void" + " recorded beside it with a plain reason." + ) + checks = (CallableCheck(VoidStrikeTodo.apply),) + sources = (ProblemFix("P47", "15d435152"),) + + +class UncountedCheckPattern(BehaviourPattern): + signature = ( + "An order whose check line does not read `repo :: command -> integer`, prints no" + " integer, or already prints its target before any work." + ) + checks = ( + CallableCheck(OrderCheck.parse), + CallableCheck(NoIntegerPrinted.unproven), + CallableCheck(CountPrinted.unproven), + ) + sources = ( + ProblemFix("P114", "d889df612"), + MeasuredWrongThingFailure(), + IntentionsAsAccomplishmentsFailure(), + ) + + +class StrikeOnNamedProblemPattern(BehaviourPattern): + signature = ( + "A strike recorded on a problem other than the one given as its argument, such" + " as one moved by a problem number inside its reason. Hunts are commander records," + " not strikes on a problem." + ) + checks = (CallableCheck(TodoStore.apply),) + sources = (ProblemFix("P194", "b26843727"), CommanderRule("Strikes, then fire.")) + + +class ReportInsteadOfStatePattern(BehaviourPattern): + signature = "A strike or send-back whose reason cites no state the record can check." + checks = (PendingCheck("P198", "#1744"),) + sources = (SeedCatalog(1),) + + +class WrongMeasurePattern(BehaviourPattern): + signature = ( + "A done report that names its own measure instead of what its order's Check printed." + ) + checks = (PendingCheck("P197", "#1733"),) + sources = (SeedCatalog(4), MeasuredWrongThingFailure()) + + +class CommanderOverloadedPattern(BehaviourPattern): + signature = ( + "The commander's unread backlog is over the limit while no second commander is forked" + " to share it." + ) + checks = (PendingCheck("P199", "#1750, closed without merge"),) + sources = (SeedCatalog(15),) + + +class DeadProviderPattern(BehaviourPattern): + signature = ( + "A unit's latest turn ended in a usage or credit limit on its own model and the unit" + " still runs on that model." + ) + checks = (PendingCheck("P200", "#1746, closed without merge"),) + sources = (SeedCatalog(16),) + + +class UnownedWorkPattern(BehaviourPattern): + signature = ( + "Agent-comms work runs under no open order, or on a runtime that publication replaced." + ) + checks = (PendingCheck("P201", "#1739"),) + sources = (SeedCatalog(18),) + + +if __name__ == "__main__": + print(BehaviourPattern.catalog(), end="") diff --git a/tests/test_behaviour_catalog.py b/tests/test_behaviour_catalog.py new file mode 100644 index 0000000000..5694fe411e --- /dev/null +++ b/tests/test_behaviour_catalog.py @@ -0,0 +1,35 @@ +"""The behaviour catalog in docs/agents is the family's output, never a hand edit.""" + +import os +import subprocess +import sys +from pathlib import Path + +from agent_comms.behaviour_patterns import BehaviourPattern, PatternCheck, PatternSource + +ROOT = Path(__file__).resolve().parents[1] + + +def test_behaviour_catalog_members_name_existing_checks_and_generate_the_file(): + members = BehaviourPattern.members_with(BehaviourPattern) + assert members + for member in members: + assert member.__dict__["signature"].strip(), member + assert member.__dict__["checks"] and member.__dict__["sources"], member + assert all(isinstance(check, PatternCheck) and check.cite() for check in member.checks) + assert all(isinstance(source, PatternSource) and source.cite() for source in member.sources) + assert (ROOT / "docs/agents/behaviour-catalog.md").read_text() == BehaviourPattern.catalog() + + +def test_catalog_command_needs_no_commander_installation_or_bus(tmp_path): + generated = subprocess.run( + [sys.executable, "-X", "faulthandler", "-m", "agent_comms.behaviour_patterns"], + cwd=tmp_path, + env=os.environ | {"HOME": str(tmp_path), "PYTHONPATH": str(ROOT / "src")}, + capture_output=True, + text=True, + timeout=30, + ) + assert generated.returncode == 0, generated.stderr + assert generated.stdout == BehaviourPattern.catalog() + assert not tuple(tmp_path.iterdir())