diff --git a/plans/commander-native-plan.md b/plans/commander-native-plan.md index 05042371b2..4fbae94e3a 100644 --- a/plans/commander-native-plan.md +++ b/plans/commander-native-plan.md @@ -61,7 +61,7 @@ | 28 | Measured load and self-report | `capacity.py` unit count | `commander.capacity.Load.of` reports orders, overdue, edits and forks; self-report comparison unverified | `control_plane` | | 29 | No handoff of finished, checked work | none; scratch transfer moved P507 after its fix was pushed and checked | Transfer refuses delivered checked work; that owner merges; check is reused | `record_plane` | | 30 | A proof is bound to the build it ran on | none; screenshots and logs are bare paths | Build run records runtime and Core/Toad revisions; close requires delivered commit | `record_plane` | -| 31 | An order states its intent as a deletion target | none; P520 renamed a decision rather than deleting it | Order owns deletion family/site/count; delivery refuses nonzero or renamed decisions | `record_plane` | +| 31 | An order states its intent as a deletion target | Current order command with a declared FamilyDeletion | CommsTodoCreateTool and CommsOrderDoneTool check the delivered family count | `record_plane` | | 32 | Where units' time goes, classified | none; activity.log lacks durations and turn boundaries | `UnitTime`: model, build/check/test/shell, dependency/commander waits and idle; classify large waits | `commander_page`, `control_plane` | | 33 | Order progress beside its deadline; the unit never sees a due time | `order.py board`, `overdue`, `due`, `send`; `rotate.py` | Derived `OrderProgress`; merged work draws no strike; first commit 10 min, later progress 15 min STALL; units see no due time | `one_order_owner`, `control_plane` | | 34 | Several commanders on one machine and bus, one per project | none; a second commander.json install overwrites the first | Registry commander/project ownership; caller from --from; scoped orders, strikes, load, board, alarms and forks; raw-send guard; duplicate project commander refused | `one_order_owner`, `cutover` | diff --git a/src/agent_comms/candidate_maintenance.py b/src/agent_comms/candidate_maintenance.py index c2186860b6..9a0fb16953 100644 --- a/src/agent_comms/candidate_maintenance.py +++ b/src/agent_comms/candidate_maintenance.py @@ -1,28 +1,22 @@ -"""Best-effort indexing of successful appends after the outer locks release. - -Losing a hint or crashing the daemon leaves the optional index unavailable; -canonical bus and sealed SQL still drive original delivery. Byte positions -include observations as well as messages, without inventing message sequences. -""" +"""Index successful appends after outer locks release, within the owning root's lifetime.""" from __future__ import annotations import logging import sqlite3 import threading -from contextlib import ExitStack from pathlib import Path from .bus_route_counts import BusRouteCounts from .coordination_errors import CoordinationError from .routing import DeliveryMessage -from .wire_log import CurrentSnapshotOpening, WireLog from .wake_candidate_index import FirstBuild, ProjectionUnavailableError, WakeCandidateIndex +from .wire_log import CurrentSnapshotOpening, WireLog _LOG = logging.getLogger(__name__) _MAX_ACTIVE_ROOTS = 32 _guard = threading.Lock() -_pending: dict[Path, tuple[WireLog, int]] = {} +_pending: dict[Path, CandidateMaintenance] = {} class CandidateMaintenanceFailed(RuntimeError): # noqa: N818 - named worker failure @@ -33,72 +27,82 @@ def __init__(self, root: Path) -> None: super().__init__(f"Candidate maintenance failed for {root}") -def schedule_candidate_catchup(log: WireLog, through_bytes: int) -> None: - """Prepare writer indexes, then signal their existing maintenance worker.""" - if through_bytes <= 0: - raise ValueError("candidate notification needs an actual committed byte position") - BusRouteCounts(log.path).prepare() - root = log.path.parent - with _guard: - current = _pending.get(root) - if current is not None: - _pending[root] = (current[0], max(current[1], through_bytes)) - return - if len(_pending) >= _MAX_ACTIVE_ROOTS: - raise CandidateMaintenanceFailed(root) - _pending[root] = (log, through_bytes) - try: - threading.Thread( - target=_drain_candidate, - args=(root,), - name="agent-comms-candidate-maintenance", - daemon=True, - ).start() - except RuntimeError as error: - _pending.pop(root, None) - raise CandidateMaintenanceFailed(root) from error - - -def _retire_candidate(root: Path) -> None: - with _guard: - _pending.pop(root, None) +class CandidateMaintenance(threading.Thread): + """One root's coalesced cursor and actual worker; retirement joins this owner.""" + def __init__(self, log: WireLog, through_bytes: int) -> None: + super().__init__(name="agent-comms-candidate-maintenance", daemon=True) + self.log = log + self.through_bytes = through_bytes -def _drain_candidate(root: Path) -> None: - with ExitStack() as cleanup: - cleanup.callback(_retire_candidate, root) + def run(self) -> None: + root = self.log.path.parent try: - with _guard: - log, _bytes = _pending[root] - # Source ownership is checked off the producer path. Maintenance cannot - # create the protocol marker or authorize delivery. - with log.checked_read() as source: + with self.log.checked_read() as source: metadata = source.marker root_id = metadata.root_id - index = WakeCandidateIndex(log) - routes = BusRouteCounts(log.path) + index = WakeCandidateIndex(self.log) + routes = BusRouteCounts(self.log.path) while True: with _guard: - target = _pending[root][1] + target = self.through_bytes hint = index.notify_committed_append(root_id=root_id, through_bytes=target) - result = index.catch_up_committed_append( - hint, start=FirstBuild() - ) - with log.opened_wire_snapshot(opening=CurrentSnapshotOpening()) as snapshot: + result = index.catch_up_committed_append(hint, start=FirstBuild()) + with self.log.opened_wire_snapshot(opening=CurrentSnapshotOpening()) as snapshot: route_result = routes.sync( snapshot, - lambda record: DeliveryMessage.from_wire(record, root_id=snapshot.metadata.root_id), + lambda record: DeliveryMessage.from_wire( + record, root_id=snapshot.metadata.root_id + ), ) if result.caught_up and route_result.complete: with _guard: - if _pending[root][1] <= result.checkpoint_bytes: - # Disarm cleanup before a successor can own this root. - cleanup.pop_all() + if self.through_bytes <= result.checkpoint_bytes: _pending.pop(root) return except ( - OSError, ValueError, sqlite3.Error, CoordinationError, ProjectionUnavailableError, + OSError, + ValueError, + sqlite3.Error, + CoordinationError, + ProjectionUnavailableError, ) as error: failure = CandidateMaintenanceFailed(root) _LOG.exception("%s", failure) raise failure from error + finally: + with _guard: + if root in _pending and _pending[root] is self: + _pending.pop(root) + + +def schedule_candidate_catchup(log: WireLog, through_bytes: int) -> None: + """Prepare writer indexes, then coalesce into the root's actual worker.""" + if through_bytes <= 0: + raise ValueError("candidate notification needs an actual committed byte position") + BusRouteCounts(log.path).prepare() + root = log.path.parent + with _guard: + if root in _pending: + worker = _pending[root] + worker.through_bytes = max(worker.through_bytes, through_bytes) + return + if len(_pending) >= _MAX_ACTIVE_ROOTS: + raise CandidateMaintenanceFailed(root) + worker = CandidateMaintenance(log, through_bytes) + _pending[root] = worker + try: + worker.start() + except RuntimeError as error: + _pending.pop(root) + raise CandidateMaintenanceFailed(root) from error + + +def drain_candidate_catchup(root: Path) -> None: + """A stopped root cannot be renamed or removed while its worker still writes.""" + while True: + with _guard: + if root not in _pending: + return + worker = _pending[root] + worker.join() diff --git a/src/agent_comms/check_root.py b/src/agent_comms/check_root.py index 02de2fb15a..7ace5f24d1 100644 --- a/src/agent_comms/check_root.py +++ b/src/agent_comms/check_root.py @@ -12,6 +12,7 @@ from typing import ClassVar from .attempt_recovery import RecoveryMonitorCapability +from .candidate_maintenance import drain_candidate_catchup from .child_process import NoProcess from .cli_result import CliResult from .comms import Comms @@ -74,6 +75,7 @@ def open(self) -> Iterator[Comms]: ): copied.owners.stop(thread.name) finally: + drain_candidate_catchup(root) RecoveryMonitorCapability.retire_check_root(root) def _populate(self, copied: Comms) -> None: diff --git a/src/agent_comms/commander/order.py b/src/agent_comms/commander/order.py index 33e6ee33de..3c4d19d81e 100644 --- a/src/agent_comms/commander/order.py +++ b/src/agent_comms/commander/order.py @@ -476,8 +476,7 @@ def problem(self, order_file, unit, due, where, disk=False, def close(self, pid, works, artifact=''): """Core owns closing: the installed evidence, the artifact and the no-open-order rule live on the Todo.""" evidence = artifact or self.comms.goals.todos.get(pid).order.closing_artifact() - ThreadCaller(self.commander).invoke(CommsProblemCloseTool(problem=pid, - works=works, artifact=evidence), self.comms) + self.call(CommsProblemCloseTool(problem=pid, works=works, artifact=evidence)) self.write_board() print(f'{pid} closed') @@ -729,8 +728,8 @@ def due(self, pid: str, at: str, why: str) -> None: moved = deadline(at) self.call(CommsChangeProblemTool(problem=pid, operation=AssignTodo( todo.revision, todo.assignment, moved, todo.order, todo.goal))) - self.on_record(f'{time.strftime("%Y-%m-%d %H:%M")} {todo.holder} ({pid} order {todo.order.id} due ' - f'{hhmm(todo.deadline)} moved to {hhmm(moved)}): {why}', 'Deadline', + self.on_record(f'{time.strftime("%Y-%m-%d %H:%M")} {todo.holder} ({pid} order {todo.order.id} ' + f'deadline changed): {why}', 'Deadline', VerdictAudience.of(todo.held().owner, todo.goal)) print(f'{pid}: due {hhmm(moved)}') diff --git a/src/agent_comms/commander_parity/__init__.py b/src/agent_comms/commander_parity/__init__.py index 22073f86a4..efe7d431bc 100644 --- a/src/agent_comms/commander_parity/__init__.py +++ b/src/agent_comms/commander_parity/__init__.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, ClassVar @@ -43,6 +43,7 @@ class ParityRow( current: ClassVar[str] native: ClassVar[str] phases: ClassVar[tuple[type[PlanPhase], ...]] + comparison: ClassVar[Callable[[], ParityComparison]] = UnverifiedParity @classmethod def compare(cls, todos: Iterable[Todo]) -> ParityComparison: @@ -51,7 +52,7 @@ def compare(cls, todos: Iterable[Todo]) -> ParityComparison: @classmethod def verify(cls) -> ParityComparison: """Run this declared workflow now; a run is not durable cutover authority.""" - return UnverifiedParity() + return cls.comparison() @classmethod def rows(cls) -> tuple[type[ParityRow], ...]: diff --git a/src/agent_comms/commander_parity/deletion.py b/src/agent_comms/commander_parity/deletion.py new file mode 100644 index 0000000000..78834661c6 --- /dev/null +++ b/src/agent_comms/commander_parity/deletion.py @@ -0,0 +1,261 @@ +"""Deletion intent parity through real orders, pushed commits and native delivery checks.""" + +from __future__ import annotations + +import contextlib +import io +import json +import shlex +import sys +import tempfile +from abc import ABC, abstractmethod +from dataclasses import dataclass, replace +from pathlib import Path + +from ..change_report import FileLine +from ..child_process import CountPrinted, ExitedOutcome +from ..commander.order import OrderFile, deadline +from ..field_codec import FieldCodec +from ..goal_actions import CreateProblemGoalAction +from ..goals import CountCheckResult, OrderCheck +from ..plan_phases import RecordPlanePhase +from ..progress_claims import MergedClaim +from ..repo import Repo +from ..todos import ( + CheckedCommitDelivery, + CommitDelivery, + DeliveredProblemOrder, + FamilyDeletion, + MechanicalOrderRun, + Todo, +) +from ..tool_caller import ThreadCaller +from ..tools import CommsOrderDoneTool, CommsTodoCreateTool, ProblemOrdered +from . import ParityComparison, ParityRow +from .lifecycle import LifecycleObservation, LifecycleRun, MeasuredCommander + + +@dataclass(frozen=True) +class DeletionComparison(ParityComparison): + current: LifecycleObservation + native: LifecycleObservation + + @property + def passed(self) -> bool: + return ( + self.current.passed and self.native.passed and self.current.checks == self.native.checks + ) + + def describe(self) -> str: + return f"current: {self.current.describe()}; native: {self.native.describe()}" + + @classmethod + def read(cls) -> DeletionComparison: + return cls(CurrentDeletionPath.read(), NativeDeletionPath.read()) + + +class DeletionPath(ABC): + """Both adapters exercise the same scenario on independent LifecycleRun copies.""" + + @classmethod + @abstractmethod + def send(cls, run: LifecycleRun, document: OrderFile) -> Todo: ... + + @classmethod + @abstractmethod + def delivery_command(cls, run: LifecycleRun, todo: Todo, commit: str) -> tuple[str, ...]: ... + + @staticmethod + def commit(repository: Path, text: str, message: str) -> str: + (repository / "family.py").write_text(text) + Repo.git(repository, "add", "family.py") + Repo.git(repository, "commit", "-qm", message) + Repo.git(repository, "push", "-q", "origin", "main") + return Repo.git(repository, "rev-parse", "HEAD").strip() + + @classmethod + def read(cls) -> LifecycleObservation: + with LifecycleRun.open() as original: + # Keep source checkouts out of the copied worker's storage accounting. + # Limits remain unchanged; all detached checks use this persistent owned base. + rule = original.commander.comms.registry.storage_rule + with tempfile.TemporaryDirectory( + prefix="parity-deletion-", dir=rule.checkout_base + ) as checkouts: + root = original.commander.root + (root / "storage.json").write_text( + json.dumps(FieldCodec.encode(replace(rule, checkout_base=Path(checkouts)))) + ) + (root / "commander.json").write_text(original.commander.model_dump_json()) + run = replace(original, commander=MeasuredCommander.read_root(root)) + return cls.exercise(run) + + @classmethod + def exercise(cls, run: LifecycleRun) -> LifecycleObservation: + repository = Path(run.commander.comms.registry.require(run.worker).worktree) + remote = repository.parent / "deletion-origin.git" + Repo.git(repository, "init", "-q", "-b", "main") + Repo.git(repository, "config", "user.name", "Deletion parity") + Repo.git(repository, "config", "user.email", "deletion-parity@localhost") + Repo.git(repository, "init", "--bare", "-q", "--initial-branch=main", str(remote)) + Repo.git(repository, "remote", "add", "origin", str(remote)) + family = "class RetiredDecision:\n pass\n\n" + cls.commit( + repository, + family + "def decisions():\n return [RetiredDecision()]\n", + "Declare the obsolete decision family and its caller", + ) + count = OrderCheck( + str(repository), + shlex.join( + ( + sys.executable, + "-c", + "import ast;from pathlib import Path;" + "print(sum(isinstance(node,ast.ClassDef) for node in " + "ast.walk(ast.parse(Path('family.py').read_text()))))", + ) + ), + CountCheckResult(0), + ) + deletion = FamilyDeletion( + "obsolete decision family", FileLine(str(repository / "family.py"), 1), count + ) + initial = OrderFile.read(run.document.read_text()) + order = replace( + initial.order, + check=OrderCheck( + str(repository), + shlex.join((sys.executable, "-c", "import family;assert not family.decisions()")), + ), + run=MechanicalOrderRun(), + deletes=deletion, + ) + document = replace(initial, order=order) + run.document.write_text(json.dumps(FieldCodec.encode(document))) + output = io.StringIO() + with contextlib.redirect_stdout(output): + before = cls.send(run, document) + partial = cls.commit( + repository, + family + "def decisions():\n return []\n", + "Repair the caller but leave one obsolete family site", + ) + refused = run.invoke_command( + cls.delivery_command(run, before, partial), env=run.python_environment() + ) + after_refusal = run.commander.comms.goals.todos.get(before.id) + remaining = count.recorded(partial, str(repository)) + complete = cls.commit( + repository, + "def decisions():\n return []\n", + "Delete the last obsolete family declaration", + ) + accepted = run.invoke_command( + cls.delivery_command(run, before, complete), env=run.python_environment() + ) + after_acceptance = run.commander.comms.goals.todos.get(before.id) + zero = count.recorded(complete, str(repository)) + primary = order.check.recorded(partial, str(repository)) + delivered = after_acceptance.order + checks = ( + ("native order retained deletion intent", before.order.checked().deletes == deletion), + ( + "partial commit passed primary check", + bool(primary) and order.check.result.distance(primary[0]) == CountPrinted(0), + ), + ( + "partial commit retained one family site", + bool(remaining) and count.result.distance(remaining[0]) == CountPrinted(1), + ), + ("nonzero delivery refused", refused.outcome == ExitedOutcome(1)), + ("refusal preserved pending order", before == after_refusal and after_refusal.pending), + ( + "zero count checked on delivered commit", + bool(zero) and count.result.distance(zero[0]) == CountPrinted(0), + ), + ( + "zero delivery accepted", + accepted.outcome == ExitedOutcome(0) and not after_acceptance.pending, + ), + ( + "accepted delivery names the zero-site commit", + isinstance(delivered, DeliveredProblemOrder) + and isinstance(delivered.delivery, CheckedCommitDelivery) + and delivered.delivery.delivered_commit == complete, + ), + ("accepted delivery retained deletion intent", delivered.checked().deletes == deletion), + ) + return LifecycleObservation( + CommsOrderDoneTool.__qualname__, + len((refused, accepted)), + checks, + output.getvalue() + f"partial={partial}; zero={complete}; " + f"refused={refused.outcome.summary}: " + f"{refused.stdout.decode(errors='replace')}{refused.stderr.decode(errors='replace')}; " + f"accepted={accepted.outcome.summary}: " + f"{accepted.stdout.decode(errors='replace')}{accepted.stderr.decode(errors='replace')}", + ) + + +class CurrentDeletionPath(DeletionPath): + @classmethod + def send(cls, run: LifecycleRun, document: OrderFile) -> Todo: + return run.create() + + @classmethod + def delivery_command(cls, run: LifecycleRun, todo: Todo, commit: str) -> tuple[str, ...]: + return ( + sys.executable, + "-c", + "import sys;from agent_comms.commander.order import Commander,DoneVerb;" + "DoneVerb(unit=sys.argv[2],message=sys.argv[3],sha=sys.argv[4]).run(" + "Commander.read_root(sys.argv[1]))", + str(run.commander.root), + run.worker, + todo.order.checked().id, + commit, + ) + + +class NativeDeletionPath(DeletionPath): + @classmethod + def send(cls, run: LifecycleRun, document: OrderFile) -> Todo: + result = run.commander.call( + CommsTodoCreateTool( + owner=run.worker, + order=CreateProblemGoalAction( + problem="P1", + repo=Repo.of(Path(document.order.check.directory)).github, + text=document.defect, + order=document.order, + request=document.order.request( + run.commander.comms.registry.require(run.commander.commander), ("P1",) + ), + deadline=deadline(run.due), + commander=run.commander.commander, + ), + ) + ) + if not isinstance(result, ProblemOrdered): + raise TypeError("Native creation returned no ordered problem") + return result.todo + + @classmethod + def delivery_command(cls, run: LifecycleRun, todo: Todo, commit: str) -> tuple[str, ...]: + request = CommsOrderDoneTool( + problem=todo.id, + order_id=todo.order.checked().id, + delivery=CommitDelivery(commit=commit, delivered_commit=commit), + claims=(MergedClaim(commit),), + ) + return request.cli_command(ThreadCaller(run.commander.commander)).argv(run.commander.root) + + +class DeletionTargetParityRow(ParityRow): + number = 31 + capability = "An order states its intent as a deletion target" + current = "Current order command with a declared FamilyDeletion" + native = "CommsTodoCreateTool and CommsOrderDoneTool check the delivered family count" + phases = (RecordPlanePhase,) + comparison = DeletionComparison.read diff --git a/src/agent_comms/commander_parity/handoff.py b/src/agent_comms/commander_parity/handoff.py new file mode 100644 index 0000000000..2507f40859 --- /dev/null +++ b/src/agent_comms/commander_parity/handoff.py @@ -0,0 +1,150 @@ +"""Actual current/native refusals to transfer delivered checked work.""" + +from __future__ import annotations + +import contextlib +import io +import json +from dataclasses import dataclass, replace +from pathlib import Path +from uuid import uuid4 + +from ..change_report import FileLine +from ..child_process import ChildOutcome, ChildResult, ExitedOutcome +from ..commander.config import CommanderConfig +from ..commander.order import OrderFile, deadline +from ..field_codec import FieldCodec +from ..goal_actions import OrderGoalAction +from ..tools import CommsOrderTool +from . import ParityComparison, UnverifiedParity +from .lifecycle import LifecycleRun, MeasuredCommander + + +@dataclass(frozen=True) +class HandoffComparison(ParityComparison): + cases: tuple[ + tuple[str, ChildOutcome, ChildOutcome, tuple[str, ...], tuple[str, ...], bool], ... + ] + + @property + def passed(self) -> bool: + return len(self.cases) == 2 and all( + all( + ( + expected == ExitedOutcome(1), + actual == ExitedOutcome(1), + bool(reference), + reference == refusal, + unchanged, + ) + ) + for _, expected, actual, reference, refusal, unchanged in self.cases + ) + + def describe(self) -> str: + return f"actual finished-order transfer refusals (current/native): {self.cases!r}" + + +class FinishedHandoffCase: + @staticmethod + def state(run: LifecycleRun, pid: str, receiver: str) -> str: + return json.dumps( + FieldCodec.encode( + ( + run.commander.comms.goals.todos.get(pid), + run.commander.comms.registry.require(run.worker).goal, + run.commander.comms.registry.require(receiver).goal, + ) + ), + sort_keys=True, + ) + + @staticmethod + def current(run: LifecycleRun, receiver: str, document: OrderFile, pid: str) -> ChildResult: + return run.invoke( + "import sys; from agent_comms.commander.order import Commander, OrderFile; " + "Commander.model_validate_json(sys.argv[1]).send(sys.argv[2], sys.argv[3], " + "OrderFile.read(sys.argv[4]), reassign=sys.argv[5], traced=sys.argv[6])", + run.commander.model_dump_json(), + receiver, + run.due, + json.dumps(FieldCodec.encode(document)), + "Compare the actual refusal to hand completed checked work to another real unit.", + f"{Path(__file__).resolve()}:1", + ) + + @staticmethod + def native(run: LifecycleRun, receiver: str, document: OrderFile, pid: str) -> ChildResult: + context = run.commander + todo = context.comms.goals.todos.get(pid) + tool = CommsOrderTool( + owner=receiver, + order=OrderGoalAction( + problem=pid, + expected_revision=todo.revision, + assigner=context.incarnation, + order=document.order, + deadline=deadline(run.due), + commander=context.commander, + request=document.order.request( + context.comms.registry.require(context.commander), (pid,) + ), + traced=FileLine.from_wire(f"{Path(__file__).resolve()}:1"), + ), + ) + return run.invoke( + "from pathlib import Path; import sys; from agent_comms.comms import Comms; " + "from agent_comms.field_codec import FieldCodec; " + "from agent_comms.tool_caller import ThreadCaller; " + "from agent_comms.tools import CommsOrderTool; " + "ThreadCaller(sys.argv[2]).invoke(FieldCodec.decode(CommsOrderTool, " + "FieldCodec.parse(sys.argv[3])), Comms(Path(sys.argv[1])))", + str(context.root), + context.commander, + json.dumps(FieldCodec.encode(tool)), + ) + + @classmethod + def read(cls) -> ParityComparison: + authority = MeasuredCommander.read(CommanderConfig.path()) + owners = tuple( + dict.fromkeys( + todo.held().owner + for todo in authority.comms.goals.todos.held_for_commander(authority.incarnation) + ) + )[-2:] + if len(owners) < 2: + return UnverifiedParity("Finished-work transfer needs two real registered unit holders") + cases = [] + for label, execute in (("current", cls.current), ("native", cls.native)): + with LifecycleRun.open(extra_threads=owners) as run: + with contextlib.redirect_stdout(io.StringIO()): + receiver = next(owner for owner in owners if owner != run.worker) + delivered = run.complete(run.create()) + before = cls.state(run, delivered.id, receiver) + document = OrderFile.read(run.document.read_text()) + document = replace( + document, + problems=(delivered.id,), + order=replace(document.order, id=uuid4().hex), + ) + run.decisions.write_text(json.dumps([receiver])) + reference = run.invoke( + "from pathlib import Path; import sys; from agent_comms.comms import Comms; " + "todo=Comms(Path(sys.argv[1])).goals.todos.get(sys.argv[2]); " + "todo.order.require_transfer(todo)", + str(run.commander.root), + delivered.id, + ) + attempt = execute(run, receiver, document, delivered.id) + cases.append( + ( + label, + reference.outcome, + attempt.outcome, + tuple(reference.stderr.decode(errors="replace").splitlines()[-1:]), + tuple(attempt.stderr.decode(errors="replace").splitlines()[-1:]), + before == cls.state(run, delivered.id, receiver), + ) + ) + return HandoffComparison(tuple(cases)) diff --git a/src/agent_comms/commander_parity/lifecycle.py b/src/agent_comms/commander_parity/lifecycle.py new file mode 100644 index 0000000000..60117649bd --- /dev/null +++ b/src/agent_comms/commander_parity/lifecycle.py @@ -0,0 +1,508 @@ +"""Exercise current order commands against real native owners on a selected bus copy.""" + +from __future__ import annotations + +import asyncio +import contextlib +import io +import json +import os +import shlex +import sys +import time +from abc import ABC, abstractmethod +from collections.abc import Iterator +from dataclasses import dataclass, replace +from pathlib import Path +from typing import ClassVar +from uuid import uuid4 + +from pydantic import PrivateAttr + +from ..change_report import FileLine +from ..check_root import CheckRoot +from ..child_process import BoundedRun, ChildResult +from ..commander.config import CommanderConfig +from ..commander.order import Commander, OrderFile, hhmm +from ..commander.request import RequestAddVerb +from ..field_codec import FieldCodec +from ..goals import CountCheckResult, OrderCheck, ProblemGoalOrder +from ..todos import ( + ApplicationOrderRun, + MergedProgress, + Moving, + NoCommitProgress, + PendingProblemOrder, + Todo, +) +from ..tool_caller import ThreadCaller, ToolCaller +from ..tool_result import ToolResult +from ..tools import ( + CommsChangeProblemTool, + CommsFireTool, + CommsOrderTool, + CommsProblemCloseTool, + CommsTodoCreateTool, + CommsTodoListTool, + ProblemOrdered, + TodoChanged, + TodoList, + ToolRequest, +) +from . import ParityComparison, UnverifiedParity + + +@dataclass(frozen=True) +class LifecycleObservation(ParityComparison): + owner: str + invocations: int + checks: tuple[tuple[str, bool], ...] + output: str + + @property + def passed(self) -> bool: + return bool(self.invocations and self.checks) and all(value for _, value in self.checks) + + def describe(self) -> str: + return ( + f"owner={self.owner}; calls={self.invocations}; " + f"checks={dict(self.checks)}; output={self.output!r}" + ) + + +type LifecycleRequest = ToolRequest[ToolCaller] | ToolRequest[ThreadCaller] + + +@dataclass(frozen=True) +class NativeInvocation: + request: LifecycleRequest + result: ToolResult + + +class MeasuredCommander(Commander): + """Keep actual native returns at the existing command boundary, not frame locals.""" + + _invocations: list[NativeInvocation] = PrivateAttr(default_factory=list) + + @property + def invocations(self) -> tuple[NativeInvocation, ...]: + return tuple(self._invocations) + + def call(self, tool: LifecycleRequest) -> ToolResult: + result: ToolResult = super().call(tool) + self._invocations.append(NativeInvocation(tool, result)) + return result + + +@dataclass(frozen=True) +class LifecycleRun: + commander: MeasuredCommander + worker: str + decisions: Path + document: Path + artifact: Path + + @classmethod + @contextlib.contextmanager + def open(cls, *, extra_threads: tuple[str, ...] = ()) -> Iterator[LifecycleRun]: + config = MeasuredCommander.read(CommanderConfig.path()) + source = config.comms + actor = ToolCaller.from_actor(os.environ.get("AGENT_COMMS_THREAD")).acting_name( + config.commander + ) + caller = source.registry.require(actor) + if actor == config.commander: + held = source.goals.todos.held_for_commander(config.incarnation) + if not held: + raise ValueError("Lifecycle comparison requires an existing native order holder") + worker = source.registry.require(held[-1].held().owner) + else: + worker = caller + directory = caller.storage_roots.scratch / "commander-parity" / uuid4().hex + threads = tuple(dict.fromkeys((config.commander, worker.name, *extra_threads))) + with CheckRoot(source, threads, directory).open() as copied: + root = copied.root + commander = MeasuredCommander( + commander=config.commander, + root=root, + cli=config.cli, + state=root / "commander-state", + board=root / "board.md", + narrative=root / "narrative.md", + repos=config.repos, + worktrees=root / "projects", + scratch=root / "scratch", + disk=root, + ) + with contextlib.redirect_stdout(io.StringIO()): + RequestAddVerb( + text=( + "Compare current order commands with their real native owners in this " + "disposable copied bus, preserving the source bus and its sessions." + ) + ).run(commander) + decisions = root / "decisions.json" + decisions.write_text(json.dumps([worker.name])) + command = shlex.join( + ( + sys.executable, + "-c", + "import json;from pathlib import Path;" + f"print(len(json.loads(Path({str(decisions)!r}).read_text())))", + ) + ) + order = PendingProblemOrder( + owner=FileLine(str(Path(__file__).resolve()), 1), + check=OrderCheck( + str(Path(__file__).resolve().parents[3]), command, CountCheckResult(0) + ), + edits=( + "Remove the copied workflow decision and verify its native order completes.", + ), + banned=("Do not change the source bus or its saved sessions.",), + report="Report the actual copied goal and native order state.", + run=ApplicationOrderRun( + "Read the copied native goal and confirm its recorded source location " + "and assigned worker." + ), + ) + document = root / "order.json" + document.write_text( + json.dumps( + FieldCodec.encode( + OrderFile( + order, + "The copied workflow decision prevents the native completion check " + "from reaching zero.", + "This is stupid because an unfinished copied decision must not count " + "as delivered work.", + ) + ) + ) + ) + yield cls(commander, worker.name, decisions, document, root / "application.json") + + @staticmethod + def python_environment() -> dict[str, str]: + return {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[2])} + + @classmethod + def invoke(cls, program: str, *arguments: str) -> ChildResult: + return cls.invoke_command( + (sys.executable, "-c", program, *arguments), env=cls.python_environment() + ) + + @staticmethod + def invoke_command( + command: tuple[str, ...], + *, + cwd: str | Path | None = None, + env: dict[str, str] | None = None, + ) -> ChildResult: + return asyncio.run(BoundedRun.run(command, timeout=120, cwd=cwd, env=env)) + + @property + def due(self) -> str: + return time.strftime("%H:%M", time.localtime(time.time() + 3600)) + + def create(self) -> Todo: + self.commander.problem( + str(self.document), self.worker, self.due, f"{Path(__file__).resolve()}:1" + ) + rows = tuple(self.commander.comms.goals.todos.for_commander(self.commander.incarnation)) + if len(rows) != 1: + raise ValueError(f"A fresh copied workflow must create one problem, got {len(rows)}") + return rows[0] + + def complete(self, todo: Todo) -> Todo: + goal = self.commander.comms.registry.require(self.worker).goal + self.artifact.write_text(json.dumps(FieldCodec.encode(goal))) + self.decisions.write_text("[]") + self.commander.done(todo.id, "op", seen=str(self.artifact)) + return self.commander.comms.goals.todos.get(todo.id) + + +class LifecycleCase(ABC): + native_owner: ClassVar[type[LifecycleRequest]] + additional_owners: ClassVar[tuple[type[LifecycleRequest], ...]] = () + + @classmethod + @abstractmethod + def exercise(cls, run: LifecycleRun) -> tuple[Todo, ...]: ... + + @classmethod + @abstractmethod + def checks( + cls, run: LifecycleRun, rows: tuple[Todo, ...], native: tuple[ToolResult, ...], output: str + ) -> tuple[tuple[str, bool], ...]: ... + + @classmethod + def read(cls) -> ParityComparison: + owners = (cls.native_owner, *cls.additional_owners) + with LifecycleRun.open() as run: + output = io.StringIO() + with contextlib.redirect_stdout(output): + rows = cls.exercise(run) + native = tuple( + invocation.result + for invocation in run.commander.invocations + if isinstance(invocation.request, owners) + ) + text = output.getvalue() + return LifecycleObservation( + ", ".join(f"{owner.__module__}.{owner.__qualname__}" for owner in owners), + len(native), + cls.checks(run, rows, native, text), + text, + ) + + +class CreateProblemCase(LifecycleCase): + native_owner: ClassVar[type[ToolRequest[ThreadCaller]]] = CommsTodoCreateTool + + @classmethod + def exercise(cls, run: LifecycleRun) -> tuple[Todo, ...]: + return (run.create(),) + + @classmethod + def checks( + cls, run: LifecycleRun, rows: tuple[Todo, ...], native: tuple[ToolResult, ...], output: str + ) -> tuple[tuple[str, bool], ...]: + if not native or not isinstance(native[-1], ProblemOrdered): + return (("native order result returned", False),) + result, saved = native[-1], rows[-1] + goal = run.commander.comms.registry.require(run.worker).goal + return ( + ("printed native identity", output.rstrip().endswith(cls.printed_identity(result))), + ( + "native source retained", + saved.order.checked().owner == result.todo.order.checked().owner, + ), + ("native assignment retained", saved.assignment == result.todo.assignment), + ("native goal published", goal.require().id == result.goal.id), + ) + + @classmethod + def printed_identity(cls, result: ProblemOrdered) -> str: + return result.todo.id + + +class AssignOrderCase(CreateProblemCase): + native_owner = CommsOrderTool + + @classmethod + def exercise(cls, run: LifecycleRun) -> tuple[Todo, ...]: + before = run.create() + run.commander.fire( + run.worker, "The copied workflow is released before its next native assignment." + ) + document = replace(OrderFile.read(run.document.read_text()), problems=(before.id,)) + run.commander.send(run.worker, run.due, document, clean=True) + return before, run.commander.comms.goals.todos.get(before.id) + + @classmethod + def checks( + cls, run: LifecycleRun, rows: tuple[Todo, ...], native: tuple[ToolResult, ...], output: str + ) -> tuple[tuple[str, bool], ...]: + inherited = super().checks(run, rows, native, output) + return ( + *inherited, + ("same problem reassigned", rows[0].id == rows[-1].id), + ("prior assignment retained", bool(rows[-1].released)), + ) + + @classmethod + def printed_identity(cls, result: ProblemOrdered) -> str: + return result.todo.order.checked().id + + +class InstalledEvidenceCase(LifecycleCase): + native_owner = CommsProblemCloseTool + + @classmethod + def exercise(cls, run: LifecycleRun) -> tuple[Todo, ...]: + created = run.create() + delivered = run.complete(created) + run.commander.close( + created.id, + "Read the copied native goal and confirm its recorded source location " + "and assigned worker.", + str(run.artifact), + ) + return delivered, run.commander.comms.goals.todos.get(created.id) + + @classmethod + def checks( + cls, run: LifecycleRun, rows: tuple[Todo, ...], native: tuple[ToolResult, ...], output: str + ) -> tuple[tuple[str, bool], ...]: + return ( + ("delivery alone unaccepted", not rows[0].order.accepted), + ("installed evidence accepted", rows[-1].order.accepted), + ("closed in native store", not rows[-1].state.open), + ("actual artifact exists", run.artifact.is_file()), + ("current close printed", f"{rows[-1].id} closed" in output), + ) + + +class BuildBoundProofCase(InstalledEvidenceCase): + @classmethod + def read(cls) -> ParityComparison: + closing = super().read() + return UnverifiedParity( + "Missing native owner: a build-bound application-run record linking the " + "runtime and Core/Toad revisions to closing evidence. The actual close " + "owner currently accepts a bare artifact path. " + closing.describe() + ) + + +class StrikeLimitCase(LifecycleCase): + native_owner = CommsChangeProblemTool + additional_owners = (CommsFireTool,) + + @classmethod + def exercise(cls, run: LifecycleRun) -> tuple[Todo, ...]: + created = run.create() + for number in range(ProblemGoalOrder.StrikeLimit): + run.commander.strike( + run.worker, + created.id, + "The copied decision remains unfinished at its recorded review " + f"number {number + 1}.", + ) + run.commander.fire( + run.worker, "The copied holder exhausted its declared live strikes on this problem." + ) + return created, run.commander.comms.goals.todos.get(created.id) + + @classmethod + def checks( + cls, run: LifecycleRun, rows: tuple[Todo, ...], native: tuple[ToolResult, ...], output: str + ) -> tuple[tuple[str, bool], ...]: + saved = rows[-1] + return ( + ("declared limit recorded", saved.strike_count == ProblemGoalOrder.StrikeLimit), + ("one native fire", sum(isinstance(result, TodoList) for result in native) == 1), + ( + "assignment released", + saved.id + not in { + todo.id + for todo in run.commander.comms.goals.todos.held_for_commander( + run.commander.incarnation + ) + }, + ), + ("current strike count printed", f"{ProblemGoalOrder.StrikeLimit} strike(s)" in output), + ) + + +class OrderProgressMutationCase(LifecycleCase): + native_owner = CommsChangeProblemTool + + @classmethod + def exercise(cls, run: LifecycleRun) -> tuple[Todo, ...]: + before = run.create() + moved = time.strftime("%H:%M", time.localtime(time.time() + 7200)) + run.commander.due( + before.id, + moved, + "The copied workflow needs time to record its actual native application evidence.", + ) + return before, run.commander.comms.goals.todos.get(before.id) + + @classmethod + def checks( + cls, run: LifecycleRun, rows: tuple[Todo, ...], native: tuple[ToolResult, ...], output: str + ) -> tuple[tuple[str, bool], ...]: + before, after = rows + messages = run.commander.comms.bus.inbox(run.worker) + unit = run.commander.comms.registry.require(run.worker) + goal = unit.goal.require() + instructions = goal.order.instructions( + goal.order.todos(run.commander.comms.goals, unit.incarnation, goal.id) + ) + native_deadlines = tuple( + result.todo.deadline for result in native if isinstance(result, TodoChanged) + ) + return ( + ( + "native deadline retained", + bool(native_deadlines) and native_deadlines[-1] == after.deadline, + ), + ("deadline actually changed", before.deadline != after.deadline), + ("assignment unchanged", before.assignment == after.assignment), + ("unit got the actual update", bool(messages)), + ( + "unit messages hide clocks", + all( + hhmm(clock) not in message.body + for clock in (before.deadline, after.deadline) + for message in messages + ), + ), + ( + "unit goal hides deadline", + "Deadline:" not in instructions and hhmm(after.deadline) not in instructions, + ), + ("commander still sees deadline", f"due {hhmm(after.deadline)}" in output), + ) + + +@dataclass(frozen=True) +class ProgressComparison(ParityComparison): + current: dict[str, str] + native: dict[str, str] + first_seconds: int + later_seconds: int + merged_no_strikes: tuple[bool, ...] + workflow: ParityComparison + + @property + def passed(self) -> bool: + return all( + ( + self.workflow.passed, + bool(self.native), + self.current == self.native, + self.first_seconds == 600, + self.later_seconds == 900, + bool(self.merged_no_strikes), + all(self.merged_no_strikes), + ) + ) + + def describe(self) -> str: + return ( + f"{self.workflow.describe()}; live_progress={len(self.native)}; " + f"read_equal={self.current == self.native}; " + f"intervals={self.first_seconds}/{self.later_seconds}; " + f"live_merged_no_strikes={self.merged_no_strikes}" + ) + + +def verify_order_progress() -> ParityComparison: + context = MeasuredCommander.read(CommanderConfig.path()) + current = {key: value.line for key, value in context.progress().items()} + results = tuple( + invocation.result + for invocation in context.invocations + if isinstance(invocation.request, CommsTodoListTool) + ) + if len(results) != 1 or not isinstance(results[0], TodoList): + return UnverifiedParity("Current progress did not return one native TodoList measurement") + result = results[0] + observed = {key: value.line for key, value in result.progress.items()} + todos = {todo.id: todo for todo in result.todos} + merged = tuple( + not progress.stalled(todos[pid].order.checked().at.recorded_at("An order"), time.time()) + for pid, progress in result.progress.items() + if isinstance(progress, MergedProgress) + ) + return ProgressComparison( + current, + observed, + NoCommitProgress.seconds, + Moving.seconds, + merged, + OrderProgressMutationCase.read(), + ) diff --git a/src/agent_comms/commander_parity/rows.py b/src/agent_comms/commander_parity/rows.py index ab22a2feb2..39c8618a94 100644 --- a/src/agent_comms/commander_parity/rows.py +++ b/src/agent_comms/commander_parity/rows.py @@ -11,6 +11,15 @@ StoryPhase, ) from . import ParityComparison, ParityRow, UnverifiedParity +from .handoff import FinishedHandoffCase +from .lifecycle import ( + AssignOrderCase, + BuildBoundProofCase, + CreateProblemCase, + InstalledEvidenceCase, + StrikeLimitCase, + verify_order_progress, +) from .load import LoadComparison from .measurements import RuleComparison, StorageComparison, measured_load @@ -22,6 +31,8 @@ class CreateProblemParityRow(ParityRow): native = "`CommsTodoCreateTool`" phases = (OneOrderOwnerPhase,) + comparison = CreateProblemCase.read + class AssignOrderParityRow(ParityRow): number = 2 @@ -30,6 +41,8 @@ class AssignOrderParityRow(ParityRow): native = "`CommsOrderTool`, `CommsTodoTool` assignment and release" phases = (OneOrderOwnerPhase,) + comparison = AssignOrderCase.read + class DeliveredCommitParityRow(ParityRow): number = 3 @@ -46,6 +59,8 @@ class InstalledEvidenceParityRow(ParityRow): native = "`CommsProblemCloseTool`" phases = (OneOrderOwnerPhase,) + comparison = InstalledEvidenceCase.read + class BoardParityRow(ParityRow): number = 5 @@ -70,6 +85,8 @@ class StrikeLimitParityRow(ParityRow): native = "`Strike` rows, author-declared `StrikeLimit`, one `comms_fire`" phases = (RecordPlanePhase,) + comparison = StrikeLimitCase.read + class VerdictParityRow(ParityRow): number = 8 @@ -270,6 +287,8 @@ class FinishedWorkTransferParityRow(ParityRow): native = "Transfer refuses delivered checked work; that owner merges; check is reused" phases = (RecordPlanePhase,) + comparison = FinishedHandoffCase.read + class BuildBoundProofParityRow(ParityRow): number = 30 @@ -278,13 +297,7 @@ class BuildBoundProofParityRow(ParityRow): native = "Build run records runtime and Core/Toad revisions; close requires delivered commit" phases = (RecordPlanePhase,) - -class DeletionTargetParityRow(ParityRow): - number = 31 - capability = "An order states its intent as a deletion target" - current = "none; P520 renamed a decision rather than deleting it" - native = "Order owns deletion family/site/count; delivery refuses nonzero or renamed decisions" - phases = (RecordPlanePhase,) + comparison = BuildBoundProofCase.read class UnitTimeParityRow(ParityRow): @@ -312,6 +325,8 @@ class OrderProgressParityRow(ParityRow): ) phases = (OneOrderOwnerPhase, ControlPlanePhase) + comparison = staticmethod(verify_order_progress) + class MultipleCommandersParityRow(ParityRow): number = 34 diff --git a/src/agent_comms/goals.py b/src/agent_comms/goals.py index 5cd5f236cf..be2daa6035 100644 --- a/src/agent_comms/goals.py +++ b/src/agent_comms/goals.py @@ -331,12 +331,13 @@ def recorded(self, sha: str, repository: str) -> tuple[ChildResult, ...]: return tuple(checked.result for checked in CheckedCommit.stored(repository, sha) if checked.command == self.command)[:1] - def at(self, revision: str, repository: str) -> ChildResult: + def at(self, revision: str, repository: str, checkout_base: Path) -> ChildResult: """Read the result on ``revision``, or explicitly run the check in a detached checkout.""" sha = CheckedCommit.git(repository, "rev-parse", f"{revision}^{{commit}}").strip() for recorded in self.recorded(sha, repository): return recorded - with tempfile.TemporaryDirectory(prefix="order-check-") as scratch: + checkout_base.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="order-check-", dir=checkout_base) as scratch: tree = str(Path(scratch) / "tree") CheckedCommit.git(repository, "worktree", "add", "--detach", tree, sha) try: diff --git a/src/agent_comms/todos.py b/src/agent_comms/todos.py index 896e0be57f..35885bbe0e 100644 --- a/src/agent_comms/todos.py +++ b/src/agent_comms/todos.py @@ -98,14 +98,14 @@ async def verified(self, todo: Todo, registry: Registration) -> OrderDelivery: check = todo.order.check for each in (check, *todo.order.deletes.checks): await self.require_source(each) - each.result.require(await self.checked(each)) + each.result.require(await self.checked(each, registry.storage_rule.checkout_base)) return replace(self, at=await self.delivery_time(check)) @abstractmethod async def require_source(self, check): ... @abstractmethod - async def checked(self, check: OrderCheck) -> ChildResult: ... + async def checked(self, check: OrderCheck, checkout_base: Path) -> ChildResult: ... @abstractmethod async def delivery_time(self, check): ... @@ -140,8 +140,8 @@ def work(self) -> str: async def require_source(self, check: OrderCheck) -> None: raise ValueError("A commit delivery names the unit's delivered commit and the checked one.") - async def checked(self, check: OrderCheck) -> ChildResult: - return await asyncio.to_thread(check.at, self.commit, check.directory) + async def checked(self, check: OrderCheck, checkout_base: Path) -> ChildResult: + return await asyncio.to_thread(check.at, self.commit, check.directory, checkout_base) async def delivery_time(self, check: OrderCheck) -> EventTime: return EventTime.capture(float( @@ -281,7 +281,7 @@ def line(self) -> str: async def require_source(self, check): pass - async def checked(self, check: OrderCheck) -> ChildResult: + async def checked(self, check: OrderCheck, checkout_base: Path) -> ChildResult: return await check.run() async def delivery_time(self, check): @@ -727,6 +727,7 @@ def stalled_between(self, ordered: float, since: float, now: float) -> bool: @dataclass(frozen=True) class Moving(Stalling): """Work whose last commit, push or PR change was observed at ``at``.""" + seconds: ClassVar[int] = 15 * 60 at: float = field(kw_only=True) readings: DistanceReadings = field(default=DistanceReadings(), kw_only=True, compare=False) @@ -1513,7 +1514,7 @@ def work_instructions(self) -> tuple[str, ...]: def instructions(self, todo): return "\n".join((f"Problem: {todo.id}: {todo.text}", *self.finding.instructions(), f"Order: {self.id}", self.pull_request_instruction(todo.id), - *self.work_instructions(), f"Deadline: {FieldCodec.encode(todo.deadline)}")) + *self.work_instructions())) def delivered(self, delivery): delivery.at.require_delivery() diff --git a/src/agent_comms/tools.py b/src/agent_comms/tools.py index cf39fb398f..673bbf334a 100644 --- a/src/agent_comms/tools.py +++ b/src/agent_comms/tools.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import sys import threading from abc import abstractmethod from collections.abc import Mapping @@ -12,6 +13,7 @@ from pydantic import JsonValue +from . import cli as comms_cli from .commander.capacity import Load from .progress_claims import ProgressClaim from .pi_vocabulary import ServiceTier, ThinkingLevel, ThinkingLevelField @@ -68,7 +70,7 @@ TodoOperation, VerdictAudience) from .change_report import ChangeReport from .messages import InfoKind, MessageKind, NoticeRole, SendableMessageKind -from .native_tools import NativeToolDeclaration, NativeRequestTools +from .native_tools import ArgumentTool, NativeToolDeclaration, NativeRequestTools from .tool_caller import ToolCaller, ThreadCaller, NoThreadCaller, ThreadListing from .thread_identity import ThreadIncarnation from .cli_result import ToolCatalogResult @@ -292,7 +294,7 @@ def tool_field( @dataclass(frozen=True, kw_only=True) class ToolRequest[Caller: ToolCaller = ToolCaller]( - ThreadCommandAvailability, NativeRequestTools, NativeToolDeclaration + ArgumentTool, ThreadCommandAvailability, NativeRequestTools, NativeToolDeclaration ): description: ClassVar[str] bus_effect: ClassVar[str] = ("Changes the bus. Tristan has authorized claude-ai " @@ -328,6 +330,11 @@ def schema(cls) -> dict[str, object]: ) return result + def cli_command(self, caller: ToolCaller) -> InvokeCliCommand: + """Project this declaration into Core's existing external tool command.""" + return InvokeCliCommand(caller=caller.acting_name(''), + tool=self.declared_name, arguments=self.arguments) + @classmethod def invoke( cls, comms: Comms, arguments: Mapping[str, object], *, caller: ToolCaller @@ -1362,6 +1369,13 @@ class InvokeCliCommand(CliCommand, declared_name="tool"): "--arguments", help="Tool arguments as a JSON object", default_factory=dict, parser_default="{}") + def argv(self, root: Path) -> tuple[str, ...]: + """The CLI boundary owns its module, grammar and argument serialization.""" + caller = ToolCaller.from_actor(self.caller) + return (sys.executable, '-m', comms_cli.__name__, '--root', str(root), + self.declared_name, '--from', caller.acting_name(''), '--tool', self.tool, + '--arguments', json.dumps(self.arguments)) + def apply(self, ctx: Comms) -> dict[str, JsonValue]: return invoke_tool(ctx, self.tool, self.arguments, caller=ToolCaller.from_actor(self.caller)) diff --git a/tests/test_candidate_maintenance.py b/tests/test_candidate_maintenance.py index 645154eaf8..2bde1e99f0 100644 --- a/tests/test_candidate_maintenance.py +++ b/tests/test_candidate_maintenance.py @@ -53,15 +53,15 @@ def test_ordinary_private_send_deferred_index_catches_up_without_wake_authority( _SCHEDULE, ) indexed = threading.Event() - original = maintenance._drain_candidate + original = maintenance.CandidateMaintenance.run - def observe(root): - original(root) + def observe(worker): + original(worker) # Candidate catch-up alone is not completion: the same worker # still owns route preparation and its final pending-root check. indexed.set() - monkeypatch.setattr(maintenance, "_drain_candidate", observe) + monkeypatch.setattr(maintenance.CandidateMaintenance, "run", observe) message = comms.messaging.send("sender", "beta", "selected original").message assert indexed.wait(timeout=5), "deferred checkpoint did not catch up" page = WakeCandidateIndex(comms.bus.log).page( @@ -111,14 +111,15 @@ def probe(): def test_failed_drain_is_named_and_retires_pending_state(monkeypatch, tmp_path, caplog) -> None: comms, _root_id = _wire(tmp_path) root = comms.bus.log.path.parent - maintenance._pending[root] = (comms.bus.log, 1) + worker = maintenance.CandidateMaintenance(comms.bus.log, 1) + maintenance._pending[root] = worker def fail(*args, **kwargs): raise ValueError("synthetic index failure") monkeypatch.setattr(WakeCandidateIndex, "catch_up_committed_append", fail) with pytest.raises(maintenance.CandidateMaintenanceFailed) as failed: - maintenance._drain_candidate(root) + worker.run() assert failed.value.root == root assert root not in maintenance._pending assert "Candidate maintenance failed for" in caplog.text @@ -152,8 +153,9 @@ def test_one_background_drain_catches_up_a_backlog_of_oversized_records( ).checkpoint_bytes == first_bytes through_bytes = index.log.path.stat().st_size hint = index.notify_committed_append(root_id=root_id, through_bytes=through_bytes) - maintenance._pending[index.log.path.parent] = (index.log, through_bytes) - maintenance._drain_candidate(comms.bus.log.path.parent) + worker = maintenance.CandidateMaintenance(index.log, through_bytes) + maintenance._pending[index.log.path.parent] = worker + worker.run() assert not maintenance._pending assert index.catch_up_committed_append(hint).caught_up page = index.page( @@ -172,7 +174,7 @@ def test_source_replacement_during_replay_does_not_kill_the_worker(monkeypatch, replay = WakeCandidateIndex._replay_prefix replaced, finished = threading.Event(), threading.Condition() failures = [] - drain = maintenance._drain_candidate + drain = maintenance.CandidateMaintenance.run def replace_once(cls, source, *args, **kwargs): batch = replay(source, *args, **kwargs) @@ -185,9 +187,9 @@ def replace_once(cls, source, *args, **kwargs): replaced.set() return batch - def observed(path): + def observed(worker): try: - drain(path) + drain(worker) except Exception as error: failures.append(error) finally: @@ -199,7 +201,7 @@ def retired(): return root not in maintenance._pending monkeypatch.setattr(WakeCandidateIndex, "_replay_prefix", classmethod(replace_once)) - monkeypatch.setattr(maintenance, "_drain_candidate", observed) + monkeypatch.setattr(maintenance.CandidateMaintenance, "run", observed) monkeypatch.setattr(maintenance, "schedule_candidate_catchup", _SCHEDULE) comms.messaging.send("sender", "beta", "first original") assert replaced.wait(5) @@ -215,11 +217,11 @@ def test_concurrent_ordinary_publication_keeps_maintenance_alive(monkeypatch, tm root = comms.bus.log.path.parent finished = threading.Condition() failures = [] - drain = maintenance._drain_candidate + drain = maintenance.CandidateMaintenance.run - def observed_drain(path): + def observed_drain(worker): try: - drain(path) + drain(worker) except Exception as error: failures.append(error) raise @@ -231,7 +233,7 @@ def retired(): with maintenance._guard: return root not in maintenance._pending - monkeypatch.setattr(maintenance, "_drain_candidate", observed_drain) + monkeypatch.setattr(maintenance.CandidateMaintenance, "run", observed_drain) monkeypatch.setattr(maintenance, "schedule_candidate_catchup", _SCHEDULE) messages = [comms.messaging.send("sender", "beta", str(n) + "x" * (128 * 1024)).message for n in range(128)] diff --git a/tests/test_candidate_publication_notifications.py b/tests/test_candidate_publication_notifications.py index 7c761dda2d..39d1a8a15a 100644 --- a/tests/test_candidate_publication_notifications.py +++ b/tests/test_candidate_publication_notifications.py @@ -65,11 +65,11 @@ def test_nested_release_scope_runs_only_after_outer_release(): def notifications(monkeypatch): def activate(comms): calls, finished = [], threading.Event() - drain = candidate_maintenance._drain_candidate + drain = candidate_maintenance.CandidateMaintenance.run - def complete(root): + def complete(worker): try: - drain(root) + drain(worker) finally: finished.set() @@ -94,7 +94,7 @@ def probe(): calls.append(through_bytes) _SCHEDULE(log, through_bytes) - monkeypatch.setattr(candidate_maintenance, "_drain_candidate", complete) + monkeypatch.setattr(candidate_maintenance.CandidateMaintenance, "run", complete) monkeypatch.setattr(candidate_maintenance, "schedule_candidate_catchup", notify) return calls, finished @@ -250,7 +250,8 @@ def test_retiring_worker_does_not_delete_successor_notification(tmp_path, monkey publish_initial(comms, "sender", "member000", "first") assert index.maintain(start=RebuildIndex()) root = index.log.path.parent - candidate_maintenance._pending[root] = (index.log, index.log.path.stat().st_size) + worker = candidate_maintenance.CandidateMaintenance(index.log, index.log.path.stat().st_size) + candidate_maintenance._pending[root] = worker original_guard = candidate_maintenance._guard successor = [] @@ -269,11 +270,11 @@ def __exit__(self, *_): monkeypatch.setattr(candidate_maintenance, "_guard", InterleavedLock()) monkeypatch.setattr(candidate_maintenance, "schedule_candidate_catchup", _SCHEDULE) monkeypatch.setattr(threading.Thread, "start", lambda _: None) - candidate_maintenance._drain_candidate(root) + worker.run() assert successor - assert candidate_maintenance._pending[root][1] == index.log.path.stat().st_size + assert candidate_maintenance._pending[root].through_bytes == index.log.path.stat().st_size # Complete the retained successor without launching another daemon. - candidate_maintenance._drain_candidate(root) + candidate_maintenance._pending[root].run() assert root not in candidate_maintenance._pending assert index._verified_checkpoint(root_id).last_seq == 2 finally: diff --git a/tests/test_order_progress.py b/tests/test_order_progress.py index da1e20fce6..25a51e2ab8 100644 --- a/tests/test_order_progress.py +++ b/tests/test_order_progress.py @@ -115,7 +115,7 @@ def test_uncounted_check_keeps_other_overdue_orders_visible(commander, commander replace(todo, order=broken).upsert(db) create(commander, 'P2', deadline=1) commit(unit) - broken.check.at(git(unit, 'rev-parse', 'HEAD'), str(unit)) + broken.check.at(git(unit, 'rev-parse', 'HEAD'), str(unit), unit.parent) progress = read((comms.goals.todos.get('P1'), unit)) assert progress.line.endswith("; check prints no integer: printf 'no count here'") @@ -213,7 +213,7 @@ def test_progress_reads_only_explicit_check_receipts(checkout, tmp_path): source = ProgressSource.of(todo, StorageRule(checkout_base=unit.parent), Listed()) assert not source.readings().readings assert not marker.exists() - check.at(git(unit, 'rev-parse', 'HEAD').strip(), str(unit)) + check.at(git(unit, 'rev-parse', 'HEAD').strip(), str(unit), unit.parent) assert marker.read_text() == 'x' assert len(source.readings().readings) == 1 assert source.readings().readings[0].distance == CountPrinted(todo.order.checked().untouched(())) diff --git a/tests/test_standing_rules.py b/tests/test_standing_rules.py index 4053c77e2d..9f64a92ac3 100644 --- a/tests/test_standing_rules.py +++ b/tests/test_standing_rules.py @@ -206,17 +206,17 @@ def commit(left): check = f"echo run >> {runs}; cat left" todo = create(commander, "P1", command=check, expected=0, deadline=time.time() + 3600) ordered_check = todo.order.checked().check - ordered_check.at(git("rev-parse", "HEAD").strip(), str(unit)) + ordered_check.at(git("rev-parse", "HEAD").strip(), str(unit), unit.parent) def progress(): return OrderProgress.read(comms.goals.todos.get("P1"), storage, PullRequests()) commit(2) - ordered_check.at(git("rev-parse", "HEAD").strip(), str(unit)) + ordered_check.at(git("rev-parse", "HEAD").strip(), str(unit), unit.parent) assert "; distance 2, projected done " in progress().line assert isinstance(progress().distance_readings().trend, MovingTrend) commit(2) - ordered_check.at(git("rev-parse", "HEAD").strip(), str(unit)) + ordered_check.at(git("rev-parse", "HEAD").strip(), str(unit), unit.parent) assert progress().line.endswith("; distance 2, not falling") assert runs.read_text().count("run") == 3 # once per commit: base, then each branch commit diff --git a/tools/proofs/cn_phase3_record.py b/tools/proofs/cn_phase3_record.py index d925660ffd..bbadb40bd5 100644 --- a/tools/proofs/cn_phase3_record.py +++ b/tools/proofs/cn_phase3_record.py @@ -7,19 +7,15 @@ from __future__ import annotations import argparse -import json import os import shutil import subprocess -import sys import time from dataclasses import replace from pathlib import Path -from agent_comms import cli as comms_cli from agent_comms.change_report import FileLine from agent_comms.comms import Comms -from agent_comms.field_codec import FieldCodec from agent_comms.goal_actions import CreateProblemGoalAction, SetGoalAction from agent_comms.goals import CountCheckResult, OrderCheck, RequestGoalOrder from agent_comms.progress_claims import MergedClaim @@ -50,22 +46,7 @@ def cli[Caller: ToolCaller](caller: str, tool: ToolRequest[Caller]) -> str: - arguments = FieldCodec.encode(tool) - del arguments[tool.family_discriminator] - command = [ - sys.executable, - "-m", - comms_cli.__name__, - "--root", - str(ROOT), - "tool", - "--from", - caller, - "--tool", - tool.declared_name, - "--arguments", - json.dumps(arguments), - ] + command = tool.cli_command(ToolCaller.from_actor(caller)).argv(ROOT) result = subprocess.run( command, capture_output=True,