From 97e7e484aef2e540d8afc53e07508b0b9a834f2a Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 16:55:42 -0800 Subject: [PATCH] pa_lab: leased run_program and stop_program (Slice 5a) Served only by serve --actions; the read-only stdio mode is unchanged. Each call is one request record: lease and app checks in one transaction, the app's start/stop channel command with the request id, and the outcome correlated from Lab/state.json (by request id, or by time when the run ended first). run_program's preflight_token is optional until 5b. --- lab/server/GUIDE.md | 15 +- lab/server/README.md | 68 +++- lab/server/contracts/v1/invalid_cases.json | 7 +- lab/server/pa_lab/__main__.py | 17 +- lab/server/pa_lab/actions.py | 298 ++++++++++++++ lab/server/pa_lab/contracts.py | 5 +- lab/server/pa_lab/control.py | 134 +++++- lab/server/pa_lab/db.py | 15 + lab/server/pa_lab/mcp_server.py | 82 +++- lab/server/tests/fake_lab.py | 51 +++ lab/server/tests/test_actions.py | 448 +++++++++++++++++++++ 11 files changed, 1106 insertions(+), 34 deletions(-) create mode 100644 lab/server/pa_lab/actions.py create mode 100644 lab/server/tests/test_actions.py diff --git a/lab/server/GUIDE.md b/lab/server/GUIDE.md index 6cc720690b..bf29805961 100644 --- a/lab/server/GUIDE.md +++ b/lab/server/GUIDE.md @@ -5,8 +5,8 @@ SerialPrograms lab app on profile `lab-B`, and its log) through MCP, the protoco uses for tools. Instead of an agent running shell scripts and grepping logs to work out what the console is doing, it asks pa-lab and gets one structured answer. -Today it only reads. Nothing in it presses a button, starts or stops a program, or writes to -the app. Console A is never part of it. +Today, as connected to your sessions, it only reads: nothing there presses a button, starts or +stops a program, or writes to the app. Console A is never part of it. Developer details (contracts, slices, tests) are in [README.md](README.md); the design is `docs/design/Agent-Integration.md` on branch `design/agent-integration`. @@ -53,12 +53,21 @@ game in front, or HOME / user select / black screen), `shot` (a fresh screenshot `ensure_game`. The control plane is also built: console leases, request records that make a retried request safe, and an event log. None of it presses buttons yet. +**Starting and stopping a program (Slice 5a).** `run_program` and `stop_program` exist, but only +when the server is started with `serve --actions`, which nothing uses yet; your session connection +stays read-only. An agent holding the console's lease can start the program selected in the app +(with its current options) and stop it. It is refused whenever you hold the console (you clicked +Start or Stop), when another program is running, when the app is not running, or when the +selected program is not the one asked for. If you click Start or Stop while an agent's start or +stop is under way, yours wins and the agent's request is marked failed. Every start and stop is +one record, and a retried request never starts the program twice. + ## What comes next | Slice | What you would get | |---|---| | 4 (rest) | FRLG AutoStory's start checks listed and run before any press (fork #488) | -| 5 | Leased actions: an agent asks to run or stop a program; the server checks the program's start checks first and refuses on a failed one, and refuses whenever you hold the console. Every action gets exactly one record. | +| 5 (rest) | The server runs the program's start checks before a start and refuses on a failed one (`preflight_program`), passes options, and runs debug button batches (`cfr_batch`); then a soak on console B | | 6 | Save facts: the server knows which save is on the console and refuses a run that would start from the wrong one; run records become readable | | 7 | The server launches and supervises the lab app itself, so a build always starts with the right profile | | 8 | Optional: a local worker for routine operations | diff --git a/lab/server/README.md b/lab/server/README.md index 111f461e8d..cf5ef31fe5 100644 --- a/lab/server/README.md +++ b/lab/server/README.md @@ -4,7 +4,9 @@ The local server designed in `docs/design/Agent-Integration.md` (fork #131, bran `design/agent-integration`). Slice 0: the version-1 contracts, golden examples and a parser. Slice 2: the attach-only read server ([below](#slice-2-the-attach-only-read-server)). Slice 3: the durable control plane ([below](#slice-3-the-durable-control-plane)): leases, request records, -tickets and events in SQLite. Nothing presses a button yet: that is Slice 5. +tickets and events in SQLite. Slice 5a: leased `run_program` and `stop_program` +([below](#slice-5a-run_program-and-stop_program)), served only by `serve --actions`, which nothing is +wired to yet. Nothing presses a button: that is Slice 5b's `cfr_batch`. New here? Start with [GUIDE.md](GUIDE.md): what pa-lab is and what it gives you today. @@ -22,12 +24,14 @@ New here? Start with [GUIDE.md](GUIDE.md): what pa-lab is and what it gives you | `pa_lab/tools.py`, `pa_lab/shot.py` | `ensure_game` (verification only) and `shot` (newer than the call) | | `pa_lab/mcp_server.py` | MCP over Streamable HTTP on `127.0.0.1`: bearer, Host/Origin, sessions, size caps | | `pa_lab/app_query.py` | `describe_program` and `game_open`: one-line reads through the app's channel (#504) | -| `tests/fake_lab.py` | A fake `Lab/` folder and a fake app answering `snapshot`, `describe_program` and `game_open` requests, in temp dirs | +| `tests/fake_lab.py` | A fake `Lab/` folder and a fake app answering `snapshot`, `describe_program`, `game_open`, `start` and `stop` requests, in temp dirs | | `tests/test_{lab_reader,ensure_game,shot,config,mcp_server}.py` | Slice 2's fake-lab tests | | `tests/test_app_query.py` | `describe_program` and `game_open` against the fake app | | `pa_lab/db.py` | The SQLite file: migrations (`PRAGMA user_version`), owner-only file, `BEGIN IMMEDIATE` writes | | `pa_lab/control.py` | Leases, idempotent requests, tickets, events, `next_events`, reconciliation, compaction | | `tests/test_{db,control,control_concurrency,control_mcp}.py` | Slice 3's tests: restarts, threads and processes racing, kills mid-write | +| `pa_lab/actions.py` | Slice 5a: `run_program` / `stop_program` through the control plane and the app's `start` / `stop` channel commands | +| `tests/test_actions.py` | Slice 5a's refusals, retries and races against the fake app | | `pa_lab/status.py`, `pa_lab/stdio_server.py`, `smoke_readonly.py` | Read-only stdio mode for `.mcp.json`: `console_status` ([below](#read-only-stdio-mode-mcpjson)) | | `tests/test_status_readonly.py` | `console_status` and the read-only gate against a fake profile | @@ -293,3 +297,63 @@ every event after an active cursor (one used in the last 7 days), every event of and every pending record; finished records go after 7 days. A cursor behind the compacted range is refused (`invalid_arguments`, field `after_seq`, `evidence.compacted_through`), never served with a silent gap. + +## Slice 5a: `run_program` and `stop_program` + +`python3 -m pa_lab serve --config ... --actions` adds the two tools to the HTTP server (and +`ACTION_INSTRUCTIONS` to `initialize`). Without `--actions` they are neither listed nor callable; +`--read-only` and `serve-stdio` (the `.mcp.json` entry) never have them, and `--read-only --actions` +is a usage error. + +```text +run_program(console, request_id, lease_token, panel, options={}) # options must be {} in 5a +stop_program(console, request_id, lease_token) +``` + +Each call is one request record (`ControlPlane.begin_action`, one transaction): an exact retry gets +the stored record, refusals included, and never sends a second line; the same `request_id` with +other arguments is `request_id_reused` (not stored). The checks run in this order; a failed one is +stored as a `refused` record with a `request_failed` event, and nothing reaches `Lab/inbox`. + +| Check | Refusal | +|---|---| +| the request's token holds the lease (idle or running) | `lease_invalid`, `lease_expired`, `console_held_by_user` (a UI Start/Stop or the operator) | +| no earlier run/stop on the console is still pending | `program_not_stopped`, `evidence.pending_tool` | +| `Lab/state.json` is live | `app_unavailable`, `evidence.reason` = the availability reason | +| run: `options` is `{}`, no `preflight_token` | `invalid_arguments` (`options`, `preflight_token`) | +| run: `panel` is the selected program | `invalid_arguments`, field `panel`, `evidence.selected`; `panel_not_program` on the settings page | +| run: `console://B/state`'s `blocked` (program state, controller, video) | `program_running`, `program_not_stopped` (loading), `controller_not_ready`, `video_stale`, `video_bad_signal` | +| the channel's owner lock is not claimed (`channel.owner_claimed`) | `console_held_by_agent`, `evidence.reason: channel_owned` | +| stop: a program is running | not a refusal: `completed` with `sent: false` and the program state | + +A passing request is stored `pending` with the app instance it was checked against (table +`actions`, migration 2), and the daemon writes `request ` / `start 10` (or `stop`) +into `Lab/inbox`, the commands `labctl.sh start|stop` sends; the client's `request_id` is the +channel's request id. It then reads the journal and `state.json` (up to 13 s for a start, 10 s for +a stop) and resolves the record: + +| The app shows | Record | +|---|---| +| `last_transition` = this request's start | `completed`, `correlated_by: request_id`; the lease enters `agent_running` in the same transaction, unless `observe` already applied a newer transition | +| `last_transition` = this request's stop, `program.state` `stopped` | `completed`, `stopped_by: lab` | +| after the start's `ok`, a stop by `program` / `error` dated after the request | `completed` with `ended: {source, at}`, `correlated_by: time`; the lease stays idle | +| (stop) a stop by `program` / `error` dated after the request | `completed`, `stopped_by` that source, `correlated_by: time` (the terminal event completes it) | +| a UI transition dated after the request | `failed`, `console_held_by_user` (the user lease is already installed) | +| another `instance.id` without this request's id | `failed`, `app_unavailable`, `instance_replaced` | +| the line failed | `failed`: `owned by` → `console_held_by_agent` (the token is not copied); `Program is RUNNING.` → `program_running`; `controller not connected` → `controller_not_ready`; else `app_unavailable` with the journal | +| no answer, request still in the inbox | withdrawn; `failed`, `timeout` (nothing was sent) | +| no answer, request taken | stays `pending`; settled from `state.json` on the next `request_status`, `next_events`, `console://B/state` read or action call, also after a daemon restart; `failed` / `app_unavailable` if the app exits | + +**What Slice 1's data does not give.** `last_transition` keeps only the newest start or stop, and +`run_id` is null until run records exist (#116). So a run that ends before the daemon reads +`state.json` is matched by time (a `program`/`error` stop dated after the request, on the same +instance, after the start's `ok` line), not by id, and its `started_at` is unknown. A `run_id` on +the stop transition, or the app keeping the start beside the stop, would make that match exact. + +**Left for Slice 5b.** `preflight_program`: the app has no channel command that runs a program's +prechecks (or stages candidate options) without starting it, so 5a sends no preflight, runs the +panel's current options, and the program's own `run_prechecks()` at Start are the only checks; +`run_program`'s `preflight_token` became optional in the v1 request contract for this and 5b makes +it required again. Also 5b: staging `options`, claiming the channel's owner token (a claimed +channel is refused today), `cfr_batch`, and the console-B soak (every mutation has exactly one +record). diff --git a/lab/server/contracts/v1/invalid_cases.json b/lab/server/contracts/v1/invalid_cases.json index 1ce2c282a1..acb79d83c2 100644 --- a/lab/server/contracts/v1/invalid_cases.json +++ b/lab/server/contracts/v1/invalid_cases.json @@ -1911,7 +1911,7 @@ } }, { - "name": "run without preflight token", + "name": "run with a preflight token that is not a string", "kind": "request", "input": { "json": { @@ -1927,13 +1927,14 @@ "START_CHECKPOINT": "13.01", "END_CHECKPOINT": "13.02", "SAVE_MODE": "none" - } + }, + "preflight_token": 5 } } }, "expectation": { "path": "arguments.preflight_token", - "contains": "required field is missing" + "contains": "expected a string" } }, { diff --git a/lab/server/pa_lab/__main__.py b/lab/server/pa_lab/__main__.py index 725f098dcd..a6761fadcd 100644 --- a/lab/server/pa_lab/__main__.py +++ b/lab/server/pa_lab/__main__.py @@ -4,7 +4,7 @@ python3 -m pa_lab describe app_state python3 -m pa_lab kinds python3 -m pa_lab init-token ~/.config/pa-lab/token - python3 -m pa_lab serve --config ~/.config/pa-lab/server.json [--read-only] + python3 -m pa_lab serve --config ~/.config/pa-lab/server.json [--read-only | --actions] python3 -m pa_lab serve-stdio --profile-dir ~/Library/Application\\ Support/SerialPrograms/Profiles/lab-B python3 -m pa_lab lease --config ~/.config/pa-lab/server.json show|claim-user|release-user @@ -35,6 +35,8 @@ def main(argv=None) -> int: serve.add_argument('--config', required=True) serve.add_argument('--read-only', action='store_true', help='serve only console_status and the resources: no inbox tools, no control plane') + serve.add_argument('--actions', action='store_true', + help='also serve run_program and stop_program (Slice 5a): they start and stop the program') stdio = sub.add_parser('serve-stdio', help='run the read-only MCP server on stdio (for .mcp.json)') stdio.add_argument('--profile-dir', required=True, help="the console's SerialPrograms profile folder") stdio.add_argument('--keepawake-log', default='~/BDSPLab/keepawake.log') @@ -77,7 +79,10 @@ def main(argv=None) -> int: print('wrote {}'.format(args.path)) return 0 if args.command == 'serve': - return _serve(args.config, args.read_only) + if args.read_only and args.actions: + print('error --read-only and --actions exclude each other') + return 2 + return _serve(args.config, args.read_only, args.actions) if args.command == 'serve-stdio': from pa_lab import stdio_server return stdio_server.main(args.profile_dir, args.keepawake_log) @@ -87,7 +92,7 @@ def main(argv=None) -> int: return 2 -def _serve(path: str, read_only: bool = False) -> int: +def _serve(path: str, read_only: bool = False, actions: bool = False) -> int: from pa_lab import lab_reader, mcp_server try: cfg = config.load(path) @@ -112,7 +117,11 @@ def _serve(path: str, read_only: bool = False) -> int: for reader in readers.values(): # restart reconciliation: apply what the app shows now reader.console_state() plane.compact() - server = mcp_server.make_server(cfg.port, token, readers, control_plane=plane) + program_actions = None + if actions: + from pa_lab import actions as actions_module + program_actions = actions_module.ProgramActions(plane, readers) + server = mcp_server.make_server(cfg.port, token, readers, control_plane=plane, actions=program_actions) return _serve_forever(server, readers, mcp_server.ENDPOINT) diff --git a/lab/server/pa_lab/actions.py b/lab/server/pa_lab/actions.py new file mode 100644 index 0000000000..f7e6a628f5 --- /dev/null +++ b/lab/server/pa_lab/actions.py @@ -0,0 +1,298 @@ +"""`run_program` and `stop_program`: leased actions on the console (Slice 5a of fork #131). + +Each call is one request record in the control plane (`pa_lab.control`), keyed by +`(client_id, tool, request_id)`, so a retried call never sends a second start or stop: + +1. Read `console://B/state` (that applies any UI takeover, run end or app restart first). +2. `ControlPlane.begin_action`, one transaction: an exact retry gets its stored record; otherwise + the request's lease token must hold the console and the checks below must pass, or the refusal + is stored. A passing request is stored `pending` with the app instance it was checked against. +3. Write the fixed request `request ` / `start 10` (or `stop`) into `Lab/inbox`, the + same channel commands `labctl.sh start|stop` sends (Lab.md "Channel"). The app records the + request id in `program.last_transition`. +4. Settle: read the journal and `Lab/state.json` until the outcome is known, then resolve the + record (`completed` or `failed`) and emit its `request_completed` / `request_failed` event. + +How an outcome is recognised, from what Slice 1 writes (`last_transition` holds only the newest +start or stop): + +- **By request id.** `last_transition` is `{action, source: lab, request_id: ours}`: our start + (the lease enters `agent_running`) or our stop (completed once `program.state` is `stopped`). +- **By time** (the app does not keep the start once a stop replaces it, and `run_id` is still + null). After our start's `ok` line, a stop by `program` or `error` dated after our request means + the run started and already ended: `completed` with `ended` set, `correlated_by: time`, and the + lease stays idle. For `stop_program` the same stop is the terminal event that completes it. +- A UI transition dated after our request is the user taking over: `failed`, + `console_held_by_user` (the control plane has already installed the user lease). +- A new `instance.id` is an app restart: `failed`, `app_unavailable` / `instance_replaced`, + unless the new instance shows our request id. + +A request the app has not answered in time stays `pending` (a ticket) if the app took it from the +inbox, since it may still act; it is settled on the next `request_status`, `next_events`, +`console://B/state` read or action call. If the app never took it, it is withdrawn from the inbox +and `failed` with `timeout`. + +Not in 5a: `preflight_program` (the app has no command that runs a program's prechecks without +starting it, so the program's own prechecks at Start are the only ones run), staging options +(`options` must be `{}`: the panel's current options run), `select`, `cfr_batch`, and claiming the +channel's owner token (a claimed channel is refused). +""" + +from __future__ import annotations + +import functools +import os +import re +import time +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple + +from pa_lab import control, shot, timestamps +from pa_lab.control import Refused +from pa_lab.lab_reader import ConsoleReader + +TOOLS = ('run_program', 'stop_program') +# `start SECONDS`: how long the app waits for controller=ready before refusing. The daemon checks +# the controller first, so this only covers a reconnect between the check and the line. +START_CONTROLLER_WAIT_S = 10 +# How long a call waits for its answer before returning a pending ticket. The app polls its inbox +# every 500 ms; a stop then runs in the background until the program reaches `stopped`. +ANSWER_WAIT_S = {'run_program': START_CONTROLLER_WAIT_S + 3.0, 'stop_program': 10.0} +POLL_S = 0.1 + +JOURNAL_ERROR_RE = re.compile(r'^error \d+ \([^)]*\): (.*)$') +PROGRAM_IS_RE = re.compile(r'^Program is ([A-Z_]+)\.$') +ENDED_SOURCES = ('program', 'error') + +__all__ = ['ProgramActions', 'TOOLS'] + + +def _journal(lab_dir: Path, name: str) -> Optional[List[str]]: + """The request's finished journal, or None while there is none.""" + try: + lines = (lab_dir / 'outbox' / (name + '.out')).read_text(encoding='utf-8', errors='replace').splitlines() + except FileNotFoundError: + return None + return lines if lines and lines[-1] == 'end' else None + + +def _line_error(lines: Optional[List[str]]) -> Optional[str]: + for line in lines or (): + match = JOURNAL_ERROR_RE.match(line) + if match: + return match.group(1) + return None + + +def _line_ok(lines: Optional[List[str]], command: str) -> bool: + ok = re.compile(r'^ok \d+ ' + command + r'\b') + return any(ok.match(line) for line in lines or ()) + + +def _withdraw(inbox: Path) -> bool: + """True if the request was still in the inbox (the app never took it) and is now gone.""" + try: + os.remove(inbox) + except FileNotFoundError: + return False + return True + + +def _refusal(code: str, message: str, **extra: Any) -> Dict[str, Any]: + return dict(code=code, message=message, **extra) + + +def _app_error(tool: str, error: str, lines: List[str]) -> Dict[str, Any]: + """The refusal for a failed start or stop line (LabChannel's messages).""" + if error.startswith('owned by'): + # The journal names the owner token: keep it out of the record. + return _refusal('console_held_by_agent', "A lab script holds the app channel's owner lock; " + 'the daemon does not claim it in Slice 5a.', evidence={'reason': 'channel_owned'}) + if 'not a program' in error: + return _refusal('panel_not_program', 'The selected panel is not a program.') + if 'controller not connected' in error: + return _refusal('controller_not_ready', 'The app refused the start: {}'.format(error)) + match = PROGRAM_IS_RE.match(error) + if match and tool == 'run_program': + code = 'program_running' if match.group(1) in ('RUNNING', 'STOPPING') else 'program_not_stopped' + return _refusal(code, 'The app refused the start: {}'.format(error)) + return _refusal('app_unavailable', 'The app refused the {}: {}'.format( + 'start' if tool == 'run_program' else 'stop', error), evidence={'journal': lines}) + + +def _first_blocked(state: Dict[str, Any], skip=('console_held_by_user',)) -> None: + """Raises the first refusal console://B/state lists for a run (the lease is checked apart).""" + for entry in state['blocked']: + if entry['code'] not in skip: + extra = {k: v for k, v in entry.items() if k not in ('code', 'message')} + raise Refused(entry['code'], entry['message'], **extra) + + +def _need_app(state: Dict[str, Any]) -> Dict[str, Any]: + if state['app'] is None: + reason = state['availability']['reason'] + message = state['blocked'][0]['message'] if state['blocked'] else 'The app is unavailable.' + raise Refused('app_unavailable', message, evidence={'reason': reason}) + return state['app'] + + +def _need_free_channel(app: Dict[str, Any]) -> None: + if (app.get('channel') or {}).get('owner_claimed'): + raise Refused('console_held_by_agent', "A lab script holds the app channel's owner lock (owner claim); " + 'the daemon does not claim it in Slice 5a.', evidence={'reason': 'channel_owned'}) + + +def check_run(state: Dict[str, Any], lease: Any, args: Dict[str, Any]) -> None: + """run_program's checks after the lease, in order. Raises Refused; None means send `start`.""" + app = _need_app(state) + if args['options']: + raise Refused('invalid_arguments', "Slice 5a runs the panel's current options; pass options {}. " + 'Staging options comes with preflight_program (Slice 5b).', field='options') + if 'preflight_token' in args: + raise Refused('invalid_arguments', 'There is no preflight_program in Slice 5a; omit preflight_token.', + field='preflight_token') + if app['panel']['kind'] == 'program' and app['panel']['id'] != args['panel']: + raise Refused('invalid_arguments', 'The selected panel is {}, not {}; run_program does not change ' + 'the selection.'.format(app['panel']['id'], args['panel']), field='panel', + evidence={'selected': app['panel']['id']}) + _first_blocked(state) + _need_free_channel(app) + + +def check_stop(state: Dict[str, Any], lease: Any, args: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """stop_program's checks after the lease. Returns the result when nothing runs (no stop is sent).""" + app = _need_app(state) + if app['panel']['kind'] != 'program': + raise Refused('panel_not_program', 'The selected panel is not a program.') + program = app['program'] + if program['state'] != 'running': + return {'console': state['console'], 'instance_id': app['instance']['id'], 'sent': False, + 'state': program['state'], 'last_transition': program['last_transition']} + _need_free_channel(app) + return None + + +class ProgramActions: + """run_program / stop_program over a control plane and the allowlisted consoles' readers.""" + + def __init__(self, plane: control.ControlPlane, consoles: Dict[str, ConsoleReader], + sleep: Callable[[float], None] = time.sleep): + self.plane = plane + self.consoles = consoles + self.sleep = sleep + + def run_program(self, client_id: str, request: Dict[str, Any]) -> Dict[str, Any]: + return self._act(client_id, dict(request, tool='run_program'), check_run, + 'start {}'.format(START_CONTROLLER_WAIT_S)) + + def stop_program(self, client_id: str, request: Dict[str, Any]) -> Dict[str, Any]: + return self._act(client_id, dict(request, tool='stop_program'), check_stop, 'stop') + + def settle_pending(self, console: str) -> None: + """Settles every pending action on `console` from what the app shows now (no waiting).""" + reader = self.consoles.get(console) if isinstance(console, str) else None + if reader is None: + return + for action in self.plane.pending_actions(console): + self._settle(reader, action, wait_s=0) + + # ---- internals + + def _reader(self, console: Any) -> ConsoleReader: + reader = self.consoles.get(console) if isinstance(console, str) else None + if reader is None: + raise Refused('console_not_allowed', 'Console {!r} is not served; only {}.'.format( + console, ', '.join(sorted(self.consoles)))) + return reader + + def _act(self, client_id: str, request: Dict[str, Any], + check: Callable[..., Optional[Dict[str, Any]]], command: str) -> Dict[str, Any]: + reader = self._reader(request.get('console')) + self.settle_pending(reader.console) + state = reader.console_state() + instance = state['app']['instance']['id'] if state['app'] is not None else '' + record, act = self.plane.begin_action(client_id, request, instance, functools.partial(check, state)) + tool = request['tool'] + action = self.plane.action(client_id, tool, record['request_id']) + if not act: + if record['status'] == 'pending' and action is not None: + return self._settle(reader, action, wait_s=0) + return record + shot.write_request(reader.lab_dir, action['inbox_name'], record['request_id'], command) + return self._settle(reader, action, wait_s=ANSWER_WAIT_S[tool]) + + def _settle(self, reader: ConsoleReader, action: Dict[str, Any], wait_s: float) -> Dict[str, Any]: + key = (action['client_id'], action['tool'], action['request_id']) + deadline = reader.now() + wait_s + while True: + outcome = self._judge(reader, action) + if outcome is not None: + return self.plane.finish_action(*key, **outcome) + if reader.now() >= deadline: + break + self.sleep(POLL_S) + age = reader.now() - timestamps.parse(action['written_at']) + if age >= ANSWER_WAIT_S[action['tool']]: + if _withdraw(reader.lab_dir / 'inbox' / (action['inbox_name'] + '.txt')): + return self.plane.finish_action(*key, refusal=_refusal( + 'timeout', 'The app did not take the {} request within {:g} s (busy with another ' + 'request?); it was withdrawn and nothing was sent.'.format( + action['tool'], ANSWER_WAIT_S[action['tool']]), evidence={'request': action['inbox_name']})) + state = reader.console_state() + if state['app'] is None and state['availability']['reason'] == 'app_not_running': + return self.plane.finish_action(*key, refusal=_refusal( + 'app_unavailable', 'The app exited before the {} settled.'.format(action['tool']), + evidence={'reason': 'app_not_running'})) + return self.plane.record(*key) + + def _judge(self, reader: ConsoleReader, action: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """The outcome the app shows now, as finish_action keyword arguments, or None to wait.""" + lines = _journal(reader.lab_dir, action['inbox_name']) + app = reader.console_state()['app'] + if app is None: + return None + tool, request_id = action['tool'], action['request_id'] + program = app['program'] + transition = program['last_transition'] + instance = app['instance']['id'] + ours = transition is not None and transition['source'] == 'lab' and transition['request_id'] == request_id + after = transition is not None and transition['at'] >= action['written_at'] + base = {'console': reader.console, 'instance_id': instance} + if tool == 'run_program': + if ours and transition['action'] == 'start': + result = dict(base, started_at=transition['at'], run_id=program['run_id'], ended=None, + correlated_by='request_id') + return {'result': result, 'run_started': (instance, transition)} + elif ours and transition['action'] == 'stop': + if program['state'] != 'stopped': + return None + return {'result': dict(base, sent=True, stopped_at=transition['at'], stopped_by='lab', + correlated_by='request_id')} + if instance != action['instance_id']: + return {'refusal': _refusal('app_unavailable', 'The app restarted before the {} settled.'.format(tool), + evidence={'reason': 'instance_replaced', 'instance_id': action['instance_id'], + 'new_instance_id': instance})} + if after and transition['source'] == 'ui': + return {'refusal': _refusal('console_held_by_user', 'The user clicked {} in the app during the {}; ' + 'console {} is theirs.'.format(transition['action'].capitalize(), tool, + reader.console), + evidence={'transition': transition, 'journal_ok': lines is not None and + _line_ok(lines, 'start' if tool == 'run_program' else 'stop')})} + ended = after and transition['action'] == 'stop' and transition['source'] in ENDED_SOURCES and \ + program['state'] == 'stopped' + if tool == 'run_program': + if ended and _line_ok(lines, 'start'): + return {'result': dict(base, started_at=None, run_id=program['run_id'], + ended={'source': transition['source'], 'at': transition['at']}, + correlated_by='time')} + elif ended: + return {'result': dict(base, sent=True, stopped_at=transition['at'], stopped_by=transition['source'], + correlated_by='time')} + error = _line_error(lines) + if error is None: + return None + if tool == 'stop_program' and PROGRAM_IS_RE.match(error) and program['state'] == 'stopped': + # The run ended between the check and the stop line, before anything this request can date. + stopped_by = transition['source'] if transition is not None and transition['action'] == 'stop' else None + return {'result': dict(base, sent=True, stopped_at=None, stopped_by=stopped_by, correlated_by='state')} + return {'refusal': _app_error(tool, error, lines)} diff --git a/lab/server/pa_lab/contracts.py b/lab/server/pa_lab/contracts.py index eaa7362747..a5c2599737 100644 --- a/lab/server/pa_lab/contracts.py +++ b/lab/server/pa_lab/contracts.py @@ -599,7 +599,10 @@ def _page_order(obj, path, ctx): 'renew_console': Obj([TTL], open=False), 'release_console': Obj([], open=False), 'preflight_program': Obj([PANEL_ARG, F('options', AnyObject())], open=False), - 'run_program': Obj([PANEL_ARG, F('options', AnyObject()), F('preflight_token', Str())], open=False), + 'run_program': Obj([PANEL_ARG, F('options', AnyObject()), + F('preflight_token', Str(), required=False, + doc='from preflight_program (Slice 5b); Slice 5a has no preflight and refuses one')], + open=False), 'stop_program': Obj([], open=False), 'cfr_batch': Obj([F('lines', ListOf(Str(), min_items=1, max_items=200))], open=False), } diff --git a/lab/server/pa_lab/control.py b/lab/server/pa_lab/control.py index 60ea27fb8b..70b440adf9 100644 --- a/lab/server/pa_lab/control.py +++ b/lab/server/pa_lab/control.py @@ -27,7 +27,7 @@ import sqlite3 import threading import time -from typing import Any, Dict, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple from pa_lab import contracts, timestamps from pa_lab.db import Database @@ -76,6 +76,16 @@ def _clock_time(at: str) -> str: return at[11:16] +def _transition_key(instance_id: str, transition: Dict[str, Any]) -> str: + return _json([instance_id, transition['at'], transition['action'], transition['source'], + transition['request_id']]) + + +def inbox_name(tool: str, request_id: str) -> str: + """The `Lab/inbox` request name of a run_program / stop_program request (fixed by its id).""" + return 'palab_{}_{}'.format(tool, request_id.replace('-', '')) + + def lease_doc(row: sqlite3.Row) -> Dict[str, Any]: return {'state': row['state'], 'owner': row['owner'], 'acquired_at': row['acquired_at'], 'expires_at': row['expires_at']} @@ -194,19 +204,11 @@ def execute(self, client_id: str, request: Dict[str, Any]) -> Dict[str, Any]: if existing is not None: return existing self._expire_due(conn, req['console']) - stamp = self._stamp() try: result, refusal = handler(conn, req, args, token_hash), None except Refused as e: result, refusal = None, e.refusal - status = 'completed' if refusal is None else 'refused' - conn.execute('INSERT INTO requests (client_id, tool, request_id, console, token_hash, status, created_at, ' - 'completed_at, arguments, result, refusal) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - (client_id, req['tool'], req['request_id'], req['console'], token_hash, status, stamp, - stamp, _json(args), None if result is None else _json(result), - None if refusal is None else _json(refusal))) - self._request_event(conn, req['console'], req['request_id'], req['tool'], result, refusal) - return self._record(self._request_row(conn, client_id, req['tool'], req['request_id'])) + return self._store(conn, req, args, token_hash, result, refusal) # ---- tickets (long operations; the daemon starts and resolves them) -------------------------- @@ -243,6 +245,102 @@ def resolve_ticket(self, client_id: str, tool: str, request_id: str, result: Opt self._request_event(conn, row['console'], request_id, tool, result, refusal) return record + # ---- actions (run_program, stop_program: pa_lab.actions talks to the app) -------------------- + + def begin_action(self, client_id: str, request: Dict[str, Any], instance_id: str, + check: Callable[[sqlite3.Row, Dict[str, Any]], Optional[Dict[str, Any]]] + ) -> Tuple[Dict[str, Any], bool]: + """Decides, in one transaction, whether this call sends a request to the app. + + An exact retry gets its stored record (pending included) and `False`. Otherwise the lease + must be held by the request's token and `check(lease_row, arguments)` must pass: it raises + Refused (stored as a `refused` record) or returns a result for a request that needs no + app action (stored as `completed`). When it returns None the record is stored `pending` + with the app instance it was checked against, and the caller must act: `(record, True)`. + """ + req, args, token_hash = self._parse_request(client_id, request) + with self._writing() as conn: + existing = self._existing(conn, req, args, token_hash) + if existing is not None: + return existing, False + self._expire_due(conn, req['console']) + try: + lease = self._held(conn, req['console'], token_hash) + busy = conn.execute( + "SELECT a.tool FROM actions a JOIN requests r USING (client_id, tool, request_id) " + "WHERE a.console = ? AND r.status = 'pending' LIMIT 1", (req['console'],)).fetchone() + if busy is not None: + raise Refused('program_not_stopped', 'An earlier {} on console {} is not settled yet; follow it ' + 'with request_status or next_events.'.format(busy['tool'], req['console']), + evidence={'pending_tool': busy['tool']}) + immediate = check(lease, args) + except Refused as e: + return self._store(conn, req, args, token_hash, None, e.refusal), False + if immediate is not None: + return self._store(conn, req, args, token_hash, immediate, None), False + stamp = self._stamp() + conn.execute('INSERT INTO requests (client_id, tool, request_id, console, token_hash, status, created_at, ' + "arguments) VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)", + (client_id, req['tool'], req['request_id'], req['console'], token_hash, stamp, _json(args))) + conn.execute('INSERT INTO actions (client_id, tool, request_id, console, instance_id, inbox_name, ' + 'written_at) VALUES (?, ?, ?, ?, ?, ?, ?)', + (client_id, req['tool'], req['request_id'], req['console'], instance_id, + inbox_name(req['tool'], req['request_id']), stamp)) + return self._record(self._request_row(conn, client_id, req['tool'], req['request_id'])), True + + def action(self, client_id: str, tool: str, request_id: str) -> Optional[Dict[str, Any]]: + """What a run_program / stop_program request sent: console, instance_id, inbox_name, written_at.""" + row = self.db.read().execute('SELECT * FROM actions WHERE client_id = ? AND tool = ? AND request_id = ?', + (client_id, tool, request_id)).fetchone() + return None if row is None else {k: row[k] for k in row.keys()} + + def record(self, client_id: str, tool: str, request_id: str) -> Optional[Dict[str, Any]]: + row = self._request_row(self.db.read(), client_id, tool, request_id) + return None if row is None else self._record(row) + + def pending_actions(self, console: str) -> List[Dict[str, Any]]: + """Unresolved run_program / stop_program requests on `console`, oldest first.""" + rows = self.db.read().execute( + "SELECT a.* FROM actions a JOIN requests r USING (client_id, tool, request_id) " + "WHERE a.console = ? AND r.status = 'pending' ORDER BY a.written_at", (console,)).fetchall() + return [{k: row[k] for k in row.keys()} for row in rows] + + def finish_action(self, client_id: str, tool: str, request_id: str, result: Optional[Dict[str, Any]] = None, + refusal: Optional[Dict[str, Any]] = None, + run_started: Optional[Tuple[str, Dict[str, Any]]] = None) -> Dict[str, Any]: + """Resolves a pending action once (later calls return the record). + + `run_started` is `(instance_id, transition)`: the app shows this request's start as its + last transition. The request's lease, if its token still holds it idle, enters + agent_running in the same transaction, unless `observe` has already applied a newer + transition (the run ended before this call got here): then it stays idle.""" + if (result is None) == (refusal is None): + raise ValueError('finish_action needs exactly one of result and refusal') + with self._writing() as conn: + row = self._request_row(conn, client_id, tool, request_id) + if row is None: + raise Refused('invalid_arguments', 'No request {}.'.format(request_id), field='request_id') + if row['status'] != 'pending': + return self._record(row) + conn.execute('UPDATE requests SET status = ?, completed_at = ?, result = ?, refusal = ? ' + 'WHERE client_id = ? AND tool = ? AND request_id = ?', + ('completed' if refusal is None else 'failed', self._stamp(), + None if result is None else _json(result), None if refusal is None else _json(refusal), + client_id, tool, request_id)) + self._request_event(conn, row['console'], request_id, tool, result, refusal) + lease = self._lease_row(conn, row['console']) + if run_started is not None and lease['state'] == 'agent_idle' and \ + lease['token_hash'] == row['token_hash']: + instance_id, transition = run_started + seen = conn.execute('SELECT transition_key FROM observations WHERE console = ?', + (row['console'],)).fetchone() + if seen is None or seen['transition_key'] == _transition_key(instance_id, transition): + self._set_lease(conn, row['console'], 'run_started', request_id=request_id, + instance_id=instance_id, state='agent_running', expires_at=None, + run_request_id=request_id, run_instance_id=instance_id, + run_started_at=self._stamp()) + return self._record(self._request_row(conn, client_id, tool, request_id)) + # ---- runs and the operator ---------------------------------------------------------------- def begin_run(self, console: str, lease_token: str, request_id: str, instance_id: str) -> Dict[str, Any]: @@ -291,8 +389,7 @@ def observe(self, console: str, app: Dict[str, Any]) -> Dict[str, Any]: self._check_console(console) instance = app['instance']['id'] transition = app['program']['last_transition'] - key = None if transition is None else _json([instance, transition['at'], transition['action'], - transition['source'], transition['request_id']]) + key = None if transition is None else _transition_key(instance, transition) with self._writing() as conn: self._expire_due(conn, console) seen = conn.execute('SELECT transition_key FROM observations WHERE console = ?', (console,)).fetchone() @@ -342,6 +439,7 @@ def compact(self) -> Dict[str, int]: conn.execute("SELECT client_id, tool, request_id, completed_at FROM requests WHERE status != 'pending'") if now - timestamps.parse(row['completed_at']) > r.request_ttl_s] conn.executemany('DELETE FROM requests WHERE client_id = ? AND tool = ? AND request_id = ?', done) + conn.executemany('DELETE FROM actions WHERE client_id = ? AND tool = ? AND request_id = ?', done) self._appended = 0 return {'events': events, 'requests': len(done), 'cursors': len(stale)} @@ -439,6 +537,18 @@ def _append(self, conn, type: str, console: str, payload: Dict[str, Any], instan self._appended += 1 return cursor.lastrowid + def _store(self, conn, req, args, token_hash, result, refusal) -> Dict[str, Any]: + """Stores a request decided at once: `completed` with its result or `refused`.""" + stamp = self._stamp() + status = 'completed' if refusal is None else 'refused' + conn.execute('INSERT INTO requests (client_id, tool, request_id, console, token_hash, status, created_at, ' + 'completed_at, arguments, result, refusal) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + (req['client_id'], req['tool'], req['request_id'], req['console'], token_hash, status, stamp, + stamp, _json(args), None if result is None else _json(result), + None if refusal is None else _json(refusal))) + self._request_event(conn, req['console'], req['request_id'], req['tool'], result, refusal) + return self._record(self._request_row(conn, req['client_id'], req['tool'], req['request_id'])) + def _request_event(self, conn, console, request_id, tool, result, refusal) -> None: if refusal is None: self._append(conn, 'request_completed', console, diff --git a/lab/server/pa_lab/db.py b/lab/server/pa_lab/db.py index 6624b6e8bf..c16691330d 100644 --- a/lab/server/pa_lab/db.py +++ b/lab/server/pa_lab/db.py @@ -86,6 +86,21 @@ )""", "CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)", ], + # 2: what a run_program / stop_program request sent to the app, so a pending one can be + # settled after a daemon restart: the app instance it was checked against, the inbox + # request's name and when it was written. + [ + """CREATE TABLE actions ( + client_id TEXT NOT NULL, + tool TEXT NOT NULL, + request_id TEXT NOT NULL, + console TEXT NOT NULL, + instance_id TEXT NOT NULL, + inbox_name TEXT NOT NULL, + written_at TEXT NOT NULL, + PRIMARY KEY (client_id, tool, request_id) + )""", + ], ] SCHEMA_VERSION = len(MIGRATIONS) diff --git a/lab/server/pa_lab/mcp_server.py b/lab/server/pa_lab/mcp_server.py index 8eb6c0ced1..57ed5785d8 100644 --- a/lab/server/pa_lab/mcp_server.py +++ b/lab/server/pa_lab/mcp_server.py @@ -7,8 +7,9 @@ Resources: `console://B/state` (console_state), `console://B/save` (save_resource). Tools: `shot`, `ensure_game`, `describe_program`, `game_open`; with a control plane also `acquire_console`, -`renew_console`, `release_console`, `next_events` and `request_status`. Nothing here presses a -button or starts or stops a program: that is not served yet. +`renew_console`, `release_console`, `next_events` and `request_status`; with program actions +(`serve --actions`, Slice 5a) also `run_program` and `stop_program`, the only tools that start or +stop a program. Nothing here presses a button. Read-only mode (`read_only=True`, `serve-stdio`): only `console_status` and the resources. Every tool that writes into `Lab/inbox` (`shot`, `describe_program`, `game_open`) and every control tool is @@ -55,6 +56,12 @@ 'only verifies, it never navigates. describe_program gives the selected program\'s option contract; ' 'game_open says whether a game is in front. Unknown observations are never success.') +ACTION_INSTRUCTIONS = ( + 'Lab console B. Reads need no lease. run_program and stop_program start and stop the selected program: ' + 'acquire_console first, give every call a new request_id (retry with the same one), and follow a pending ' + 'record with request_status or next_events. A refusal names the check that failed; when the user holds ' + 'the console, stop and wait for them.') + TOOL_LIST = [ { 'name': 'shot', @@ -190,6 +197,29 @@ def _schema(properties: Dict[str, Any], required) -> Dict[str, Any]: LEASE_TOOLS = ('acquire_console', 'renew_console', 'release_console') +_ACTING = {'readOnlyHint': False, 'destructiveHint': True, 'idempotentHint': True} + +ACTION_TOOL_LIST = [ + {'name': 'run_program', 'title': 'Start the selected program', + 'description': "Starts the program selected in the app with its current options, once per request_id. Needs " + 'the lease token. panel must be the selected program id (this does not change the selection) ' + 'and options must be {} (staging options comes with preflight_program, Slice 5b). Refused ' + 'while the user holds the console, a program runs, the controller or video is not ready, or ' + 'the app is not running. Returns the request record: completed (the app shows this ' + "request's start; the lease is agent_running until the run ends), failed, or pending " + '(follow it with request_status or next_events).', + 'inputSchema': _schema({'request_id': _REQUEST_ID, 'lease_token': _LEASE_TOKEN, + 'panel': {'type': 'string', 'description': "':', the selected program"}, + 'options': {'type': 'object', 'maxProperties': 0}}, + ['request_id', 'lease_token', 'panel', 'options']), 'annotations': _ACTING}, + {'name': 'stop_program', 'title': 'Stop the running program', + 'description': 'Stops the running program once per request_id. Needs the lease token. Completed when the app ' + "shows this request's stop and the program is stopped, or the run ended by itself first; " + 'sent false when nothing was running. Refused while the user holds the console.', + 'inputSchema': _schema({'request_id': _REQUEST_ID, 'lease_token': _LEASE_TOKEN}, + ['request_id', 'lease_token']), 'annotations': _ACTING}, +] + class RpcError(Exception): def __init__(self, code: int, message: str, data: Any = None): @@ -218,9 +248,13 @@ def __init__(self, consoles: Dict[str, ConsoleReader], take_shot: Callable[..., Dict[str, Any]] = shot.take_shot, control_plane: Optional[control.ControlPlane] = None, statuses: Optional[Dict[str, status.ProfileStatus]] = None, - read_only: bool = False): - if read_only and control_plane is not None: + read_only: bool = False, + actions: Optional[Any] = None): + if read_only and (control_plane is not None or actions is not None): raise ValueError('a read-only server has no control plane') + if actions is not None and actions.plane is not control_plane: + raise ValueError('program actions need the server\'s control plane') + self.actions = actions self.consoles = consoles self.take_shot = take_shot self.control = control_plane @@ -231,7 +265,8 @@ def tool_list(self): status_tools = [CONSOLE_STATUS_TOOL] if self.statuses else [] if self.read_only: return status_tools - return TOOL_LIST + status_tools + (CONTROL_TOOL_LIST if self.control is not None else []) + return TOOL_LIST + status_tools + (CONTROL_TOOL_LIST if self.control is not None else []) + \ + (ACTION_TOOL_LIST if self.actions is not None else []) def call(self, method: str, params: Dict[str, Any], client: str = 'unnamed') -> Dict[str, Any]: if method == 'ping': @@ -249,17 +284,19 @@ def call(self, method: str, params: Dict[str, Any], client: str = 'unnamed') -> raise RpcError(INVALID_PARAMS, 'Unknown tool: {}'.format(params.get('name'))) if params.get('name') == 'console_status': return self._call_status(params.get('arguments', {})) + if self.actions is not None and params.get('name') in [t['name'] for t in ACTION_TOOL_LIST]: + return self._call_action(client, params.get('name'), params.get('arguments', {})) if self.control is not None and params.get('name') in [t['name'] for t in CONTROL_TOOL_LIST]: return self._call_control(client, params.get('name'), params.get('arguments', {})) return self._call_tool(params.get('name'), params.get('arguments', {})) raise RpcError(METHOD_NOT_FOUND, 'method not found: {}'.format(method)) - @staticmethod - def initialize_result(params: Dict[str, Any]) -> Dict[str, Any]: + def initialize_result(self, params: Dict[str, Any]) -> Dict[str, Any]: asked = params.get('protocolVersion') version = asked if asked in PROTOCOL_VERSIONS else PROTOCOL_VERSIONS[0] return {'protocolVersion': version, 'capabilities': {'resources': {}, 'tools': {}}, - 'serverInfo': SERVER_INFO, 'instructions': INSTRUCTIONS} + 'serverInfo': SERVER_INFO, + 'instructions': ACTION_INSTRUCTIONS if self.actions is not None else INSTRUCTIONS} @staticmethod def _resources(console: str): @@ -287,6 +324,7 @@ def _read_resource(self, uri: Any) -> Dict[str, Any]: 'uri': uri, 'refusal': {'code': 'console_not_allowed', 'message': 'Console {} is not served.'.format(match.group(1))}}) if match.group(2) == 'state': + self._settle(match.group(1)) doc = reader.console_state() else: try: @@ -358,6 +396,7 @@ def _call_control(self, client: str, name: str, arguments: Any) -> Dict[str, Any args = dict(arguments) console = args.pop('console', None) request_id = args.get('request_id') if isinstance(args.get('request_id'), str) else None + self._settle(console) try: if name == 'next_events': unknown = sorted(set(args) - {'after_seq', 'timeout_s', 'limit'}) @@ -385,6 +424,31 @@ def _call_control(self, client: str, name: str, arguments: Any) -> Dict[str, Any return {'content': [{'type': 'text', 'text': _json_text(result)}], 'structuredContent': result, 'isError': False} + def _settle(self, console: Any) -> None: + """Settles pending run/stop requests before a read, so status and events show their outcome.""" + if self.actions is not None and isinstance(console, str) and console in self.consoles: + self.actions.settle_pending(console) + + def _call_action(self, client: str, name: str, arguments: Any) -> Dict[str, Any]: + if not isinstance(arguments, dict): + return self._refused(name, {'code': 'invalid_arguments', 'field': 'arguments', + 'message': 'expected an object'}) + args = dict(arguments) + request = {'schema': 1, 'request_id': args.pop('request_id', None), 'tool': name, + 'console': args.pop('console', None), 'arguments': args} + if 'lease_token' in args: + request['lease_token'] = args.pop('lease_token') + call = self.actions.run_program if name == 'run_program' else self.actions.stop_program + try: + record = call(client, request) + except control.Refused as e: + return self._refused(name, e.refusal, request['request_id'] if isinstance(request['request_id'], str) + else None) + if record['refusal'] is not None: + return self._refused(name, record['refusal'], record['request_id']) + return {'content': [{'type': 'text', 'text': _json_text(record)}], 'structuredContent': record, + 'isError': False} + @staticmethod def _refused(tool: str, refusal: Dict[str, Any], request_id: Optional[str] = None) -> Dict[str, Any]: try: @@ -579,7 +643,7 @@ def _initialize(self, message: Dict[str, Any], params: Any) -> None: client = str(info.get('name') or 'unnamed')[:64] session = self.server.sessions.open(client) LOG.info('client=%s initialize', client) - self._send(200, {'jsonrpc': '2.0', 'id': message['id'], 'result': LabMcp.initialize_result(params)}, + self._send(200, {'jsonrpc': '2.0', 'id': message['id'], 'result': self.server.mcp.initialize_result(params)}, {'Mcp-Session-Id': session}) diff --git a/lab/server/tests/fake_lab.py b/lab/server/tests/fake_lab.py index 060795fcf0..8142b14988 100644 --- a/lab/server/tests/fake_lab.py +++ b/lab/server/tests/fake_lab.py @@ -111,6 +111,15 @@ class FakeApp: `describe_program` and `game_open` (#504) are answered at once: `replies[command]` is the printed text (default below), `line_errors[command]` fails the line instead, and `restart_on` names a command whose answer comes from a new app instance. + + `start [SECONDS]` and `stop` do what LabChannel's start_or_stop does: refused with `Program is + STATE.` unless the program is stopped (start) or running (stop), or with `owned by TOKEN` + when `owner` is set and the request has no `owner TOKEN` line; otherwise the program state + changes and `last_transition` names the request's `request ` with source `lab`. A stop + stays `stopping` for `stop_ticks` ticks. `after[command]` is called with the app right after + that command succeeds, before the journal is written: a UI click or the run's end landing + before the daemon reads the answer. A command in `hold` is taken from the inbox and never + answered. """ def __init__(self, lab, clock): @@ -130,6 +139,11 @@ def __init__(self, lab, clock): } self.line_errors = {} self.restart_on = None + self.owner = None + self.stop_ticks = 1 + self.after = {} + self._stopping_left = None + self.hold = set() self.requests = [] self.pending = [] clock.on_sleep.append(self.tick) @@ -141,6 +155,11 @@ def tick(self): changes['video.frame_seq'] = (self.lab.state['video']['frame_seq'] or 0) + 1 changes['video.state'] = 'live' changes['video.captured_at'] = now + if self._stopping_left is not None: + self._stopping_left -= 1 + if self._stopping_left <= 0: + self._stopping_left = None + changes['program.state'] = 'stopped' self.lab.update_state(**changes) for pending in list(self.pending): self.continue_snapshot(pending) @@ -157,6 +176,11 @@ def pick_up(self, request): if words[0] in self.replies: self.answer(request.stem, line, words[0]) return + if words[0] in self.hold: # taken from the inbox, never answered (the app hung or quit) + return + if words[0] in ('start', 'stop'): + self.start_or_stop(request.stem, text, line, words[0]) + return pending = {'name': request.stem, 'line': line, 'label': words[1], 'timeout_ms': int(words[2]), 'started': self.clock.now(), 'after': self.lab.state['video']['frame_seq'] or 0} if self.journal_error is not None: @@ -176,6 +200,33 @@ def answer(self, name, line, command): else: self.finish(pending, self.replies[command]) + def start_or_stop(self, name, text, line, command): + lines = text.splitlines() + request_id = next((l.split()[1] for l in lines if l.startswith('request ')), None) + pending = {'name': name, 'line': line} + if self.owner is not None and 'owner {}'.format(self.owner) not in lines: + self.finish(pending, '', 'owned by {}'.format(self.owner)) + return + state = self.lab.state['program']['state'] + if state != ('stopped' if command == 'start' else 'running'): + self.finish(pending, '', 'Program is {}.'.format(state.upper())) + return + self.transition(command, 'lab', request_id, 'running' if command == 'start' else 'stopping') + if command == 'stop': + self._stopping_left = self.stop_ticks + if command in self.after: + self.after[command](self) + self.finish(pending, '') + + def transition(self, action, source, request_id=None, state=None): + """A start or stop as the app records it; `state` defaults to where the action leads.""" + self._stopping_left = None + self.lab.update_state(**{ + 'program.state': state or ('running' if action == 'start' else 'stopped'), + 'program.last_transition': {'action': action, 'source': source, 'request_id': request_id, + 'at': timestamps.format(self.clock.now())}, + 'written_at': timestamps.format(self.clock.now())}) + def continue_snapshot(self, pending): video = self.lab.state['video'] if video['frame_seq'] is not None and video['frame_seq'] > pending['after']: diff --git a/lab/server/tests/test_actions.py b/lab/server/tests/test_actions.py new file mode 100644 index 0000000000..64a05dc1c8 --- /dev/null +++ b/lab/server/tests/test_actions.py @@ -0,0 +1,448 @@ +"""run_program and stop_program against the fake app (Slice 5a, fork #131): the refusals, an exact +retry, and the races of the design's Slice 5 acceptance that need no console: a duplicate request, +a UI takeover during start and stop, lease loss, an app restart between requests, and a terminal +event before the response. A FakeClock drives the app; nothing touches a real profile. + +Run from the repo root: python3 -m unittest discover -s lab/server/tests -t lab/server -v +""" + +import copy +import unittest + +from pa_lab import actions, contracts, control, lab_reader, mcp_server +from tests.control_fixtures import START, acquire, new_id, open_plane, temp_db_path +from tests.fake_lab import FakeApp, FakeClock, FakeLab, golden + +PANEL = 'PokemonFRLG:AutoStory' +CLIENT = 'claude:frlg-lane' +NEW_INSTANCE = '0b5e3f6c-9d2a-4e71-8c40-6f1a2b3c4d5e' + + +def stopped_app(): + doc = copy.deepcopy(golden('app_state', 'stopped_ui')) + doc['program']['last_transition'] = None + doc['written_at'] = START + doc['video']['captured_at'] = START + return doc + + +def run_request(token, request_id=None, panel=PANEL, options=None, **arguments): + return {'schema': 1, 'request_id': request_id or new_id(), 'console': 'B', 'lease_token': token, + 'arguments': dict({'panel': panel, 'options': {} if options is None else options}, **arguments)} + + +def stop_request(token, request_id=None): + return {'schema': 1, 'request_id': request_id or new_id(), 'console': 'B', 'lease_token': token, + 'arguments': {}} + + +class ActionTest(unittest.TestCase): + def setUp(self): + self.clock = FakeClock(START) + self.lab = FakeLab(self) + self.lab.write_state(stopped_app()) + self.app = FakeApp(self.lab, self.clock) + self.alive = True + self.path = temp_db_path(self) + self.open_daemon() + + def open_daemon(self): + """A (re)started daemon: a new control plane, reader and actions on the same SQLite file.""" + self.plane = open_plane(self, self.path, self.clock) + self.reader = lab_reader.ConsoleReader('B', self.lab.dir, profile='lab-B', now=self.clock.now, + pid_alive=lambda pid: self.alive, control=self.plane) + self.actions = actions.ProgramActions(self.plane, {'B': self.reader}, sleep=self.clock.sleep) + + def acquired(self): + record = self.plane.execute(CLIENT, acquire()) + self.assertEqual(record['status'], 'completed', record) + return record['result']['lease_token'] + + def run_program(self, request): + return self.actions.run_program(CLIENT, request) + + def stop_program(self, request): + return self.actions.stop_program(CLIENT, request) + + def started(self, token): + record = self.run_program(run_request(token)) + self.assertEqual(record['status'], 'completed', record) + return record + + def sent(self, command=None): + """The start/stop requests the app took: [(request_id, command line)].""" + out = [] + for text in self.app.requests: + lines = text.splitlines() + if lines[-1].split()[0] in ('start', 'stop') and (command is None or lines[-1].startswith(command)): + out.append((lines[0].split()[1], lines[-1])) + return out + + def inbox(self): + return sorted(p.name for p in (self.lab.dir / 'inbox').iterdir()) + + def lease(self): + return self.plane.lease('B')['state'] + + def event_types(self): + return [(e['type'], e['payload'].get('reason') or e['payload'].get('tool')) + for e in self.plane.next_events('reader', 'B', 0, limit=500)['events']] + + def on_first_sleep(self, hook): + """Runs `hook` once, at the first sleep after this call, before the app's own tick.""" + def once(): + self.clock.on_sleep.remove(once) + hook() + self.clock.on_sleep.insert(0, once) + + +class RunTests(ActionTest): + def test_a_run_starts_once_and_the_lease_runs(self): + token = self.acquired() + request = run_request(token) + record = self.run_program(request) + self.assertEqual(record['status'], 'completed') + self.assertEqual(record['arguments'], {'panel': PANEL, 'options': {}}) + self.assertEqual(record['result']['correlated_by'], 'request_id') + self.assertIsNone(record['result']['ended']) + self.assertEqual(self.sent(), [(request['request_id'], 'start 10')]) + self.assertEqual(self.lease(), 'agent_running') + self.assertEqual(self.lab.state['program']['last_transition']['request_id'], request['request_id']) + self.assertIn(('request_completed', 'run_program'), self.event_types()) + self.assertIn(('lease_changed', 'run_started'), self.event_types()) + contracts.validate('request_record', record) + + def test_an_exact_retry_returns_the_stored_record_and_sends_nothing(self): + token = self.acquired() + request = run_request(token) + first = self.run_program(request) + self.clock.sleep(5) + again = self.run_program(request) + self.assertEqual(first, again) + self.assertEqual(len(self.sent()), 1) + + def test_a_retry_of_a_refusal_returns_the_same_refusal(self): + token = self.acquired() + request = run_request(token, panel='PokemonBDSP:AutoStory') + first = self.run_program(request) + self.lab.update_state(**{'panel.id': 'PokemonBDSP:AutoStory'}) + again = self.run_program(request) + self.assertEqual((first['status'], first['refusal']['code']), ('refused', 'invalid_arguments')) + self.assertEqual(first, again) + self.assertEqual(self.sent(), []) + + def test_a_duplicate_while_the_first_is_in_flight_gets_the_pending_record(self): + token = self.acquired() + request = run_request(token) + seen = {} + self.on_first_sleep(lambda: seen.update( + duplicate=self.run_program(request), + other=self.run_program(run_request(token)))) + record = self.run_program(request) + self.assertEqual(seen['duplicate']['status'], 'pending') + self.assertEqual((seen['other']['status'], seen['other']['refusal']['code']), + ('refused', 'program_not_stopped')) + self.assertEqual(record['status'], 'completed') + self.assertEqual(len(self.sent()), 1) + + def test_the_same_request_id_with_other_arguments_is_refused_unstored(self): + token = self.acquired() + request_id = new_id() + self.run_program(run_request(token, request_id)) + with self.assertRaises(control.Refused) as raised: + self.run_program(run_request(token, request_id, panel='PokemonBDSP:AutoStory')) + self.assertEqual(raised.exception.refusal['code'], 'request_id_reused') + + def test_refusals_send_nothing_and_are_stored(self): + def ui_click(test): + test.app.transition('stop', 'ui') + + def expire(test): + test.clock.sleep(1200) + + def other_program(test): + test.app.transition('start', 'lab', new_id()) + + def app_gone(test): + test.alive = False + + def other_panel(test): + test.lab.update_state(**{'panel.id': 'PokemonBDSP:AutoStory'}) + + def settings_page(test): + test.lab.update_state(**{'panel.id': None, 'panel.kind': 'other', 'program.state': 'no_program'}) + + def controller(test): + test.lab.update_state(**{'controller.state': 'not_connected'}) + + def owner_claimed(test): + test.lab.update_state(**{'channel.owner_claimed': True}) + + cases = [ + dict(name='the user holds the lease', input=(ui_click, {}), expectation='console_held_by_user'), + dict(name='no lease', input=(None, {'lease_token': 'not-a-lease'}), expectation='lease_invalid'), + dict(name='the lease expired', input=(expire, {}), expectation='lease_expired'), + dict(name='another program runs', input=(other_program, {}), expectation='program_running'), + dict(name='the app is not running', input=(app_gone, {}), expectation='app_unavailable'), + dict(name='another panel is selected', input=(other_panel, {}), expectation='invalid_arguments'), + dict(name='the settings page is selected', input=(settings_page, {}), expectation='panel_not_program'), + dict(name='options to stage', input=(None, {'options': {'START_SEGMENT': 3}}), + expectation='invalid_arguments'), + dict(name='a preflight token', input=(None, {'preflight_token': 'x'}), expectation='invalid_arguments'), + dict(name='the controller is not ready', input=(controller, {}), expectation='controller_not_ready'), + dict(name='a lab script owns the channel', input=(owner_claimed, {}), expectation='console_held_by_agent'), + ] + for case in cases: + with self.subTest(case['name']): + self.setUp() + token = self.acquired() + setup, changes = case['input'] + if setup is not None: + setup(self) + request = run_request(changes.pop('lease_token', token), **changes) + record = self.run_program(request) + self.assertEqual((record['status'], record['refusal']['code']), ('refused', case['expectation'])) + self.assertEqual(self.run_program(request), record) + self.assertEqual(self.sent(), []) + self.assertEqual(self.inbox(), []) + self.assertIn(('request_failed', 'run_program'), self.event_types()) + + +class StopTests(ActionTest): + def test_a_stop_completes_when_the_program_is_stopped(self): + token = self.acquired() + self.started(token) + self.app.stop_ticks = 3 + request = stop_request(token) + record = self.stop_program(request) + self.assertEqual(record['status'], 'completed') + self.assertEqual(record['result']['stopped_by'], 'lab') + self.assertEqual(record['result']['correlated_by'], 'request_id') + self.assertEqual(self.lab.state['program']['state'], 'stopped') + self.assertEqual(self.sent('stop'), [(request['request_id'], 'stop')]) + self.assertEqual(self.lease(), 'agent_idle') + self.assertIn(('lease_changed', 'run_ended'), self.event_types()) + + def test_nothing_running_is_answered_without_sending(self): + token = self.acquired() + record = self.stop_program(stop_request(token)) + self.assertEqual(record['status'], 'completed') + self.assertEqual((record['result']['sent'], record['result']['state']), (False, 'stopped')) + self.assertEqual(self.sent(), []) + + def test_refusals_send_nothing(self): + def ui_click(test): + test.app.transition('start', 'ui') + + def app_gone(test): + test.alive = False + + cases = [ + dict(name='the user started a program', input=(ui_click, None), expectation='console_held_by_user'), + dict(name='no lease', input=(None, 'not-a-lease'), expectation='lease_invalid'), + dict(name='the app is not running', input=(app_gone, None), expectation='app_unavailable'), + ] + for case in cases: + with self.subTest(case['name']): + self.setUp() + token = self.acquired() + setup, other_token = case['input'] + if setup is not None: + setup(self) + record = self.stop_program(stop_request(other_token or token)) + self.assertEqual((record['status'], record['refusal']['code']), ('refused', case['expectation'])) + self.assertEqual(self.sent('stop'), []) + + +class RaceTests(ActionTest): + def test_ui_takeover_during_start(self): + token = self.acquired() + self.app.after['start'] = lambda app: app.transition('stop', 'ui') + record = self.run_program(run_request(token)) + self.assertEqual((record['status'], record['refusal']['code']), ('failed', 'console_held_by_user')) + self.assertTrue(record['refusal']['evidence']['journal_ok']) + self.assertEqual(self.lease(), 'user') + retry = self.run_program(run_request(token)) + self.assertEqual(retry['refusal']['code'], 'console_held_by_user') + self.assertEqual(len(self.sent()), 1) + + def test_ui_takeover_during_stop(self): + token = self.acquired() + self.started(token) + self.app.stop_ticks = 5 + self.app.after['stop'] = lambda app: app.transition('stop', 'ui', state='stopping') + record = self.stop_program(stop_request(token)) + self.assertEqual((record['status'], record['refusal']['code']), ('failed', 'console_held_by_user')) + self.assertEqual(self.lease(), 'user') + + def test_lease_lost_to_the_user_while_running(self): + token = self.acquired() + self.started(token) + self.clock.sleep(1) + self.app.transition('stop', 'ui') + record = self.stop_program(stop_request(token)) + self.assertEqual((record['status'], record['refusal']['code']), ('refused', 'console_held_by_user')) + self.assertEqual(self.sent('stop'), []) + + def test_lease_lost_by_expiry_before_the_start(self): + token = self.acquired() + self.clock.sleep(1199) + self.clock.sleep(1) + record = self.run_program(run_request(token)) + self.assertEqual(record['refusal']['code'], 'lease_expired') + self.assertEqual(self.sent(), []) + + def test_app_restart_between_requests(self): + token = self.acquired() + request = run_request(token) + self.run_program(request) + self.assertEqual(self.lease(), 'agent_running') + restarted = stopped_app() + restarted['instance']['id'] = NEW_INSTANCE + restarted['written_at'] = self.lab.state['written_at'] + self.lab.write_state(restarted) + # The retry is answered from the record: no second start on the new app. + self.assertEqual(self.run_program(request)['status'], 'completed') + self.assertEqual(len(self.sent()), 1) + self.assertEqual(self.lease(), 'agent_idle') + self.assertIn(('lease_changed', 'app_restarted'), self.event_types()) + # The stop finds nothing running on the new app. + stop = self.stop_program(stop_request(token)) + self.assertEqual((stop['result']['sent'], stop['result']['instance_id']), (False, NEW_INSTANCE)) + # A new request starts on the new app. + record = self.started(token) + self.assertEqual(record['result']['instance_id'], NEW_INSTANCE) + self.assertEqual(self.lease(), 'agent_running') + + def test_app_restart_during_the_start(self): + token = self.acquired() + + def restart(app): + app.lab.update_state(**{'instance.id': NEW_INSTANCE, 'program.state': 'stopped', + 'program.last_transition': None}) + self.app.hold.add('start') + self.on_first_sleep(lambda: restart(self.app)) + record = self.run_program(run_request(token)) + self.assertEqual((record['status'], record['refusal']['code']), ('failed', 'app_unavailable')) + self.assertEqual(record['refusal']['evidence']['reason'], 'instance_replaced') + self.assertEqual(self.lease(), 'agent_idle') + + def test_the_run_ends_before_the_start_is_answered(self): + for source in ('program', 'error'): + with self.subTest(source): + self.setUp() + token = self.acquired() + self.app.after['start'] = lambda app: app.transition('stop', source) + record = self.run_program(run_request(token)) + self.assertEqual(record['status'], 'completed') + self.assertEqual(record['result']['ended']['source'], source) + self.assertEqual(record['result']['correlated_by'], 'time') + self.assertEqual(self.lease(), 'agent_idle') + types = self.event_types() + self.assertIn(('stopped', None), types) + self.assertNotIn(('lease_changed', 'run_started'), types) + + def test_the_run_ends_before_the_stop_line(self): + token = self.acquired() + self.started(token) + self.on_first_sleep(lambda: self.app.transition('stop', 'program')) + record = self.stop_program(stop_request(token)) + self.assertEqual(record['status'], 'completed') + self.assertEqual((record['result']['stopped_by'], record['result']['correlated_by']), ('program', 'time')) + self.assertEqual(self.lease(), 'agent_idle') + + def test_the_run_ends_after_the_status_read_but_before_the_lease_runs(self): + """observe applies the run's end before finish_action: the lease must not enter agent_running.""" + token = self.acquired() + request = run_request(token) + self.app.hold.add('start') + record = self.run_program(request) + self.assertEqual(record['status'], 'pending') + self.app.transition('start', 'lab', request['request_id']) + self.clock.sleep(1) + start = self.lab.state['program']['last_transition'] + self.app.transition('stop', 'program') + self.reader.console_state() # observe sees the stop first + app = self.lab.state + self.plane.finish_action(CLIENT, 'run_program', request['request_id'], + result={'console': 'B'}, run_started=(app['instance']['id'], start)) + self.assertEqual(self.lease(), 'agent_idle') + + +class TicketTests(ActionTest): + def test_a_request_the_app_never_took_is_withdrawn(self): + token = self.acquired() + self.app.serve = False + record = self.run_program(run_request(token)) + self.assertEqual((record['status'], record['refusal']['code']), ('failed', 'timeout')) + self.assertEqual(self.inbox(), []) + self.app.serve = True + self.clock.sleep(1) + self.assertEqual(self.sent(), []) + self.assertEqual(self.lab.state['program']['state'], 'stopped') + + def test_a_taken_request_stays_pending_and_settles_after_a_daemon_restart(self): + token = self.acquired() + request = run_request(token) + self.app.hold.add('start') + record = self.run_program(request) + self.assertEqual(record['status'], 'pending') + self.open_daemon() + self.app.transition('start', 'lab', request['request_id']) + self.actions.settle_pending('B') + settled = self.plane.request_status(CLIENT, 'B', request['request_id']) + self.assertEqual(settled['status'], 'completed') + self.assertEqual(self.lease(), 'agent_running') + self.assertEqual(self.run_program(request), self.plane.record(CLIENT, 'run_program', request['request_id'])) + + def test_a_pending_request_fails_when_the_app_exits(self): + token = self.acquired() + self.app.hold.add('start') + request = run_request(token) + self.assertEqual(self.run_program(request)['status'], 'pending') + self.alive = False + self.actions.settle_pending('B') + record = self.plane.record(CLIENT, 'run_program', request['request_id']) + self.assertEqual((record['status'], record['refusal']['code']), ('failed', 'app_unavailable')) + + +class McpTests(ActionTest): + def test_the_tools_are_listed_only_with_actions(self): + cases = [ + dict(name='actions', input=dict(control_plane=True, actions=True), expectation=True), + dict(name='control plane only', input=dict(control_plane=True, actions=False), expectation=False), + ] + for case in cases: + with self.subTest(case['name']): + server = mcp_server.LabMcp({'B': self.reader}, control_plane=self.plane, + actions=self.actions if case['input']['actions'] else None) + names = [t['name'] for t in server.call('tools/list', {})['tools']] + self.assertEqual('run_program' in names and 'stop_program' in names, case['expectation']) + + def test_read_only_mode_refuses_actions(self): + with self.assertRaises(ValueError): + mcp_server.LabMcp({'B': self.reader}, read_only=True, actions=self.actions) + + def test_run_and_stop_through_tools_call(self): + server = mcp_server.LabMcp({'B': self.reader}, control_plane=self.plane, actions=self.actions) + token = self.acquired() + run = server.call('tools/call', {'name': 'run_program', 'arguments': { + 'console': 'B', 'request_id': new_id(), 'lease_token': token, 'panel': PANEL, 'options': {}}}, + client=CLIENT) + self.assertFalse(run['isError'], run) + self.assertEqual(run['structuredContent']['status'], 'completed') + refused = server.call('tools/call', {'name': 'run_program', 'arguments': { + 'console': 'B', 'request_id': new_id(), 'lease_token': token, 'panel': PANEL, 'options': {}}}, + client=CLIENT) + self.assertTrue(refused['isError']) + self.assertEqual(refused['structuredContent']['refusal']['code'], 'program_running') + contracts.validate('refusal', refused['structuredContent']) + stop = server.call('tools/call', {'name': 'stop_program', 'arguments': { + 'console': 'B', 'request_id': new_id(), 'lease_token': token}}, client=CLIENT) + self.assertEqual(stop['structuredContent']['status'], 'completed') + self.assertNotIn(token, str(server.call('tools/call', {'name': 'next_events', 'arguments': { + 'console': 'B', 'after_seq': 0, 'limit': 500}}, client=CLIENT))) + + +if __name__ == '__main__': + unittest.main()