From c9b45d06e7e03ec0c9eb29bce8fb827730d6ca95 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 03:01:59 -0800 Subject: [PATCH] lab/server: Slice 3 durable control plane (#131) SQLite (stdlib sqlite3) becomes the source of truth for leases, request records, tickets and events: migrations by PRAGMA user_version, an owner-only file, and every change in one BEGIN IMMEDIATE transaction. acquire/renew/release with opaque tokens (TTL 1200 s; agent_running never expires), a UI Start/Stop in state.json revokes agent leases into a user lease that only the operator CLI ends, idempotent (client, tool, request_id) records, next_events with durable cursors, request_status, restart reconciliation, and compaction that keeps active cursors and unresolved tickets. Tests cover restarts, racing threads and processes, kills mid-write, stable retries and gap-free catch-up. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- lab/server/README.md | 55 +- lab/server/pa_lab/__main__.py | 39 +- lab/server/pa_lab/config.py | 11 +- lab/server/pa_lab/control.py | 534 +++++++++++++++++++ lab/server/pa_lab/db.py | 175 ++++++ lab/server/pa_lab/lab_reader.py | 25 +- lab/server/pa_lab/mcp_server.py | 106 +++- lab/server/tests/control_fixtures.py | 60 +++ lab/server/tests/test_control.py | 392 ++++++++++++++ lab/server/tests/test_control_concurrency.py | 171 ++++++ lab/server/tests/test_control_mcp.py | 100 ++++ lab/server/tests/test_db.py | 55 ++ 12 files changed, 1699 insertions(+), 24 deletions(-) create mode 100644 lab/server/pa_lab/control.py create mode 100644 lab/server/pa_lab/db.py create mode 100644 lab/server/tests/control_fixtures.py create mode 100644 lab/server/tests/test_control.py create mode 100644 lab/server/tests/test_control_concurrency.py create mode 100644 lab/server/tests/test_control_mcp.py create mode 100644 lab/server/tests/test_db.py diff --git a/lab/server/README.md b/lab/server/README.md index 78ea12ccb5..e879c3c9cf 100644 --- a/lab/server/README.md +++ b/lab/server/README.md @@ -2,8 +2,9 @@ The local server designed in `docs/design/Agent-Integration.md` (fork #131, branch `design/agent-integration`). Slice 0: the version-1 contracts, golden examples and a parser. -Slice 2: the attach-only read server ([below](#slice-2-the-attach-only-read-server)). It has no -lease store, no write path and no press: those are Slices 3 and 5. +Slice 2: the attach-only read server ([below](#slice-2-the-attach-only-read-server)). Slice 3: the +durable control plane ([below](#slice-3-the-durable-control-plane)): leases, request records, +tickets and events in SQLite. Nothing presses a button yet: that is Slice 5. | Path | What | |---|---| @@ -20,6 +21,9 @@ lease store, no write path and no press: those are Slices 3 and 5. | `pa_lab/mcp_server.py` | MCP over Streamable HTTP on `127.0.0.1`: bearer, Host/Origin, sessions, size caps | | `tests/fake_lab.py` | A fake `Lab/` folder and a fake app answering `snapshot` requests, in temp dirs | | `tests/test_{lab_reader,ensure_game,shot,config,mcp_server}.py` | Slice 2's fake-lab tests | +| `pa_lab/db.py` | The SQLite file: migrations (`PRAGMA user_version`), owner-only file, `BEGIN IMMEDIATE` writes | +| `pa_lab/control.py` | Leases, idempotent requests, tickets, events, `next_events`, reconciliation, compaction | +| `tests/test_{db,control,control_concurrency,control_mcp}.py` | Slice 3's tests: restarts, threads and processes racing, kills mid-write | Stdlib only (Python 3.8+), so no venv: the MCP transport is small enough to write on `http.server`. Run the tests from the repo root: @@ -184,3 +188,50 @@ journal names, and only when complete (signature and `IEND`). Refusals: `timeout or the saved frame is older than `max_age_ms`), `video_bad_signal`, `app_unavailable` (a reply from another instance: `evidence.reason` `instance_replaced`; `panel_changed`, `app_closing`, `save_failed` as `evidence.reason`; a line error or an incomplete PNG). + +## Slice 3: the durable control plane + +SQLite (`db_file` in the config, default `~/.local/state/pa-lab/control.sqlite`, created `0600`) +is the source of truth; every change is one `BEGIN IMMEDIATE` transaction, so racing threads, +processes or a restarted daemon cannot both win, and a kill mid-write leaves nothing. + +**Leases.** `acquire_console(console, request_id, owner, ttl_s=1200)` takes a `free` console and +returns an opaque `lease_token` (only its hash is in the lease row) and `expires_at`. +`renew_console` / `release_console` need the token. `agent_idle` expires after its TTL; +`agent_running` (entered by Slice 5's run start, `ControlPlane.begin_run`) never expires and ends +when the app reports the run's stop (back to `agent_idle` with a fresh TTL) or a new app instance. +A UI Start/Stop in `Lab/state.json` (`last_transition.source == 'ui'`) revokes any agent lease and +installs `user`, with no timeout; only the operator ends it: + +```sh +python3 -m pa_lab lease --config ~/.config/pa-lab/server.json show|claim-user|release-user +``` + +A token whose lease ended is refused with the reason: `lease_expired`, `console_held_by_user`, +otherwise `lease_invalid`. `lease_changed` events carry `old`, `new` and a `reason` (`acquired`, +`released`, `expired`, `run_started`, `run_ended`, `app_restarted`, `ui_takeover`, +`operator_claim`, `operator_release`). + +**Requests.** `(client_id, tool, request_id)` is unique; `client_id` is the MCP session's client +name. The record keeps the normalized arguments (defaults filled in) and the result or refusal. +An exact retry gets the stored record, refusals included, even after a restart; the same key with +other arguments is `request_id_reused`. `request_status(console, request_id)` reads the caller's +own record; the lease token never appears in a status read or an event. + +**Tickets** are request records with status `pending`: `open_ticket` / `resolve_ticket` are the +store side of Slice 5's long operations. They survive restarts and resolve once. + +**Events.** `next_events(console, after_seq, timeout_s<=60, limit<=500)` returns the events after +the cursor and `next_after_seq`; it waits for one up to `timeout_s`. `after_seq` is also the +client's confirmed position: pass the last cursor you processed, so a lost response loses nothing. +`seq` comes from `AUTOINCREMENT` and is never reused. + +**Reconciliation.** Every read of a live app (`console://B/state`) and the daemon's start apply +`observe`: each transition once (UI takeover, `stopped` with its source), a run's end, an app +restart. Expiries due during downtime apply on the next read, once. + +**Retention.** `compact()` runs at start and every 500 events: it keeps the newest 10 000 events, +every event after an active cursor (one used in the last 7 days), every event of a pending ticket +and every pending record; finished records go after 7 days. A cursor behind the compacted range +is refused (`invalid_arguments`, field `after_seq`, `evidence.compacted_through`), never served +with a silent gap. diff --git a/lab/server/pa_lab/__main__.py b/lab/server/pa_lab/__main__.py index 181b7ea5c8..3571870aef 100644 --- a/lab/server/pa_lab/__main__.py +++ b/lab/server/pa_lab/__main__.py @@ -5,6 +5,7 @@ python3 -m pa_lab kinds python3 -m pa_lab init-token ~/.config/pa-lab/token python3 -m pa_lab serve --config ~/.config/pa-lab/server.json + python3 -m pa_lab lease --config ~/.config/pa-lab/server.json show|claim-user|release-user Run from lab/server (or with lab/server on PYTHONPATH). Exit 0 valid, 1 invalid, 2 usage. """ @@ -31,6 +32,10 @@ def main(argv=None) -> int: init_token.add_argument('path') serve = sub.add_parser('serve', help='run the read-only MCP server (Streamable HTTP on 127.0.0.1)') serve.add_argument('--config', required=True) + lease = sub.add_parser('lease', help="show console B's lease, or (operator) claim or release it for the user") + lease.add_argument('--config', required=True) + lease.add_argument('action', choices=('show', 'claim-user', 'release-user')) + lease.add_argument('--console', default='B') args = parser.parse_args(argv) if args.command == 'check': @@ -63,6 +68,8 @@ def main(argv=None) -> int: return 0 if args.command == 'serve': return _serve(args.config) + if args.command == 'lease': + return _lease(args.config, args.action, args.console) parser.print_help() return 2 @@ -76,9 +83,14 @@ def _serve(path: str) -> int: print('error {}'.format(e)) return 1 logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)s %(message)s') - readers = {name: lab_reader.ConsoleReader(name, entry.lab_dir, profile=entry.profile) + from pa_lab import control, db + plane = control.ControlPlane(db.Database(cfg.db_file), sorted(cfg.consoles)) + readers = {name: lab_reader.ConsoleReader(name, entry.lab_dir, profile=entry.profile, control=plane) for name, entry in cfg.consoles.items()} - server = mcp_server.make_server(cfg.port, token, readers) + for reader in readers.values(): # restart reconciliation: apply what the app shows now + reader.console_state() + plane.compact() + server = mcp_server.make_server(cfg.port, token, readers, control_plane=plane) logging.getLogger('pa_lab.server').info('serving %s on http://127.0.0.1:%d%s', ', '.join(sorted(readers)), server.server_address[1], mcp_server.ENDPOINT) @@ -91,5 +103,28 @@ def _serve(path: str) -> int: return 0 +def _lease(path: str, action: str, console: str) -> int: + import json + from pa_lab import control, db + try: + cfg = config.load(path) + except config.ConfigError as e: + print('error {}'.format(e)) + return 1 + plane = control.ControlPlane(db.Database(cfg.db_file), sorted(cfg.consoles)) + try: + if action == 'claim-user': + lease = plane.claim_user(console) + elif action == 'release-user': + lease = plane.release_user(console) + else: + lease = plane.lease(console) + except control.Refused as e: + print('error {}'.format(e.refusal['message'])) + return 1 + print(json.dumps(lease, indent=2)) + return 0 + + if __name__ == '__main__': sys.exit(main()) diff --git a/lab/server/pa_lab/config.py b/lab/server/pa_lab/config.py index ed7cf89d4f..d193e19e27 100644 --- a/lab/server/pa_lab/config.py +++ b/lab/server/pa_lab/config.py @@ -2,6 +2,7 @@ port and the bearer token file. It never reads `~/BDSPLab/consoles.json`, which also lists A. {"schema": 1, "host": "127.0.0.1", "port": 8765, "token_file": "~/.config/pa-lab/token", + "db_file": "~/.local/state/pa-lab/control.sqlite", "consoles": {"B": {"lab_dir": "~/Library/Application Support/SerialPrograms/Profiles/lab-B/Lab", "profile": "lab-B"}}} """ @@ -17,7 +18,8 @@ ALLOWED_CONSOLES = ('B',) LOOPBACK_HOSTS = ('127.0.0.1',) -TOP_KEYS = ('schema', 'host', 'port', 'token_file', 'consoles') +TOP_KEYS = ('schema', 'host', 'port', 'token_file', 'db_file', 'consoles') +DEFAULT_DB_FILE = '~/.local/state/pa-lab/control.sqlite' CONSOLE_KEYS = ('lab_dir', 'profile') @@ -32,11 +34,13 @@ def __init__(self, lab_dir: Path, profile: Optional[str]): class Config: - def __init__(self, host: str, port: int, token_file: Path, consoles: Dict[str, ConsoleConfig]): + def __init__(self, host: str, port: int, token_file: Path, consoles: Dict[str, ConsoleConfig], + db_file: Optional[Path] = None): self.host = host self.port = port self.token_file = token_file self.consoles = consoles + self.db_file = db_file if db_file is not None else Path(os.path.expanduser(DEFAULT_DB_FILE)) def _path(value, field: str) -> Path: @@ -82,7 +86,8 @@ def load(path: Path) -> Config: if profile is not None and (not isinstance(profile, str) or not profile): raise ConfigError('{}.profile: expected a profile name'.format(field)) out[name] = ConsoleConfig(_path(entry.get('lab_dir'), field + '.lab_dir'), profile) - return Config(host, port, _path(doc.get('token_file'), 'token_file'), out) + return Config(host, port, _path(doc.get('token_file'), 'token_file'), out, + _path(doc.get('db_file', DEFAULT_DB_FILE), 'db_file')) def read_token(path: Path) -> str: diff --git a/lab/server/pa_lab/control.py b/lab/server/pa_lab/control.py new file mode 100644 index 0000000000..8164b40da3 --- /dev/null +++ b/lab/server/pa_lab/control.py @@ -0,0 +1,534 @@ +"""The durable control plane: leases, request records, tickets and events (Slice 3 of fork #131). + +Everything here is a row in the daemon's SQLite file (`pa_lab.db`), changed only inside one +`BEGIN IMMEDIATE` transaction, so a crash or a second daemon cannot split a decision: + +- **Leases** (`free`, `agent_idle`, `agent_running`, `user`; docs/design/Agent-Integration.md + "Leases, requests and events"). An agent lease is an opaque token; only its hash is stored in + the lease row. `agent_idle` expires after its TTL (1200 s by default); `agent_running` never + expires. A UI Start/Stop the app reports (`last_transition.source == 'ui'`) revokes any agent + lease and installs a `user` lease with no timeout; only the operator releases it. +- **Requests.** Every mutating call is keyed by `(client_id, tool, request_id)` and stores its + normalized arguments and terminal result. An exact retry gets the stored record back; the same + key with other arguments is refused. A `pending` record is an unresolved ticket. +- **Events** get a strictly increasing `seq` that is never reused. `next_events` returns the + events after a cursor and records that cursor; compaction never deletes an event after an + active cursor or one that belongs to an unresolved ticket. +- **Reconciliation.** `observe(app)` is applied whenever the daemon reads the app's state and + once at start: it applies each new transition exactly once (UI takeover, `stopped` events) + and ends a running lease whose app instance is gone. +""" + +from __future__ import annotations + +import hashlib +import json +import secrets +import sqlite3 +import threading +import time +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from pa_lab import contracts, timestamps +from pa_lab.db import Database + +DEFAULT_TTL_S = 1200 +MAX_WAIT_S = 60.0 +DEFAULT_PAGE = 100 +MAX_PAGE = 500 +POLL_S = 0.2 # another process's writes are seen by polling; this process's wake the waiters + + +class Refused(Exception): + """A refusal (contracts REFUSAL) the control plane could not or did not store as a record.""" + + def __init__(self, code: str, message: str, **extra: Any): + super().__init__(message) + self.refusal = dict(code=code, message=message, **extra) + + +class Retention: + """Compaction bounds. Events after an active cursor and events of unresolved tickets are kept + whatever these say; a cursor not used for `cursor_ttl_s` stops being active.""" + + def __init__(self, keep_events: int = 10000, cursor_ttl_s: float = 7 * 86400, + request_ttl_s: float = 7 * 86400, compact_every: int = 500): + self.keep_events = keep_events + self.cursor_ttl_s = cursor_ttl_s + self.request_ttl_s = request_ttl_s + self.compact_every = compact_every + + +def _hash(token: Optional[str]) -> Optional[str]: + return None if token is None else hashlib.sha256(token.encode('utf-8')).hexdigest() + + +def _json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False) + + +def _clock_time(at: str) -> str: + return at[11:16] + + +def lease_doc(row: sqlite3.Row) -> Dict[str, Any]: + return {'state': row['state'], 'owner': row['owner'], 'acquired_at': row['acquired_at'], + 'expires_at': row['expires_at']} + + +def normalized_arguments(tool: str, arguments: Dict[str, Any]) -> Dict[str, Any]: + """The arguments a record stores and a retry is compared against: defaults filled in.""" + out = dict(arguments) + if tool in ('acquire_console', 'renew_console'): + out.setdefault('ttl_s', DEFAULT_TTL_S) + return out + + +def _redacted(result: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """A result as events and status reads show it: never the lease token.""" + if result is None or 'lease_token' not in result: + return result + return {k: v for k, v in result.items() if k != 'lease_token'} + + +class ControlPlane: + def __init__(self, db: Database, consoles: Sequence[str], now=timestamps.wall_clock, + retention: Optional[Retention] = None): + self.db = db + self.consoles = tuple(consoles) + self.now = now + self.retention = retention or Retention() + self._changed = threading.Condition() + self._appended = 0 + with self.db.write() as conn: + stamp = self._stamp() + for console in self.consoles: + conn.execute("INSERT OR IGNORE INTO leases (console, state, updated_at) VALUES (?, 'free', ?)", + (console, stamp)) + + # ---- reads ------------------------------------------------------------------------------- + + def lease(self, console: str) -> Dict[str, Any]: + """The current lease, after applying a due expiry (which is a write and an event).""" + self._check_console(console) + with self._writing() as conn: + self._expire_due(conn, console) + return lease_doc(self._lease_row(conn, console)) + + def request_status(self, client_id: str, request_id: str) -> Dict[str, Any]: + """The caller's own request record (a ticket while pending); the lease token is redacted.""" + row = self.db.read().execute('SELECT * FROM requests WHERE client_id = ? AND request_id = ?', + (client_id, request_id)).fetchone() + if row is None: + raise Refused('invalid_arguments', 'No request {} from this client.'.format(request_id), + field='request_id') + record = self._record(row) + record['result'] = _redacted(record['result']) + return record + + def next_events(self, client_id: str, console: str, after_seq: Any, timeout_s: Any = 0, + limit: Any = DEFAULT_PAGE) -> Dict[str, Any]: + """Events with seq > after_seq, oldest first, waiting up to timeout_s (<= 60) for one. + + `after_seq` is also the client's confirmed position: compaction keeps everything after it, + so a client that lost a response asks again with the same cursor and loses nothing. + """ + self._check_console(console) + after_seq = self._int_arg('after_seq', after_seq, 0, None) + limit = self._int_arg('limit', limit, 1, MAX_PAGE) + if isinstance(timeout_s, bool) or not isinstance(timeout_s, (int, float)) or \ + not 0 <= timeout_s <= MAX_WAIT_S: + raise Refused('invalid_arguments', 'timeout_s must be a number of seconds 0..{:g}.'.format(MAX_WAIT_S), + field='timeout_s') + with self._writing() as conn: + head = self._head(conn) + if after_seq > head: + raise Refused('invalid_arguments', 'after_seq {} is ahead of the newest event {}.'.format( + after_seq, head), field='after_seq', evidence={'head_seq': head}) + compacted = int(self._meta(conn, 'compacted_through', '0')) + if after_seq < compacted: + raise Refused('invalid_arguments', + 'Events up to seq {} were compacted; this cursor ({}) is no longer active. ' + 'Read console://{}/state and resume from {}.'.format( + compacted, after_seq, console, compacted), + field='after_seq', evidence={'compacted_through': compacted}) + conn.execute('INSERT OR REPLACE INTO cursors (client_id, console, after_seq, seen_at) VALUES (?, ?, ?, ?)', + (client_id, console, after_seq, self._stamp())) + deadline = time.monotonic() + timeout_s + while True: + rows = self.db.read().execute( + 'SELECT * FROM events WHERE console = ? AND seq > ? ORDER BY seq LIMIT ?', + (console, after_seq, limit)).fetchall() + remaining = deadline - time.monotonic() + if rows or remaining <= 0: + break + with self._changed: + self._changed.wait(min(remaining, POLL_S)) + events = [self._event(row) for row in rows] + page = {'schema': 1, 'events': events, 'next_after_seq': events[-1]['seq'] if events else after_seq} + return contracts.validate('event_page', page).data + + # ---- requests (acquire / renew / release) ------------------------------------------------- + + def execute(self, client_id: str, request: Dict[str, Any]) -> Dict[str, Any]: + """Runs one lease request once. Returns its request_record: `completed` or `refused`, the + stored one for an exact retry. Raises Refused for a request that cannot be keyed or whose + key was used with other arguments; those are not stored.""" + req, args, token_hash = self._parse_request(client_id, request) + handler = {'acquire_console': self._acquire, 'renew_console': self._renew, + 'release_console': self._release}.get(req['tool']) + if handler is None: + raise Refused('invalid_arguments', '{} is not served before Slice 5.'.format(req['tool']), field='tool') + with self._writing() as conn: + existing = self._existing(conn, req, args, token_hash) + if existing is not None: + return existing + self._expire_due(conn, req['console']) + stamp = self._stamp() + try: + result, refusal = handler(conn, req, args, token_hash), None + except Refused as e: + result, refusal = None, e.refusal + status = 'completed' if refusal is None else 'refused' + conn.execute('INSERT INTO requests (client_id, tool, request_id, console, token_hash, status, created_at, ' + 'completed_at, arguments, result, refusal) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + (client_id, req['tool'], req['request_id'], req['console'], token_hash, status, stamp, + stamp, _json(args), None if result is None else _json(result), + None if refusal is None else _json(refusal))) + self._request_event(conn, req['console'], req['request_id'], req['tool'], result, refusal) + return self._record(self._request_row(conn, client_id, req['tool'], req['request_id'])) + + # ---- tickets (long operations; Slice 5 starts and resolves them) -------------------------- + + def open_ticket(self, client_id: str, request: Dict[str, Any]) -> Dict[str, Any]: + """Stores a `pending` record for a long operation, once. An exact retry gets the same record.""" + req, args, token_hash = self._parse_request(client_id, request) + with self._writing() as conn: + existing = self._existing(conn, req, args, token_hash) + if existing is not None: + return existing + conn.execute('INSERT INTO requests (client_id, tool, request_id, console, token_hash, status, created_at, ' + "arguments) VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)", + (client_id, req['tool'], req['request_id'], req['console'], token_hash, self._stamp(), + _json(args))) + return self._record(self._request_row(conn, client_id, req['tool'], req['request_id'])) + + def resolve_ticket(self, client_id: str, tool: str, request_id: str, result: Optional[Dict[str, Any]] = None, + refusal: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Completes (result) or fails (refusal) a pending ticket once; later calls return the record.""" + if (result is None) == (refusal is None): + raise ValueError('resolve_ticket needs exactly one of result and refusal') + with self._writing() as conn: + row = self._request_row(conn, client_id, tool, request_id) + if row is None: + raise Refused('invalid_arguments', 'No ticket {}.'.format(request_id), field='request_id') + if row['status'] != 'pending': + return self._record(row) + status = 'completed' if refusal is None else 'failed' + conn.execute('UPDATE requests SET status = ?, completed_at = ?, result = ?, refusal = ? ' + 'WHERE client_id = ? AND tool = ? AND request_id = ?', + (status, self._stamp(), None if result is None else _json(result), + None if refusal is None else _json(refusal), client_id, tool, request_id)) + record = self._record(self._request_row(conn, client_id, tool, request_id)) + self._request_event(conn, row['console'], request_id, tool, result, refusal) + return record + + # ---- runs and the operator ---------------------------------------------------------------- + + def begin_run(self, console: str, lease_token: str, request_id: str, instance_id: str) -> Dict[str, Any]: + """agent_idle -> agent_running for the lease holder (Slice 5's run_program calls this once + the app reports the start). A running lease never expires.""" + self._check_console(console) + with self._writing() as conn: + self._expire_due(conn, console) + row = self._held(conn, console, _hash(lease_token)) + if row['state'] == 'agent_running': + return lease_doc(row) + stamp = self._stamp() + self._set_lease(conn, console, 'run_started', request_id=request_id, instance_id=instance_id, + state='agent_running', expires_at=None, run_request_id=request_id, + run_instance_id=instance_id, run_started_at=stamp) + return lease_doc(self._lease_row(conn, console)) + + def claim_user(self, console: str) -> Dict[str, Any]: + """The operator takes the console by hand (CLI); same effect as a UI click.""" + self._check_console(console) + with self._writing() as conn: + self._take_over(conn, console, self._stamp(), 'operator_claim', None) + return lease_doc(self._lease_row(conn, console)) + + def release_user(self, console: str) -> Dict[str, Any]: + """The operator says the user is done (CLI only; an agent token cannot end a user lease).""" + self._check_console(console) + with self._writing() as conn: + row = self._lease_row(conn, console) + if row['state'] != 'user': + raise Refused('invalid_arguments', 'Console {} is {}, not held by the user.'.format( + console, row['state']), field='console') + self._set_lease(conn, console, 'operator_release', **self._freed()) + return lease_doc(self._lease_row(conn, console)) + + # ---- reconciliation ----------------------------------------------------------------------- + + def observe(self, console: str, app: Dict[str, Any]) -> Dict[str, Any]: + """Applies what a parsed `Lab/state.json` shows, each transition once; returns the lease. + + - a new UI transition revokes any agent lease and installs the user lease (no timeout); + - a new stop transition is a `stopped` event with its source; + - a running lease ends (back to agent_idle with a fresh TTL) when its run's stop is + reported or its app instance is gone. + """ + self._check_console(console) + instance = app['instance']['id'] + transition = app['program']['last_transition'] + key = None if transition is None else _json([instance, transition['at'], transition['action'], + transition['source'], transition['request_id']]) + with self._writing() as conn: + self._expire_due(conn, console) + seen = conn.execute('SELECT transition_key FROM observations WHERE console = ?', (console,)).fetchone() + new = transition is not None and (seen is None or seen['transition_key'] != key) + row = self._lease_row(conn, console) + if row['state'] == 'agent_running': + if instance != row['run_instance_id']: + self._end_run(conn, console, row, 'app_restarted', instance) + elif new and transition['action'] == 'stop' and transition['source'] != 'ui': + self._end_run(conn, console, row, 'run_ended', instance) + if new: + if transition['action'] == 'stop': + self._append(conn, 'stopped', console, {'source': transition['source'], + 'run_id': app['program']['run_id']}, + instance_id=instance, request_id=transition['request_id'], at=transition['at']) + if transition['source'] == 'ui': + self._take_over(conn, console, transition['at'], 'ui_takeover', instance) + conn.execute('INSERT OR REPLACE INTO observations (console, instance_id, transition_key, program_state, ' + 'observed_at) VALUES (?, ?, ?, ?, ?)', + (console, instance, key if transition is not None else + (seen['transition_key'] if seen is not None else None), + app['program']['state'], self._stamp())) + return lease_doc(self._lease_row(conn, console)) + + # ---- retention ---------------------------------------------------------------------------- + + def compact(self) -> Dict[str, int]: + """Deletes old events and finished records within the Retention bounds. Never deletes an + event after an active cursor, an event of a pending ticket, or a pending record.""" + r = self.retention + with self.db.write() as conn: + now = self.now() + stale = [(row['client_id'], row['console']) for row in conn.execute('SELECT * FROM cursors') + if now - timestamps.parse(row['seen_at']) > r.cursor_ttl_s] + conn.executemany('DELETE FROM cursors WHERE client_id = ? AND console = ?', stale) + head = conn.execute('SELECT MAX(seq) FROM events').fetchone()[0] or 0 + floor = conn.execute('SELECT MIN(after_seq) FROM cursors').fetchone()[0] + limit = min(head if floor is None else floor, head - r.keep_events) + keep = "(request_id IS NULL OR request_id NOT IN (SELECT request_id FROM requests WHERE status = 'pending'))" + through = conn.execute('SELECT MAX(seq) FROM events WHERE seq <= ? AND ' + keep, (limit,)).fetchone()[0] + events = conn.execute('DELETE FROM events WHERE seq <= ? AND ' + keep, (limit,)).rowcount + if through is not None: + old = int(self._meta(conn, 'compacted_through', '0')) + conn.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('compacted_through', ?)", + (str(max(old, through)),)) + done = [(row['client_id'], row['tool'], row['request_id']) for row in + conn.execute("SELECT client_id, tool, request_id, completed_at FROM requests WHERE status != 'pending'") + if now - timestamps.parse(row['completed_at']) > r.request_ttl_s] + conn.executemany('DELETE FROM requests WHERE client_id = ? AND tool = ? AND request_id = ?', done) + self._appended = 0 + return {'events': events, 'requests': len(done), 'cursors': len(stale)} + + # ---- internals ---------------------------------------------------------------------------- + + def _writing(self): + plane = self + + class _Write: + def __enter__(self): + self._cm = plane.db.write() + return self._cm.__enter__() + + def __exit__(self, *exc): + result = self._cm.__exit__(*exc) + if exc[0] is None: + with plane._changed: + plane._changed.notify_all() + if plane._appended >= plane.retention.compact_every: + plane.compact() + return result + + return _Write() + + def _stamp(self) -> str: + return timestamps.format(self.now()) + + def _check_console(self, console: Any) -> None: + if console not in self.consoles: + raise Refused('console_not_allowed', 'Console {!r} is not served; only {}.'.format( + console, ', '.join(self.consoles))) + + @staticmethod + def _int_arg(name: str, value: Any, minimum: int, maximum: Optional[int]) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum or \ + (maximum is not None and value > maximum): + raise Refused('invalid_arguments', '{} must be an integer {}{}.'.format( + name, minimum, '..{}'.format(maximum) if maximum is not None else ' or more'), field=name) + return value + + def _parse_request(self, client_id: str, request: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any], Optional[str]]: + self._check_console(request.get('console')) + try: + req = contracts.validate('request', dict(request, client_id=client_id)).data + except contracts.ContractError as e: + raise Refused('invalid_arguments', str(e), field=e.errors[0].path) + return req, normalized_arguments(req['tool'], req['arguments']), _hash(req.get('lease_token')) + + def _request_row(self, conn, client_id: str, tool: str, request_id: str) -> Optional[sqlite3.Row]: + return conn.execute('SELECT * FROM requests WHERE client_id = ? AND tool = ? AND request_id = ?', + (client_id, tool, request_id)).fetchone() + + def _existing(self, conn, req, args, token_hash) -> Optional[Dict[str, Any]]: + row = self._request_row(conn, req['client_id'], req['tool'], req['request_id']) + if row is None: + return None + if row['console'] != req['console'] or row['arguments'] != _json(args) or row['token_hash'] != token_hash: + raise Refused('request_id_reused', 'Request {} was already used for {} with other arguments.'.format( + req['request_id'], req['tool']), evidence={'status': row['status']}) + return self._record(row) + + @staticmethod + def _record(row: sqlite3.Row) -> Dict[str, Any]: + doc = {'schema': 1, 'request_id': row['request_id'], 'client_id': row['client_id'], 'tool': row['tool'], + 'status': row['status'], 'created_at': row['created_at'], 'completed_at': row['completed_at'], + 'arguments': json.loads(row['arguments']), + 'result': None if row['result'] is None else json.loads(row['result']), + 'refusal': None if row['refusal'] is None else json.loads(row['refusal'])} + return contracts.validate('request_record', doc).data + + @staticmethod + def _event(row: sqlite3.Row) -> Dict[str, Any]: + return {'schema': 1, 'seq': row['seq'], 'type': row['type'], 'at': row['at'], 'console': row['console'], + 'instance_id': row['instance_id'], 'request_id': row['request_id'], + 'payload': json.loads(row['payload'])} + + @staticmethod + def _meta(conn, key: str, default: str) -> str: + row = conn.execute('SELECT value FROM meta WHERE key = ?', (key,)).fetchone() + return default if row is None else row['value'] + + @staticmethod + def _head(conn) -> int: + row = conn.execute("SELECT seq FROM sqlite_sequence WHERE name = 'events'").fetchone() + return 0 if row is None else row['seq'] + + def _append(self, conn, type: str, console: str, payload: Dict[str, Any], instance_id: Optional[str] = None, + request_id: Optional[str] = None, at: Optional[str] = None) -> int: + doc = {'schema': 1, 'seq': 1, 'type': type, 'at': at or self._stamp(), 'console': console, + 'instance_id': instance_id, 'request_id': request_id, 'payload': payload} + doc = contracts.validate('event', doc).data + cursor = conn.execute('INSERT INTO events (type, at, console, instance_id, request_id, payload) ' + 'VALUES (?, ?, ?, ?, ?, ?)', + (type, doc['at'], console, instance_id, request_id, _json(doc['payload']))) + self._appended += 1 + return cursor.lastrowid + + def _request_event(self, conn, console, request_id, tool, result, refusal) -> None: + if refusal is None: + self._append(conn, 'request_completed', console, + {'request_id': request_id, 'tool': tool, 'result': _redacted(result)}, request_id=request_id) + else: + self._append(conn, 'request_failed', console, + {'request_id': request_id, 'tool': tool, 'refusal': refusal}, request_id=request_id) + + # ---- lease rows + + @staticmethod + def _lease_row(conn, console: str) -> sqlite3.Row: + return conn.execute('SELECT * FROM leases WHERE console = ?', (console,)).fetchone() + + @staticmethod + def _freed(ended_reason: Optional[str] = None, ended_token_hash: Optional[str] = None) -> Dict[str, Any]: + return dict(state='free', owner=None, client_id=None, token_hash=None, acquired_at=None, expires_at=None, + ttl_s=None, run_request_id=None, run_instance_id=None, run_started_at=None, + ended_token_hash=ended_token_hash, ended_reason=ended_reason) + + def _set_lease(self, conn, console: str, reason: str, request_id: Optional[str] = None, + instance_id: Optional[str] = None, at: Optional[str] = None, **fields: Any) -> None: + old = lease_doc(self._lease_row(conn, console)) + fields['updated_at'] = self._stamp() + assignments = ', '.join('{} = ?'.format(name) for name in fields) + conn.execute('UPDATE leases SET {} WHERE console = ?'.format(assignments), (*fields.values(), console)) + new = lease_doc(self._lease_row(conn, console)) + # `reason` is an extra payload field: the v1 contract keeps it as an unknown field. + self._append(conn, 'lease_changed', console, {'old': old, 'new': new, 'reason': reason}, + instance_id=instance_id, request_id=request_id, at=at) + + def _expire_due(self, conn, console: str) -> None: + row = self._lease_row(conn, console) + if row['state'] == 'agent_idle' and timestamps.parse(row['expires_at']) <= self.now(): + self._set_lease(conn, console, 'expired', at=row['expires_at'], + **self._freed('expired', row['token_hash'])) + + def _end_run(self, conn, console: str, row: sqlite3.Row, reason: str, instance_id: str) -> None: + self._set_lease(conn, console, reason, request_id=row['run_request_id'], instance_id=instance_id, + state='agent_idle', expires_at=timestamps.format(self.now() + row['ttl_s']), + run_request_id=None, run_instance_id=None, run_started_at=None) + + def _take_over(self, conn, console: str, at: str, reason: str, instance_id: Optional[str]) -> None: + row = self._lease_row(conn, console) + if row['state'] == 'user': + return + fields = self._freed(reason, row['token_hash'] or row['ended_token_hash']) + fields.update(state='user', owner='user', acquired_at=at) + self._set_lease(conn, console, reason, instance_id=instance_id, **fields) + + def _held(self, conn, console: str, token_hash: Optional[str]) -> sqlite3.Row: + """The lease row if `token_hash` holds it; else the refusal that says why not.""" + row = self._lease_row(conn, console) + if token_hash is not None and row['token_hash'] == token_hash and row['state'] in ('agent_idle', 'agent_running'): + return row + if token_hash is not None and row['ended_token_hash'] == token_hash: + if row['state'] == 'user' or row['ended_reason'] in ('ui_takeover', 'operator_claim'): + raise self._held_by_user(console, row) + if row['ended_reason'] == 'expired': + raise Refused('lease_expired', 'The lease on console {} expired; acquire it again.'.format(console)) + raise Refused('lease_invalid', 'That token does not hold console {}.'.format(console)) + + @staticmethod + def _held_by_user(console: str, row: sqlite3.Row) -> Refused: + return Refused('console_held_by_user', 'The user holds console {} since {}; ask them to release it.'.format( + console, _clock_time(row['acquired_at'])), evidence={'since': row['acquired_at']}) + + def _acquire(self, conn, req, args, token_hash) -> Dict[str, Any]: + console = req['console'] + row = self._lease_row(conn, console) + if row['state'] == 'user': + raise self._held_by_user(console, row) + if row['state'] != 'free': + raise Refused('console_held_by_agent', 'Console {} is leased to {} ({}).'.format( + console, row['owner'], row['state']), evidence=lease_doc(row)) + token = secrets.token_urlsafe(32) + stamp = self._stamp() + fields = self._freed() + fields.update(state='agent_idle', owner=args['owner'], client_id=req['client_id'], token_hash=_hash(token), + acquired_at=stamp, expires_at=timestamps.format(timestamps.parse(stamp) + args['ttl_s']), + ttl_s=args['ttl_s']) + self._set_lease(conn, console, 'acquired', request_id=req['request_id'], at=stamp, **fields) + return dict(lease_doc(self._lease_row(conn, console)), lease_token=token, ttl_s=args['ttl_s']) + + def _renew(self, conn, req, args, token_hash) -> Dict[str, Any]: + console = req['console'] + row = self._held(conn, console, token_hash) + if row['state'] == 'agent_idle': + conn.execute('UPDATE leases SET expires_at = ?, ttl_s = ?, updated_at = ? WHERE console = ?', + (timestamps.format(self.now() + args['ttl_s']), args['ttl_s'], self._stamp(), console)) + else: # running: never expires; the new TTL applies when the run ends + conn.execute('UPDATE leases SET ttl_s = ?, updated_at = ? WHERE console = ?', + (args['ttl_s'], self._stamp(), console)) + return dict(lease_doc(self._lease_row(conn, console)), ttl_s=args['ttl_s']) + + def _release(self, conn, req, args, token_hash) -> Dict[str, Any]: + console = req['console'] + row = self._held(conn, console, token_hash) + if row['state'] == 'agent_running': + raise Refused('program_running', 'A run holds console {}; stop it before releasing.'.format(console)) + self._set_lease(conn, console, 'released', request_id=req['request_id'], + **self._freed('released', row['token_hash'])) + return lease_doc(self._lease_row(conn, console)) diff --git a/lab/server/pa_lab/db.py b/lab/server/pa_lab/db.py new file mode 100644 index 0000000000..dfe018620f --- /dev/null +++ b/lab/server/pa_lab/db.py @@ -0,0 +1,175 @@ +"""The daemon's SQLite file: migrations, connections and write transactions (Slice 3 of fork #131). + +SQLite is the source of truth for leases, request records (idempotency and tickets), events and +event cursors; JSONL is only ever an export. Every write runs in `BEGIN IMMEDIATE`, so two +daemons, two threads or a daemon and the operator CLI that share the file serialize on SQLite's +lock: there is no read-then-write window in which both could win a lease. A process that dies +mid-transaction leaves nothing behind (the journal rolls it back on the next open). + +The schema version is `PRAGMA user_version`. `MIGRATIONS[i]` takes the file from version `i` to +`i + 1`; a file newer than this code is refused rather than guessed at. +""" + +from __future__ import annotations + +from contextlib import contextmanager +import os +from pathlib import Path +import sqlite3 +import threading +from typing import Iterator, List + +BUSY_TIMEOUT_S = 30.0 + +MIGRATIONS: List[List[str]] = [ + # 1: leases, requests (idempotency + tickets), events, cursors, observations, meta. + [ + """CREATE TABLE leases ( + console TEXT PRIMARY KEY, + state TEXT NOT NULL CHECK (state IN ('free', 'agent_idle', 'agent_running', 'user')), + owner TEXT, + client_id TEXT, + token_hash TEXT, + acquired_at TEXT, + expires_at TEXT, + ttl_s INTEGER, + run_request_id TEXT, + run_instance_id TEXT, + run_started_at TEXT, + ended_token_hash TEXT, + ended_reason TEXT, + updated_at TEXT NOT NULL + )""", + # One row per mutating call; status 'pending' is an unresolved ticket. + """CREATE TABLE requests ( + client_id TEXT NOT NULL, + tool TEXT NOT NULL, + request_id TEXT NOT NULL, + console TEXT NOT NULL, + token_hash TEXT, + status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'refused', 'failed')), + created_at TEXT NOT NULL, + completed_at TEXT, + arguments TEXT NOT NULL, + result TEXT, + refusal TEXT, + PRIMARY KEY (client_id, tool, request_id) + )""", + "CREATE INDEX requests_by_id ON requests (client_id, request_id)", + "CREATE INDEX requests_unresolved ON requests (status) WHERE status = 'pending'", + # AUTOINCREMENT: a seq is never reused, even after compaction deletes the newest rows. + """CREATE TABLE events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + at TEXT NOT NULL, + console TEXT NOT NULL, + instance_id TEXT, + request_id TEXT, + payload TEXT NOT NULL + )""", + "CREATE INDEX events_by_request ON events (request_id) WHERE request_id IS NOT NULL", + # The last after_seq each client confirmed; compaction never deletes past an active one. + """CREATE TABLE cursors ( + client_id TEXT NOT NULL, + console TEXT NOT NULL, + after_seq INTEGER NOT NULL, + seen_at TEXT NOT NULL, + PRIMARY KEY (client_id, console) + )""", + # The last app transition applied per console, so one UI click is applied exactly once. + """CREATE TABLE observations ( + console TEXT PRIMARY KEY, + instance_id TEXT, + transition_key TEXT, + program_state TEXT, + observed_at TEXT NOT NULL + )""", + "CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)", + ], +] + +SCHEMA_VERSION = len(MIGRATIONS) + + +class DatabaseError(RuntimeError): + pass + + +def _connect(path: Path) -> sqlite3.Connection: + conn = sqlite3.connect(str(path), timeout=BUSY_TIMEOUT_S, isolation_level=None, + check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.execute('PRAGMA journal_mode=WAL') + conn.execute('PRAGMA synchronous=FULL') + return conn + + +def _create_owner_only(path: Path) -> None: + """The file holds lease tokens of in-flight acquires: owner-only from its first byte.""" + path.parent.mkdir(parents=True, exist_ok=True) + try: + os.close(os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) + except FileExistsError: + pass + + +def migrate(conn: sqlite3.Connection) -> int: + """Brings the file to SCHEMA_VERSION in one transaction per step; returns the version found.""" + conn.execute('BEGIN IMMEDIATE') + try: + found = conn.execute('PRAGMA user_version').fetchone()[0] + if found > SCHEMA_VERSION: + raise DatabaseError('the control database is at schema {}; this daemon knows up to {}'.format( + found, SCHEMA_VERSION)) + for version in range(found, SCHEMA_VERSION): + for statement in MIGRATIONS[version]: + conn.execute(statement) + conn.execute('PRAGMA user_version = {:d}'.format(version + 1)) + conn.execute('COMMIT') + except BaseException: + conn.execute('ROLLBACK') + raise + return found + + +class Database: + """One SQLite file shared by every thread of a daemon (one connection per thread).""" + + def __init__(self, path: Path): + self.path = Path(path) + _create_owner_only(self.path) + self._local = threading.local() + self._all: List[sqlite3.Connection] = [] + self._all_lock = threading.Lock() + self.found_version = migrate(self.conn()) + + def conn(self) -> sqlite3.Connection: + conn = getattr(self._local, 'conn', None) + if conn is None: + conn = _connect(self.path) + self._local.conn = conn + with self._all_lock: + self._all.append(conn) + return conn + + @contextmanager + def write(self) -> Iterator[sqlite3.Connection]: + """`BEGIN IMMEDIATE` ... `COMMIT`: the whole block is one atomic, serialized write.""" + conn = self.conn() + conn.execute('BEGIN IMMEDIATE') + try: + yield conn + except BaseException: + conn.execute('ROLLBACK') + raise + conn.execute('COMMIT') + + def read(self) -> sqlite3.Connection: + return self.conn() + + def close(self) -> None: + with self._all_lock: + for conn in self._all: + conn.close() + self._all = [] + self._local = threading.local() diff --git a/lab/server/pa_lab/lab_reader.py b/lab/server/pa_lab/lab_reader.py index 0ec55ec185..7c3cd14a3b 100644 --- a/lab/server/pa_lab/lab_reader.py +++ b/lab/server/pa_lab/lab_reader.py @@ -6,8 +6,10 @@ (no file, a torn or malformed file, an unsupported schema, another profile's app, a dead pid, a stale heartbeat) is an explicit `unavailable` state with the reason, never a guess. -Slice 2 has no lease store (that is Slice 3), so the lease is only reported: the app's last -transition from a UI Start/Stop means the user holds the console, with no timeout. +With a control plane (Slice 3, `pa_lab.control`) the lease comes from its SQLite store, and every +read of a live app is applied to it first (`observe`: UI takeover, run end, `stopped` events). +Without one (Slice 2) the lease is only reported: the app's last transition from a UI Start/Stop +means the user holds the console, with no timeout. """ from __future__ import annotations @@ -111,8 +113,10 @@ class ConsoleReader: def __init__(self, console: str, lab_dir: Path, profile: Optional[str] = None, now: Callable[[], float] = timestamps.wall_clock, pid_alive: Callable[[int], bool] = pid_alive, - heartbeat_timeout_s: float = HEARTBEAT_TIMEOUT_S): + heartbeat_timeout_s: float = HEARTBEAT_TIMEOUT_S, + control: Optional[Any] = None): self.console = console + self.control = control self.lab_dir = Path(lab_dir) self.profile = profile self.now = now @@ -150,11 +154,12 @@ def console_state(self) -> Dict[str, Any]: try: app = self.app_state() except Unavailable as e: + lease = self.control.lease(self.console) if self.control is not None else free_lease() doc = {'schema': 1, 'console': self.console, 'observed_at': observed_at, 'availability': {'state': 'unavailable', 'reason': e.reason}, - 'app': None, 'lease': free_lease(), 'blocked': [_app_unavailable(e)]} + 'app': None, 'lease': lease, 'blocked': [_app_unavailable(e)]} return contracts.validate('console_state', doc).data - lease = reported_lease(app) + lease = self.control.observe(self.console, app) if self.control is not None else reported_lease(app) doc = {'schema': 1, 'console': self.console, 'observed_at': observed_at, 'availability': {'state': 'available', 'reason': None}, 'app': app, 'lease': lease, 'blocked': blocked(self.console, app, lease)} @@ -209,9 +214,13 @@ def blocked(console: str, app: Dict[str, Any], lease: Dict[str, Any]) -> List[Di out = [] if lease['state'] == 'user': transition = app['program']['last_transition'] - out.append({'code': 'console_held_by_user', 'message': - 'The user clicked {} in the app at {}; console {} is theirs until they release it.'.format( - transition['action'].capitalize(), _clock_time(transition['at']), console)}) + if transition is not None and transition['source'] == 'ui' and transition['at'] == lease['acquired_at']: + message = 'The user clicked {} in the app at {}; console {} is theirs until they release it.'.format( + transition['action'].capitalize(), _clock_time(transition['at']), console) + else: + message = 'The user holds console {} since {}; it is theirs until they release it.'.format( + console, _clock_time(lease['acquired_at'])) + out.append({'code': 'console_held_by_user', 'message': message}) state = app['program']['state'] panel = app['panel']['id'] if state == 'no_program': diff --git a/lab/server/pa_lab/mcp_server.py b/lab/server/pa_lab/mcp_server.py index 6e1383eb7d..2ef34b088b 100644 --- a/lab/server/pa_lab/mcp_server.py +++ b/lab/server/pa_lab/mcp_server.py @@ -6,8 +6,9 @@ `Mcp-Session-Id`. The bearer token is never logged or returned. Resources: `console://B/state` (console_state), `console://B/save` (save_resource). -Tools: `shot`, `ensure_game`. Nothing here presses a button, starts or stops a program, or -holds a lease: those arrive in Slices 3 and 5. +Tools: `shot`, `ensure_game`; with a control plane (Slice 3) also `acquire_console`, +`renew_console`, `release_console`, `next_events` and `request_status`. Nothing here presses a +button or starts or stops a program: that arrives in Slice 5. """ from __future__ import annotations @@ -24,7 +25,7 @@ import threading from typing import Any, Callable, Dict, Optional, Tuple -from pa_lab import contracts, shot, tools +from pa_lab import contracts, control, shot, tools from pa_lab.lab_reader import ConsoleReader, Unavailable LOG = logging.getLogger('pa_lab.server') @@ -89,6 +90,50 @@ ] +_REQUEST_ID = {'type': 'string', 'description': 'client-generated lower-case UUID; a retry reuses it'} +_LEASE_TOKEN = {'type': 'string', 'description': 'the token acquire_console returned'} +_TTL = {'type': 'integer', 'minimum': 60, 'maximum': 3600, 'default': control.DEFAULT_TTL_S} +_MUTATING = {'readOnlyHint': False, 'destructiveHint': False, 'idempotentHint': True} +_READ = {'readOnlyHint': True, 'destructiveHint': False, 'idempotentHint': True} + + +def _schema(properties: Dict[str, Any], required) -> Dict[str, Any]: + return {'type': 'object', 'properties': dict({'console': {'type': 'string', 'enum': ['B']}}, **properties), + 'required': ['console'] + list(required), 'additionalProperties': False} + + +CONTROL_TOOL_LIST = [ + {'name': 'acquire_console', 'title': 'Lease the console', + 'description': 'Takes the free console for this client and returns an opaque lease_token and its expiry ' + '(agent_idle, ttl_s, default 1200 s; renew to keep it). Refused while the user or another ' + 'agent holds it. A retry with the same request_id returns the same result.', + 'inputSchema': _schema({'request_id': _REQUEST_ID, 'owner': {'type': 'string'}, 'ttl_s': _TTL}, + ['request_id', 'owner']), 'annotations': _MUTATING}, + {'name': 'renew_console', 'title': 'Renew the lease', + 'description': 'Extends the lease to now + ttl_s. A running lease never expires.', + 'inputSchema': _schema({'request_id': _REQUEST_ID, 'lease_token': _LEASE_TOKEN, 'ttl_s': _TTL}, + ['request_id', 'lease_token']), 'annotations': _MUTATING}, + {'name': 'release_console', 'title': 'Release the lease', + 'description': 'Frees the console. A user lease is released only by the operator.', + 'inputSchema': _schema({'request_id': _REQUEST_ID, 'lease_token': _LEASE_TOKEN}, + ['request_id', 'lease_token']), 'annotations': _MUTATING}, + {'name': 'next_events', 'title': 'Events after a cursor', + 'description': 'Returns the events with seq > after_seq (oldest first) and next_after_seq, waiting up to ' + 'timeout_s for one. Pass the last next_after_seq you processed: events after it are kept.', + 'inputSchema': _schema({'after_seq': {'type': 'integer', 'minimum': 0}, + 'timeout_s': {'type': 'number', 'minimum': 0, 'maximum': control.MAX_WAIT_S, + 'default': 0}, + 'limit': {'type': 'integer', 'minimum': 1, 'maximum': control.MAX_PAGE, + 'default': control.DEFAULT_PAGE}}, + ['after_seq']), 'annotations': _READ}, + {'name': 'request_status', 'title': 'Request or ticket status', + 'description': "This client's request record: pending, completed, refused or failed.", + 'inputSchema': _schema({'request_id': _REQUEST_ID}, ['request_id']), 'annotations': _READ}, +] + +LEASE_TOOLS = ('acquire_console', 'renew_console', 'release_console') + + class RpcError(Exception): def __init__(self, code: int, message: str, data: Any = None): super().__init__(message) @@ -110,11 +155,13 @@ class LabMcp: """JSON-RPC methods, without the transport. `consoles` is the allowlist: console id -> reader.""" def __init__(self, consoles: Dict[str, ConsoleReader], - take_shot: Callable[..., Dict[str, Any]] = shot.take_shot): + take_shot: Callable[..., Dict[str, Any]] = shot.take_shot, + control_plane: Optional[control.ControlPlane] = None): self.consoles = consoles self.take_shot = take_shot + self.control = control_plane - def call(self, method: str, params: Dict[str, Any]) -> Dict[str, Any]: + def call(self, method: str, params: Dict[str, Any], client: str = 'unnamed') -> Dict[str, Any]: if method == 'ping': return {} if method == 'resources/list': @@ -124,8 +171,10 @@ def call(self, method: str, params: Dict[str, Any]) -> Dict[str, Any]: if method == 'resources/read': return self._read_resource(params.get('uri')) if method == 'tools/list': - return {'tools': TOOL_LIST} + return {'tools': TOOL_LIST + (CONTROL_TOOL_LIST if self.control is not None else [])} if method == 'tools/call': + if self.control is not None and params.get('name') in [t['name'] for t in CONTROL_TOOL_LIST]: + return self._call_control(client, params.get('name'), params.get('arguments', {})) return self._call_tool(params.get('name'), params.get('arguments', {})) raise RpcError(METHOD_NOT_FOUND, 'method not found: {}'.format(method)) @@ -199,9 +248,47 @@ def _call_tool(self, name: Any, arguments: Any) -> Dict[str, Any]: {'type': 'text', 'text': _json_text(result)}], 'structuredContent': result, 'isError': False} + def _call_control(self, client: str, name: str, arguments: Any) -> Dict[str, Any]: + if not isinstance(arguments, dict): + return self._refused(name, {'code': 'invalid_arguments', 'field': 'arguments', + 'message': 'expected an object'}) + args = dict(arguments) + console = args.pop('console', None) + request_id = args.get('request_id') if isinstance(args.get('request_id'), str) else None + try: + if name == 'next_events': + unknown = sorted(set(args) - {'after_seq', 'timeout_s', 'limit'}) + if unknown: + raise control.Refused('invalid_arguments', 'unknown argument', field=unknown[0]) + if 'after_seq' not in args: + raise control.Refused('invalid_arguments', 'required', field='after_seq') + result = self.control.next_events(client, console, args['after_seq'], args.get('timeout_s', 0), + args.get('limit', control.DEFAULT_PAGE)) + elif name == 'request_status': + if set(args) != {'request_id'} or request_id is None: + raise control.Refused('invalid_arguments', 'expects console and request_id', field='request_id') + result = self.control.request_status(client, request_id) + else: + request = {'schema': 1, 'request_id': args.pop('request_id', None), 'tool': name, + 'console': console, 'arguments': args} + if 'lease_token' in args: + request['lease_token'] = args.pop('lease_token') + record = self.control.execute(client, request) + if record['refusal'] is not None: + return self._refused(name, record['refusal'], record['request_id']) + result = record + except control.Refused as e: + return self._refused(name, e.refusal, request_id if name in LEASE_TOOLS else None) + return {'content': [{'type': 'text', 'text': _json_text(result)}], 'structuredContent': result, + 'isError': False} + @staticmethod - def _refused(tool: str, refusal: Dict[str, Any]) -> Dict[str, Any]: - doc = refusal_doc(tool, refusal) + def _refused(tool: str, refusal: Dict[str, Any], request_id: Optional[str] = None) -> Dict[str, Any]: + try: + doc = contracts.validate('refusal', {'schema': 1, 'request_id': request_id, 'tool': tool, + 'refusal': refusal}).data + except contracts.ContractError: # a request_id that is not a UUID is not echoed + doc = refusal_doc(tool, refusal) return {'content': [{'type': 'text', 'text': _json_text(doc)}], 'structuredContent': doc, 'isError': True} @@ -365,7 +452,8 @@ def do_POST(self): try: if not isinstance(params, dict): raise RpcError(INVALID_PARAMS, 'params must be an object') - reply = {'jsonrpc': '2.0', 'id': message['id'], 'result': self.server.mcp.call(method, params)} + reply = {'jsonrpc': '2.0', 'id': message['id'], + 'result': self.server.mcp.call(method, params, client=found[1])} except RpcError as e: error = {'code': e.code, 'message': e.message} if e.data is not None: diff --git a/lab/server/tests/control_fixtures.py b/lab/server/tests/control_fixtures.py new file mode 100644 index 0000000000..ba72e6644d --- /dev/null +++ b/lab/server/tests/control_fixtures.py @@ -0,0 +1,60 @@ +"""Shared setup for the Slice 3 control-plane tests: a temp SQLite file and request builders.""" + +import copy +from pathlib import Path +import tempfile +import uuid + +from pa_lab import control, db +from tests.fake_lab import FakeClock, golden + +START = '2026-11-14T06:00:00.000Z' + + +def temp_db_path(test) -> Path: + tmp = tempfile.TemporaryDirectory(prefix='pa-lab-control-') + test.addCleanup(tmp.cleanup) + return Path(tmp.name) / 'state' / 'control.sqlite' + + +def open_plane(test, path, clock, retention=None): + database = db.Database(path) + test.addCleanup(database.close) + return control.ControlPlane(database, ['B'], now=clock.now, retention=retention) + + +def new_id(): + return str(uuid.uuid4()) + + +def acquire(owner='agent:frlg-lane', request_id=None, **arguments): + return {'schema': 1, 'request_id': request_id or new_id(), 'tool': 'acquire_console', 'console': 'B', + 'arguments': dict({'owner': owner}, **arguments)} + + +def renew(token, request_id=None, **arguments): + return {'schema': 1, 'request_id': request_id or new_id(), 'tool': 'renew_console', 'console': 'B', + 'lease_token': token, 'arguments': arguments} + + +def release(token, request_id=None): + return {'schema': 1, 'request_id': request_id or new_id(), 'tool': 'release_console', 'console': 'B', + 'lease_token': token, 'arguments': {}} + + +def app_state(transition=None, state='stopped', instance=None): + """The running_lab golden with this program state, last transition and instance id.""" + doc = copy.deepcopy(golden('app_state', 'running_lab')) + doc['program']['state'] = state + doc['program']['last_transition'] = transition + if instance is not None: + doc['instance']['id'] = instance + return doc + + +def transition(action, source, at, request_id=None): + return {'action': action, 'source': source, 'request_id': request_id, 'at': at} + + +def clock(): + return FakeClock(START) diff --git a/lab/server/tests/test_control.py b/lab/server/tests/test_control.py new file mode 100644 index 0000000000..2f8385886c --- /dev/null +++ b/lab/server/tests/test_control.py @@ -0,0 +1,392 @@ +"""The control plane on one SQLite file: leases, retries, user takeover, restart reconciliation, +events and compaction (Slice 3 of fork #131). A FakeClock drives every TTL; restarts reopen the +same file with a new ControlPlane, as a restarted daemon would. + +Run from the repo root: python3 -m unittest discover -s lab/server/tests -t lab/server -v +""" + +import threading +import time +import unittest + +from pa_lab import contracts, control +from tests.control_fixtures import (acquire, app_state, clock, new_id, open_plane, release, renew, + temp_db_path, transition) + +INSTANCE = 'd8ea6a9d-1eab-4a40-b355-d88072b289a1' +NEW_INSTANCE = '0b5e3f6c-9d2a-4e71-8c40-6f1a2b3c4d5e' + + +class PlaneTest(unittest.TestCase): + def setUp(self): + self.clock = clock() + self.path = temp_db_path(self) + self.plane = open_plane(self, self.path, self.clock) + + def restart(self, retention=None): + """A new daemon on the same file: nothing carried over but the SQLite rows.""" + self.plane = open_plane(self, self.path, self.clock, retention) + return self.plane + + def acquired(self, **arguments): + record = self.plane.execute('claude:frlg-lane', acquire(**arguments)) + self.assertEqual(record['status'], 'completed', record) + return record['result']['lease_token'] + + def events(self, after=0): + return self.plane.next_events('reader', 'B', after)['events'] + + def types(self, after=0): + return [(e['type'], e['payload'].get('reason')) for e in self.events(after)] + + +class LeaseTests(PlaneTest): + def test_acquire_gives_an_idle_lease_with_the_default_ttl(self): + record = self.plane.execute('claude:frlg-lane', acquire()) + self.assertEqual(record['result']['state'], 'agent_idle') + self.assertEqual(record['result']['expires_at'], '2026-11-14T06:20:00.000Z') + self.assertEqual(record['arguments'], {'owner': 'agent:frlg-lane', 'ttl_s': 1200}) + self.assertEqual(self.plane.lease('B')['owner'], 'agent:frlg-lane') + + def test_a_held_console_is_refused_to_everyone_else(self): + self.acquired() + cases = [ + dict(name='another agent', input=('claude:bdsp-lane', 'agent:bdsp-lane'), + expectation='console_held_by_agent'), + dict(name='the same owner name without the token', input=('claude:frlg-lane', 'agent:frlg-lane'), + expectation='console_held_by_agent'), + ] + for case in cases: + with self.subTest(case['name']): + client, owner = case['input'] + record = self.plane.execute(client, acquire(owner)) + self.assertEqual((record['status'], record['refusal']['code']), ('refused', case['expectation'])) + + def test_renew_moves_the_expiry(self): + token = self.acquired() + self.clock.sleep(1000) + record = self.plane.execute('claude:frlg-lane', renew(token, ttl_s=600)) + self.assertEqual(record['result']['expires_at'], '2026-11-14T06:26:40.000Z') + self.clock.sleep(500) + self.assertEqual(self.plane.lease('B')['state'], 'agent_idle') + + def test_an_idle_lease_expires_after_its_ttl(self): + token = self.acquired() + self.clock.sleep(1200) + self.assertEqual(self.plane.lease('B')['state'], 'free') + record = self.plane.execute('claude:frlg-lane', renew(token)) + self.assertEqual(record['refusal']['code'], 'lease_expired') + self.assertIn(('lease_changed', 'expired'), self.types()) + + def test_tokens_that_do_not_hold_the_console_are_refused(self): + token = self.acquired() + cases = [ + dict(name='made-up token', input=renew('not-a-token'), expectation='lease_invalid'), + dict(name='release with a made-up token', input=release('not-a-token'), expectation='lease_invalid'), + ] + for case in cases: + with self.subTest(case['name']): + record = self.plane.execute('claude:frlg-lane', case['input']) + self.assertEqual(record['refusal']['code'], case['expectation']) + self.assertEqual(self.plane.execute('claude:frlg-lane', release(token))['result']['state'], 'free') + self.assertEqual(self.plane.execute('claude:frlg-lane', renew(token))['refusal']['code'], 'lease_invalid') + + def test_a_running_lease_never_expires(self): + token = self.acquired() + self.plane.begin_run('B', token, new_id(), INSTANCE) + self.clock.sleep(10 * 3600) + self.plane.observe('B', app_state(transition('start', 'lab', '2026-11-14T06:00:01.000Z', new_id()), + 'running', INSTANCE)) + lease = self.plane.lease('B') + self.assertEqual((lease['state'], lease['expires_at']), ('agent_running', None)) + self.assertEqual(self.plane.execute('claude:frlg-lane', release(token))['refusal']['code'], + 'program_running') + + def test_a_reported_stop_ends_the_run_with_a_fresh_ttl(self): + token = self.acquired() + run = new_id() + self.plane.begin_run('B', token, run, INSTANCE) + self.clock.sleep(3600) + self.plane.observe('B', app_state(transition('stop', 'program', '2026-11-14T07:00:00.000Z'), 'stopped', + INSTANCE)) + lease = self.plane.lease('B') + self.assertEqual((lease['state'], lease['expires_at']), ('agent_idle', '2026-11-14T07:20:00.000Z')) + self.assertEqual(self.types()[-2:], [('lease_changed', 'run_ended'), ('stopped', None)]) + + +class RetryTests(PlaneTest): + def test_an_exact_retry_returns_the_stored_result(self): + request = acquire() + first = self.plane.execute('claude:frlg-lane', request) + self.clock.sleep(5) + again = self.plane.execute('claude:frlg-lane', request) + self.assertEqual(again, first) + self.assertEqual([t for t, _ in self.types()].count('request_completed'), 1) + + def test_an_exact_retry_after_a_restart_returns_the_stored_result(self): + request = acquire() + first = self.plane.execute('claude:frlg-lane', request) + self.restart() + self.assertEqual(self.plane.execute('claude:frlg-lane', request), first) + + def test_a_refusal_is_stable_too(self): + self.acquired() + request = acquire('agent:bdsp-lane') + first = self.plane.execute('claude:bdsp-lane', request) + self.clock.sleep(1300) # the console is free now, but the stored answer stands + self.assertEqual(self.plane.execute('claude:bdsp-lane', request), first) + + def test_reusing_a_request_id_with_other_arguments_is_refused(self): + request = acquire() + self.plane.execute('claude:frlg-lane', request) + cases = [ + dict(name='other owner', input=dict(request, arguments={'owner': 'agent:other'})), + dict(name='other ttl', input=dict(request, arguments={'owner': 'agent:frlg-lane', 'ttl_s': 600})), + ] + for case in cases: + with self.subTest(case['name']): + with self.assertRaises(control.Refused) as caught: + self.plane.execute('claude:frlg-lane', case['input']) + self.assertEqual(caught.exception.refusal['code'], 'request_id_reused') + + def test_the_default_ttl_is_part_of_the_normalized_arguments(self): + request = acquire() + first = self.plane.execute('claude:frlg-lane', request) + explicit = dict(request, arguments={'owner': 'agent:frlg-lane', 'ttl_s': 1200}) + self.assertEqual(self.plane.execute('claude:frlg-lane', explicit), first) + + def test_status_reads_and_events_never_show_the_lease_token(self): + request = acquire() + token = self.plane.execute('claude:frlg-lane', request)['result']['lease_token'] + status = self.plane.request_status('claude:frlg-lane', request['request_id']) + self.assertNotIn('lease_token', status['result']) + for event in self.events(): + self.assertNotIn(token, str(event)) + + def test_a_status_read_of_another_clients_request_is_refused(self): + request = acquire() + self.plane.execute('claude:frlg-lane', request) + with self.assertRaises(control.Refused): + self.plane.request_status('claude:bdsp-lane', request['request_id']) + + def test_an_invalid_request_is_refused_and_not_stored(self): + cases = [ + dict(name='no owner', input=dict(acquire(), arguments={}), expectation='arguments.owner'), + dict(name='ttl over an hour', input=acquire(ttl_s=7200), expectation='arguments.ttl_s'), + dict(name='token on acquire', input=dict(acquire(), lease_token='x'), expectation='lease_token'), + ] + for case in cases: + with self.subTest(case['name']): + with self.assertRaises(control.Refused) as caught: + self.plane.execute('claude:frlg-lane', case['input']) + self.assertEqual(caught.exception.refusal['field'], case['expectation']) + self.assertEqual(self.events(), []) + + +class UserTakeoverTests(PlaneTest): + UI_STOP = transition('stop', 'ui', '2026-11-14T06:10:10.002Z') + + def test_a_ui_click_revokes_the_agent_lease(self): + cases = [ + dict(name='idle lease, UI stop', input=('idle', self.UI_STOP)), + dict(name='running lease, UI stop', input=('running', self.UI_STOP)), + dict(name='idle lease, UI start', input=('idle', transition('start', 'ui', '2026-11-14T06:10:10.002Z'))), + ] + for case in cases: + with self.subTest(case['name']): + self.setUp() + held, click = case['input'] + token = self.acquired() + if held == 'running': + self.plane.begin_run('B', token, new_id(), INSTANCE) + lease = self.plane.observe('B', app_state(click, 'stopped', INSTANCE)) + self.assertEqual(lease, {'state': 'user', 'owner': 'user', 'acquired_at': click['at'], + 'expires_at': None}) + self.assertEqual(self.types()[-1], ('lease_changed', 'ui_takeover')) + self.assertEqual(self.plane.execute('claude:frlg-lane', renew(token))['refusal']['code'], + 'console_held_by_user') + + def test_the_user_lease_has_no_timeout_and_survives_restarts(self): + self.plane.observe('B', app_state(self.UI_STOP, 'stopped', INSTANCE)) + self.clock.sleep(30 * 86400) + self.restart() + self.plane.observe('B', app_state(None, 'stopped', NEW_INSTANCE)) + self.assertEqual(self.plane.lease('B')['state'], 'user') + record = self.plane.execute('claude:frlg-lane', acquire()) + self.assertEqual(record['refusal']['code'], 'console_held_by_user') + + def test_only_the_operator_ends_a_user_lease_and_the_same_click_is_not_applied_again(self): + app = app_state(self.UI_STOP, 'stopped', INSTANCE) + self.plane.observe('B', app) + self.assertEqual(self.plane.release_user('B')['state'], 'free') + self.assertEqual(self.plane.observe('B', app)['state'], 'free') + self.acquired() + + def test_the_ui_stop_is_one_stopped_event(self): + for _ in range(3): + self.plane.observe('B', app_state(self.UI_STOP, 'stopped', INSTANCE)) + stopped = [e for e in self.events() if e['type'] == 'stopped'] + self.assertEqual([e['payload']['source'] for e in stopped], ['ui']) + + def test_the_operator_can_claim_the_console(self): + token = self.acquired() + self.assertEqual(self.plane.claim_user('B')['state'], 'user') + self.assertEqual(self.plane.execute('claude:frlg-lane', release(token))['refusal']['code'], + 'console_held_by_user') + + +class ReconciliationTests(PlaneTest): + def test_an_idle_lease_and_its_token_survive_a_restart(self): + token = self.acquired() + self.restart() + self.assertEqual(self.plane.execute('claude:frlg-lane', renew(token))['status'], 'completed') + + def test_a_running_lease_survives_a_restart_while_its_app_lives(self): + token = self.acquired() + self.plane.begin_run('B', token, new_id(), INSTANCE) + self.clock.sleep(7200) + self.restart() + lease = self.plane.observe('B', app_state(transition('start', 'lab', '2026-11-14T06:00:01.000Z', new_id()), + 'running', INSTANCE)) + self.assertEqual(lease['state'], 'agent_running') + + def test_a_running_lease_whose_app_restarted_goes_back_to_idle(self): + token = self.acquired() + self.plane.begin_run('B', token, new_id(), INSTANCE) + self.restart() + lease = self.plane.observe('B', app_state(None, 'stopped', NEW_INSTANCE)) + self.assertEqual(lease['state'], 'agent_idle') + self.assertEqual(self.types()[-1], ('lease_changed', 'app_restarted')) + + def test_a_ui_click_while_the_daemon_was_down_is_applied_on_restart(self): + token = self.acquired() + self.restart() + lease = self.plane.observe('B', app_state(UserTakeoverTests.UI_STOP, 'stopped', INSTANCE)) + self.assertEqual(lease['state'], 'user') + self.assertEqual(self.plane.execute('claude:frlg-lane', renew(token))['refusal']['code'], + 'console_held_by_user') + + def test_an_expiry_due_during_downtime_is_applied_once(self): + self.acquired() + self.clock.sleep(5000) + self.restart() + self.restart() + self.assertEqual(self.plane.lease('B')['state'], 'free') + self.assertEqual(self.types().count(('lease_changed', 'expired')), 1) + + def test_tickets_survive_a_restart_and_resolve_once(self): + ticket = {'schema': 1, 'request_id': new_id(), 'tool': 'stop_program', 'console': 'B', + 'lease_token': 't', 'arguments': {}} + self.assertEqual(self.plane.open_ticket('claude:frlg-lane', ticket)['status'], 'pending') + self.restart() + self.assertEqual(self.plane.request_status('claude:frlg-lane', ticket['request_id'])['status'], 'pending') + done = self.plane.resolve_ticket('claude:frlg-lane', 'stop_program', ticket['request_id'], + result={'stopped_at': '2026-11-14T06:00:02.000Z'}) + again = self.plane.resolve_ticket('claude:frlg-lane', 'stop_program', ticket['request_id'], + refusal={'code': 'timeout', 'message': 'late'}) + self.assertEqual((done['status'], again), ('completed', done)) + self.assertEqual([t for t, _ in self.types()], ['request_completed']) + + +class EventTests(PlaneTest): + def test_every_event_and_page_meets_the_v1_contract(self): + token = self.acquired() + self.plane.begin_run('B', token, new_id(), INSTANCE) + self.plane.observe('B', app_state(UserTakeoverTests.UI_STOP, 'stopped', INSTANCE)) + page = self.plane.next_events('reader', 'B', 0) + contracts.validate('event_page', page) + self.assertEqual([e['seq'] for e in page['events']], list(range(1, len(page['events']) + 1))) + + def test_catch_up_in_small_pages_is_gap_free_across_restarts(self): + for _ in range(7): + self.plane.execute('claude:frlg-lane', acquire()) + seen, cursor = [], 0 + for _ in range(20): + page = self.plane.next_events('reader', 'B', cursor, limit=3) + if not page['events']: + break + seen.extend(e['seq'] for e in page['events']) + cursor = page['next_after_seq'] + self.restart() + self.assertEqual(seen, list(range(1, 9))) # one acquire (2 events), six refusals + self.assertEqual(self.plane.next_events('reader', 'B', cursor)['next_after_seq'], cursor) + + def test_a_cursor_ahead_of_the_newest_event_is_refused(self): + with self.assertRaises(control.Refused) as caught: + self.plane.next_events('reader', 'B', 5) + self.assertEqual(caught.exception.refusal['field'], 'after_seq') + + def test_a_waiting_reader_wakes_on_a_new_event(self): + got = {} + reader = threading.Thread(target=lambda: got.update(page=self.plane.next_events('reader', 'B', 0, 10))) + reader.start() + time.sleep(0.1) + self.plane.execute('claude:frlg-lane', acquire()) + reader.join(5) + self.assertFalse(reader.is_alive()) + self.assertEqual(got['page']['events'][0]['seq'], 1) + + def test_a_wait_with_nothing_new_returns_the_same_cursor(self): + self.plane.execute('claude:frlg-lane', acquire()) + started = time.monotonic() + page = self.plane.next_events('reader', 'B', 2, timeout_s=0.3) + self.assertGreaterEqual(time.monotonic() - started, 0.25) + self.assertEqual(page, {'schema': 1, 'events': [], 'next_after_seq': 2}) + + def test_seqs_are_never_reused_after_compaction(self): + self.plane.execute('claude:frlg-lane', acquire()) + self.restart(control.Retention(keep_events=0)).compact() + self.plane.execute('claude:bdsp-lane', acquire('agent:bdsp-lane')) + self.assertEqual([e['seq'] for e in self.plane.next_events('reader', 'B', 2)['events']], [3]) + + +class CompactionTests(PlaneTest): + def fill(self, n): + for _ in range(n): + self.plane.execute('claude:frlg-lane', acquire('agent:x')) + + def test_events_after_an_active_cursor_are_kept(self): + self.fill(10) # 2 events (acquire) + 9 request_failed = 11 + self.plane.next_events('slow-reader', 'B', 4) + self.restart(control.Retention(keep_events=0)) + self.assertEqual(self.plane.compact()['events'], 4) + page = self.plane.next_events('slow-reader', 'B', 4) + self.assertEqual([e['seq'] for e in page['events']], list(range(5, 12))) + + def test_a_cursor_behind_compaction_is_refused_with_the_resume_point(self): + self.fill(5) + self.restart(control.Retention(keep_events=2)).compact() + with self.assertRaises(control.Refused) as caught: + self.plane.next_events('late-reader', 'B', 0) + self.assertEqual(caught.exception.refusal['evidence'], {'compacted_through': 4}) + self.assertEqual(len(self.plane.next_events('late-reader', 'B', 4)['events']), 2) + + def test_an_unused_cursor_stops_protecting_events(self): + self.fill(5) + self.plane.next_events('gone-reader', 'B', 0) + self.clock.sleep(8 * 86400) + self.restart(control.Retention(keep_events=0)) + self.assertEqual(self.plane.compact(), {'events': 6, 'requests': 5, 'cursors': 1}) + + def test_unresolved_tickets_and_their_events_are_kept(self): + ticket = {'schema': 1, 'request_id': new_id(), 'tool': 'stop_program', 'console': 'B', + 'lease_token': 't', 'arguments': {}} + self.plane.open_ticket('claude:frlg-lane', ticket) + with self.plane.db.write() as conn: # an event of the in-flight request (Slice 5 emits these) + self.plane._append(conn, 'stopped', 'B', {'source': 'lab'}, request_id=ticket['request_id']) + self.fill(3) + self.clock.sleep(30 * 86400) + self.restart(control.Retention(keep_events=0)).compact() + self.assertEqual(self.plane.request_status('claude:frlg-lane', ticket['request_id'])['status'], 'pending') + kept = self.plane.db.read().execute('SELECT seq, request_id FROM events').fetchall() + self.assertEqual([(r['seq'], r['request_id']) for r in kept], [(1, ticket['request_id'])]) + + def test_compaction_runs_by_itself_after_enough_events(self): + self.restart(control.Retention(keep_events=3, compact_every=4)) + self.fill(6) + count = self.plane.db.read().execute('SELECT COUNT(*) FROM events').fetchone()[0] + self.assertLessEqual(count, 6) + + +if __name__ == '__main__': + unittest.main() diff --git a/lab/server/tests/test_control_concurrency.py b/lab/server/tests/test_control_concurrency.py new file mode 100644 index 0000000000..757dcb9370 --- /dev/null +++ b/lab/server/tests/test_control_concurrency.py @@ -0,0 +1,171 @@ +"""Concurrent clients and crashed daemons on one control database (Slice 3, fork #131): one lease +winner, one result per request key, and nothing half-written after a kill. + +Threads each open their own Database (their own SQLite connection), as separate daemons would; +the process tests start real Python processes on the same temp file. +""" + +import json +from pathlib import Path +import subprocess +import sys +import threading +import unittest + +from pa_lab import control, db +from tests.control_fixtures import acquire, clock, new_id, open_plane, temp_db_path + +SERVER_DIR = Path(__file__).resolve().parent.parent +RACERS = 8 + + +def race(count, work): + """Runs `work(i)` on `count` threads released together; returns results in thread order.""" + barrier = threading.Barrier(count) + results = [None] * count + + def run(i): + barrier.wait(10) + try: + results[i] = work(i) + except Exception as e: # surfaced to the test as a value + results[i] = e + + threads = [threading.Thread(target=run, args=(i,)) for i in range(count)] + for t in threads: + t.start() + for t in threads: + t.join(30) + return results + + +def run_python(code, *args): + return subprocess.run([sys.executable, '-c', code, *[str(a) for a in args]], cwd=str(SERVER_DIR), + capture_output=True, text=True, timeout=60) + + +ACQUIRE_IN_A_PROCESS = """ +import json, sys +from pa_lab import control, db +plane = control.ControlPlane(db.Database(sys.argv[1]), ['B']) +request = {'schema': 1, 'request_id': sys.argv[2], 'tool': 'acquire_console', 'console': 'B', + 'arguments': {'owner': sys.argv[3]}} +print(json.dumps(plane.execute(sys.argv[3], request))) +""" + +CRASH_MID_WRITE = """ +import os, sys +from pa_lab import control, db +plane = control.ControlPlane(db.Database(sys.argv[1]), ['B']) +with plane.db.write() as conn: + conn.execute("UPDATE leases SET state = 'agent_idle', owner = 'half', acquired_at = '2026-11-14T06:00:00.000Z', " + "expires_at = '2026-11-14T07:00:00.000Z' WHERE console = 'B'") + conn.execute("INSERT INTO events (type, at, console, payload) VALUES ('stopped', '2026-11-14T06:00:00.000Z', 'B', '{}')") + os._exit(9) +""" + + +class RaceTests(unittest.TestCase): + def setUp(self): + self.clock = clock() + self.path = temp_db_path(self) + self.plane = open_plane(self, self.path, self.clock) + + def planes(self, count): + return [open_plane(self, self.path, self.clock) for _ in range(count)] + + def test_concurrent_acquires_have_one_winner(self): + planes = self.planes(RACERS) + records = race(RACERS, lambda i: planes[i].execute('claude:lane{}'.format(i), acquire('agent:lane{}'.format(i)))) + statuses = sorted(r['status'] for r in records) + self.assertEqual(statuses, ['completed'] + ['refused'] * (RACERS - 1)) + winner = next(r for r in records if r['status'] == 'completed') + self.assertEqual(self.plane.lease('B')['owner'], winner['arguments']['owner']) + self.assertEqual({r['refusal']['code'] for r in records if r['refusal']}, {'console_held_by_agent'}) + + def test_concurrent_retries_of_one_request_get_one_result(self): + planes = self.planes(RACERS) + request = acquire() + records = race(RACERS, lambda i: planes[i].execute('claude:frlg-lane', request)) + self.assertTrue(all(r == records[0] for r in records), records) + page = self.plane.next_events('reader', 'B', 0) + self.assertEqual([e['type'] for e in page['events']], ['lease_changed', 'request_completed']) + + def test_acquires_racing_an_expiry_have_one_winner_and_one_expiry(self): + self.plane.execute('claude:old', acquire('agent:old')) + self.clock.sleep(1200) + planes = self.planes(RACERS) + records = race(RACERS, lambda i: planes[i].execute('claude:lane{}'.format(i), acquire('agent:lane{}'.format(i)))) + self.assertEqual([r['status'] for r in records].count('completed'), 1) + reasons = [e['payload'].get('reason') for e in self.plane.next_events('reader', 'B', 0)['events'] + if e['type'] == 'lease_changed'] + self.assertEqual(reasons, ['acquired', 'expired', 'acquired']) + + def test_concurrent_readers_see_every_event_once_in_order(self): + planes = self.planes(4) + + def read_all(i): + seen, cursor = [], 0 + for _ in range(200): + page = planes[i].next_events('reader{}'.format(i), 'B', cursor, timeout_s=0.5, limit=2) + seen.extend(e['seq'] for e in page['events']) + cursor = page['next_after_seq'] + if len(seen) >= 2 + RACERS - 1: + break + return seen + + readers = [] + results = {} + for i in range(4): + t = threading.Thread(target=lambda i=i: results.update({i: read_all(i)})) + t.start() + readers.append(t) + writers = self.planes(RACERS) + race(RACERS, lambda i: writers[i].execute('claude:lane{}'.format(i), acquire('agent:lane{}'.format(i)))) + for t in readers: + t.join(30) + expected = list(range(1, 2 + RACERS)) + self.assertEqual(results, {i: expected for i in range(4)}) + + +class ProcessTests(unittest.TestCase): + def setUp(self): + self.path = temp_db_path(self) + db.Database(self.path).close() + + def test_racing_processes_have_one_winner(self): + procs = [subprocess.Popen([sys.executable, '-c', ACQUIRE_IN_A_PROCESS, str(self.path), new_id(), + 'agent:p{}'.format(i)], cwd=str(SERVER_DIR), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) for i in range(4)] + outputs = [p.communicate(timeout=60) for p in procs] + self.assertEqual([p.returncode for p in procs], [0] * 4, [err for _, err in outputs]) + statuses = sorted(json.loads(out)['status'] for out, _ in outputs) + self.assertEqual(statuses, ['completed', 'refused', 'refused', 'refused']) + + def test_a_retry_from_another_process_gets_the_stored_result(self): + request_id = new_id() + first = run_python(ACQUIRE_IN_A_PROCESS, self.path, request_id, 'agent:p') + again = run_python(ACQUIRE_IN_A_PROCESS, self.path, request_id, 'agent:p') + self.assertEqual(json.loads(first.stdout), json.loads(again.stdout)) + self.assertEqual(json.loads(first.stdout)['status'], 'completed') + + def test_a_daemon_killed_mid_write_leaves_nothing(self): + crashed = run_python(CRASH_MID_WRITE, self.path) + self.assertEqual(crashed.returncode, 9, crashed.stderr) + plane = control.ControlPlane(db.Database(self.path), ['B']) + self.addCleanup(plane.db.close) + self.assertEqual(plane.lease('B')['state'], 'free') + self.assertEqual(plane.next_events('reader', 'B', 0)['events'], []) + + def test_a_lease_from_a_killed_daemon_is_still_held(self): + out = run_python(ACQUIRE_IN_A_PROCESS + '\nimport os\nos._exit(9)\n', self.path, new_id(), 'agent:p') + self.assertEqual(out.returncode, 9, out.stderr) + plane = control.ControlPlane(db.Database(self.path), ['B']) + self.addCleanup(plane.db.close) + self.assertEqual(plane.lease('B')['owner'], 'agent:p') + record = plane.execute('claude:other', acquire('agent:other')) + self.assertEqual(record['refusal']['code'], 'console_held_by_agent') + + +if __name__ == '__main__': + unittest.main() diff --git a/lab/server/tests/test_control_mcp.py b/lab/server/tests/test_control_mcp.py new file mode 100644 index 0000000000..5965ea55fc --- /dev/null +++ b/lab/server/tests/test_control_mcp.py @@ -0,0 +1,100 @@ +"""The control tools over the MCP endpoint (Slice 3, fork #131): lease calls with request ids, +console://B/state reading the stored lease, a UI click in the fake app becoming the user lease, +and event catch-up.""" + +import json +import threading +import time +import unittest +import uuid + +from pa_lab import control, db, lab_reader, mcp_server, timestamps +from tests.control_fixtures import temp_db_path +from tests.fake_lab import FakeApp, FakeLab, golden +from tests.test_mcp_server import TOKEN, Client, WallClock + + +class ControlServerTest(unittest.TestCase): + def setUp(self): + self.lab = FakeLab(self) + state = golden('app_state', 'stopped_ui') + state['program']['last_transition'] = None + state['written_at'] = timestamps.format(time.time()) + state['video']['captured_at'] = state['written_at'] + self.lab.write_state(state) + self.app = FakeApp(self.lab, WallClock()) + database = db.Database(temp_db_path(self)) + self.addCleanup(database.close) + self.plane = control.ControlPlane(database, ['B']) + console = lab_reader.ConsoleReader('B', self.lab.dir, profile='lab-B', pid_alive=lambda pid: True, + control=self.plane) + self.server = mcp_server.make_server(0, TOKEN, {'B': console}, control_plane=self.plane) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.addCleanup(self._shutdown) + self.client = Client(self.server.server_address[1]) + self.client.initialize() + + def _shutdown(self): + self.server.shutdown() + self.server.server_close() + + def heartbeat(self, **changes): + changes['written_at'] = timestamps.format(time.time()) + self.lab.update_state(**changes) + + def test_the_control_tools_are_listed(self): + _, body = self.client.rpc('tools/list') + names = [t['name'] for t in body['result']['tools']] + self.assertEqual(names, ['shot', 'ensure_game', 'acquire_console', 'renew_console', 'release_console', + 'next_events', 'request_status']) + + def test_acquire_then_state_shows_the_lease_and_a_retry_matches(self): + request_id = str(uuid.uuid4()) + first = self.client.tool('acquire_console', console='B', request_id=request_id, owner='agent:frlg-lane') + again = self.client.tool('acquire_console', console='B', request_id=request_id, owner='agent:frlg-lane') + self.assertFalse(first['isError']) + self.assertEqual(first['structuredContent'], again['structuredContent']) + self.heartbeat() + lease = self.client.read('console://B/state')['lease'] + self.assertEqual((lease['state'], lease['owner']), ('agent_idle', 'agent:frlg-lane')) + status = self.client.tool('request_status', console='B', request_id=request_id)['structuredContent'] + self.assertNotIn('lease_token', status['result']) + + def test_a_ui_click_in_the_app_becomes_the_user_lease(self): + token = self.client.tool('acquire_console', console='B', request_id=str(uuid.uuid4()), + owner='agent:frlg-lane')['structuredContent']['result']['lease_token'] + at = timestamps.format(time.time()) + self.heartbeat(**{'program.last_transition': {'action': 'stop', 'source': 'ui', 'request_id': None, + 'at': at}}) + state = self.client.read('console://B/state') + self.assertEqual(state['lease']['state'], 'user') + self.assertEqual(state['blocked'][0]['code'], 'console_held_by_user') + renewed = self.client.tool('renew_console', console='B', request_id=str(uuid.uuid4()), lease_token=token) + self.assertTrue(renewed['isError']) + self.assertEqual(renewed['structuredContent']['refusal']['code'], 'console_held_by_user') + page = self.client.tool('next_events', console='B', after_seq=0)['structuredContent'] + self.assertEqual([e['type'] for e in page['events']], + ['lease_changed', 'request_completed', 'stopped', 'lease_changed', 'request_failed']) + self.assertNotIn(token, json.dumps(page)) + + def test_bad_control_calls_are_refusals(self): + cases = [ + dict(name='console A', input=('acquire_console', dict(console='A', request_id=str(uuid.uuid4()), + owner='x')), expectation='console_not_allowed'), + dict(name='request id not a uuid', input=('acquire_console', dict(console='B', request_id='r1', owner='x')), + expectation='invalid_arguments'), + dict(name='unknown next_events argument', input=('next_events', dict(console='B', after_seq=0, wait=1)), + expectation='invalid_arguments'), + dict(name='wait over 60 s', input=('next_events', dict(console='B', after_seq=0, timeout_s=61)), + expectation='invalid_arguments'), + ] + for case in cases: + with self.subTest(case['name']): + name, arguments = case['input'] + result = self.client.tool(name, **arguments) + self.assertTrue(result['isError']) + self.assertEqual(result['structuredContent']['refusal']['code'], case['expectation']) + + +if __name__ == '__main__': + unittest.main() diff --git a/lab/server/tests/test_db.py b/lab/server/tests/test_db.py new file mode 100644 index 0000000000..2901299dad --- /dev/null +++ b/lab/server/tests/test_db.py @@ -0,0 +1,55 @@ +"""The control database file: migrations, version guard and permissions (Slice 3, fork #131).""" + +import os +import sqlite3 +import stat +import unittest + +from pa_lab import db +from tests.control_fixtures import temp_db_path + + +class MigrationTests(unittest.TestCase): + def test_a_new_file_is_migrated_to_the_current_version(self): + database = db.Database(temp_db_path(self)) + self.addCleanup(database.close) + conn = database.read() + tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} + self.assertEqual(conn.execute('PRAGMA user_version').fetchone()[0], db.SCHEMA_VERSION) + self.assertTrue({'leases', 'requests', 'events', 'cursors', 'observations', 'meta'} <= tables) + self.assertEqual(database.found_version, 0) + + def test_reopening_a_migrated_file_changes_nothing(self): + path = temp_db_path(self) + db.Database(path).close() + again = db.Database(path) + self.addCleanup(again.close) + self.assertEqual(again.found_version, db.SCHEMA_VERSION) + + def test_a_file_from_a_newer_daemon_is_refused(self): + path = temp_db_path(self) + db.Database(path).close() + conn = sqlite3.connect(str(path)) + conn.execute('PRAGMA user_version = {}'.format(db.SCHEMA_VERSION + 1)) + conn.commit() + conn.close() + with self.assertRaisesRegex(db.DatabaseError, 'schema {}'.format(db.SCHEMA_VERSION + 1)): + db.Database(path) + + def test_the_file_is_owner_only(self): + path = temp_db_path(self) + db.Database(path).close() + self.assertEqual(stat.S_IMODE(os.stat(path).st_mode) & 0o077, 0) + + def test_a_failed_write_leaves_nothing(self): + database = db.Database(temp_db_path(self)) + self.addCleanup(database.close) + with self.assertRaises(RuntimeError): + with database.write() as conn: + conn.execute("INSERT INTO meta (key, value) VALUES ('k', 'v')") + raise RuntimeError('crash between statements') + self.assertIsNone(database.read().execute("SELECT value FROM meta WHERE key = 'k'").fetchone()) + + +if __name__ == '__main__': + unittest.main()