From c927723978765d32add6f4984c249e119ecd2c9c Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 31 Oct 2026 23:17:57 -0700 Subject: [PATCH 01/16] Prototype an agent interface over live program sessions --- .../Source/CommonFramework/ProgramSession.h | 4 +- .../CommonFramework/Windows/LabChannel.cpp | 15 ++- .../Source/Integrations/ProgramTracker.cpp | 6 +- .../Integrations/ProgramTrackerInterfaces.h | 4 +- .../SessionAgentPrototype/PrototypeDemo.cpp | 104 ++++++++++++++++++ .../SessionAgentPrototype/README.md | 98 +++++++++++++++++ .../SessionAgentPrototype.cpp | 64 +++++++++++ .../SessionAgentPrototype.h | 81 ++++++++++++++ SerialPrograms/cmake/SourceFiles.cmake | 2 + devtools/prototypes/session-agent/run-demo.sh | 9 ++ 10 files changed, 377 insertions(+), 10 deletions(-) create mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp create mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md create mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp create mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h create mode 100644 devtools/prototypes/session-agent/run-demo.sh diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.h b/SerialPrograms/Source/CommonFramework/ProgramSession.h index af12d8c820..664709cd3e 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.h +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.h @@ -114,8 +114,8 @@ class ProgramSession : public TrackableProgram{ std::string stop_program(); // Returns before the program is completely stopped. // Temporary - virtual void async_start() override{ start_program(); } - virtual void async_stop() override{ stop_program(); } + virtual std::string async_start() override{ return start_program(); } + virtual std::string async_stop() override{ return stop_program(); } protected: diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabChannel.cpp b/SerialPrograms/Source/CommonFramework/Windows/LabChannel.cpp index 1a1651dff6..b4f386a6dd 100644 --- a/SerialPrograms/Source/CommonFramework/Windows/LabChannel.cpp +++ b/SerialPrograms/Source/CommonFramework/Windows/LabChannel.cpp @@ -17,6 +17,7 @@ #include "CommonFramework/Logging/Logger.h" #include "CommonFramework/ProgramSession.h" #include "CommonFramework/Tools/GlobalThreadPools.h" +#include "Integrations/SessionAgentPrototype/SessionAgentPrototype.h" #include "LabChannel.h" namespace PokemonAutomation{ @@ -48,6 +49,16 @@ void write_file_atomically(const Filesystem::Path& path, const std::string& text Filesystem::rename(temp, path, ec); } +// The prototype runs through the same owner check as other channel writes. +std::string run_session_agent_request(const LabRequestLine& line, LabCommandOutput& output){ + const size_t begin = line.text.find_first_not_of(" \t"); + const size_t space = line.text.find_first_of(" \t", begin); + const std::string text = space == std::string::npos ? "" : line.text.substr(space + 1); + const auto reply = SessionAgentPrototype::handle_live_request(text); + output.out << reply.dump() << "\n"; + return reply.at("ok").get() ? "" : reply.at("error").at("message").get(); +} + // Returns the error message of the start or stop, or empty on success. // Unlike the Start/Stop button, a refused start (e.g. an option that fails // check_validity()) goes to the journal, so the request stops there. @@ -342,10 +353,10 @@ std::string LabChannel::run_line(const LabRequest& request, const LabRequestLine if (command == "keepawake"){ return m_keep_awake.set(words, current_time()); } - if (kind == LabConsoleCommands::Kind::UNKNOWN){ + if (kind == LabConsoleCommands::Kind::UNKNOWN && command != "session_agent"){ return "Unknown command: " + command + " (try \"help\")"; } - return run_console_command(words, output); + return command == "session_agent" ? run_session_agent_request(line, output) : run_console_command(words, output); } std::string LabChannel::run_console_command(const std::vector& words, LabCommandOutput& output){ diff --git a/SerialPrograms/Source/Integrations/ProgramTracker.cpp b/SerialPrograms/Source/Integrations/ProgramTracker.cpp index ca3c242416..b7734cb55c 100644 --- a/SerialPrograms/Source/Integrations/ProgramTracker.cpp +++ b/SerialPrograms/Source/Integrations/ProgramTracker.cpp @@ -173,8 +173,7 @@ std::string ProgramTracker::start_program(uint64_t program_id){ if (program == nullptr){ return error; } - program->async_start(); - return ""; + return program->async_start(); } std::string ProgramTracker::stop_program(uint64_t program_id){ std::string header = make_header("stop_program", program_id); @@ -184,8 +183,7 @@ std::string ProgramTracker::stop_program(uint64_t program_id){ if (program == nullptr){ return error; } - program->async_stop(); - return ""; + return program->async_stop(); } std::string ProgramTracker::run_controller_command( uint64_t console_id, uint64_t controller_index, diff --git a/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h b/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h index 9d4f4c3125..e771bb80cb 100644 --- a/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h +++ b/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h @@ -36,8 +36,8 @@ class TrackableProgram{ virtual ProgramState current_state() const = 0; virtual std::string current_stats() const = 0; - virtual void async_start() = 0; - virtual void async_stop() = 0; + virtual std::string async_start() = 0; + virtual std::string async_stop() = 0; }; diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp b/SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp new file mode 100644 index 0000000000..152f04a120 --- /dev/null +++ b/SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp @@ -0,0 +1,104 @@ +/* Program Session Agent - Hardware-Free Terminal Demo + * + * From: https://github.com/PokemonAutomation/ + * + * Simulated lifecycle only. The JSON dispatcher is shared with the live adapter. + */ + +#include +#include +#include +#include "SessionAgentPrototype.h" + +using namespace PokemonAutomation::SessionAgentPrototype; + +namespace{ + +struct DemoSession{ + std::string state = "STOPPED"; + std::string start_error; +}; + +std::string operate(std::map& sessions, uint64_t id, const std::string& action){ + auto it = sessions.find(id); + if (it == sessions.end()){ + return "Session ID not found."; + } + DemoSession& session = it->second; + if (action == "start"){ + if (session.state != "STOPPED"){ + return "Program is " + session.state + "."; + } + if (!session.start_error.empty()){ + return session.start_error; + } + session.state = "RUNNING"; + }else{ + if (session.state != "RUNNING"){ + return "Program is " + session.state + "."; + } + session.state = "STOPPING"; + } + return ""; +} + +Json frame(const std::map& sessions){ + Json result = Json::array(); + for (const auto& [id, session] : sessions){ + result.push_back({ + {"session_id", id}, {"program", "Prototype:Session"}, + {"state", session.state}, {"start_error", session.start_error}, + {"console_ids", Json::array()}, {"stats", ""} + }); + } + return result; +} + +Json demo_action(std::map& sessions, Bindings& bindings, const std::string& line){ + if (line == "finish"){ + for (auto& [id, session] : sessions){ + if (session.state == "STOPPING"){ + session.state = "STOPPED"; + } + } + return {{"demo", "Stop completion simulated."}}; + } + if (line == "unload"){ + sessions.erase(1); + return {{"demo", "Session 1 unloaded; its ID is now stale."}}; + } + try{ + return dispatch(bindings, Json::parse(line)); + }catch (const Json::exception& e){ + return failure("invalid_request", e.what()); + } +} + +} + +int main(int argc, char** argv){ + const bool plain = argc > 1 && std::string(argv[1]) == "--plain"; + std::map sessions{ + {1, {.state = "STOPPED", .start_error = ""}}, + {2, {.state = "STOPPED", .start_error = "Example configuration is invalid."}}, + }; + Bindings bindings{ + .list = [&]{ return frame(sessions); }, + .start = [&](uint64_t id){ return operate(sessions, id, "start"); }, + .stop = [&](uint64_t id){ return operate(sessions, id, "stop"); }, + }; + Json reply; + std::string line; + while (true){ + if (!plain){ + std::cout << "\033[2J\033[H\033[1mTHROWAWAY PROTOTYPE: simulated sessions, no console\033[0m\n"; + } + std::cout << "sessions: " << frame(sessions).dump(2) << "\nreply: " << reply.dump(2) << "\n"; + std::cout << "JSON method: sessions.list/status/start/stop; session_id for all except list\n" + << "demo controls: finish | unload | quit\n" << std::flush; + if (!std::getline(std::cin, line) || line == "quit"){ + break; + } + reply = demo_action(sessions, bindings, line); + } +} diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md b/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md new file mode 100644 index 0000000000..a2ca555883 --- /dev/null +++ b/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md @@ -0,0 +1,98 @@ +# ProgramSession agent interface — throwaway proof of concept + +Question: can an agent list existing sessions and request start/stop by explicit +session ID, receiving the real rejection reason and distinguishing acceptance +from completion, without owning a special automation program? + +This prototype adds four JSON methods and a lab-channel adapter. It reuses +`ProgramTracker`; it does not introduce a second session registry. The same +dispatcher can be driven in a hardware-free terminal demo. Delete or absorb +this prototype once the interface decision is made. + +## Try the terminal demo + +From the repo root, with a C++23 compiler: + +```sh +sh devtools/prototypes/session-agent/run-demo.sh +``` + +This compiles only the small native demo, into a temporary directory. It does +not build SerialPrograms, connect hardware, or control a running app. Session 1 +can start; session 2 has a simulated configuration error. Enter one request per line: + +```json +{"method":"sessions.list"} +{"method":"sessions.start","session_id":2} +{"method":"sessions.start","session_id":1} +{"method":"sessions.status","session_id":1} +{"method":"sessions.stop","session_id":1} +{"method":"sessions.status","session_id":1} +``` + +The rejected start reports `ok:false`. The accepted stop leaves session 1 in +`STOPPING`; the demo-only command `finish` simulates its eventual completion. +`unload` removes session 1, so requests using that ID fail. `quit` exits; +`--plain` disables terminal clearing for a piped walkthrough. Full simulated +state and the last reply are displayed after each action. + +## Live app adapter + +A build of this branch accepts `session_agent ` through the existing +lab channel. Select and configure a suitable program in the app first. Within +the lab's existing console lock and owner session: + +```sh +printf '%s\n' 'session_agent {"method":"sessions.list"}' | + sh devtools/lab/labctl.sh send sessions-list +``` + +The journal contains a JSON result, followed by its normal `ok`/`end` lines. +Take the `session_id` from that result, then use it explicitly: + +```sh +printf '%s\n' 'session_agent {"method":"sessions.status","session_id":1}' | + sh devtools/lab/labctl.sh send session-status +``` + +Use `sessions.start`/`sessions.stop` with the same shape only for the program +under test: these invoke its actual lifecycle and may produce game/save effects. +When the channel is claimed, all prototype requests require `LAB_OWNER`, +including observations. A failed request emits both the JSON error and the +channel's `error` line, so the batch stops and `labctl.sh` exits unsuccessfully. + +## Reply contract + +- `sessions.list`: current live session snapshots, each with `session_id`, + `program` (descriptor identifier), `state`, `stats`, and `console_ids`. +- `sessions.status`: one snapshot or `session_not_found`. +- `sessions.start` / `sessions.stop`: `operation_rejected` with the session's + actual error, or `{"ok":true,"result":{"session_id":1,"accepted":true}}`. +- Poll status to observe the outcome; `accepted` is not completion. A program + may also finish between polls. There is no per-run history in this prototype. +- IDs belong to this app process. Rediscover after restart. IDs 0, UINT64_MAX, + negative numbers, fractions and booleans are rejected; no implicit selection. + +## What this proves, and what it does not + +`SessionAgentPrototype.h` contains the transport-independent dispatcher; +`SessionAgentPrototype.cpp` binds it to the live tracker. `PrototypeDemo.cpp` +supplies simulated bindings and is excluded from the app build. + +One prerequisite fixes an existing information loss: `TrackableProgram`'s +`async_start`/`async_stop` now return their error strings, and the tracker +propagates them. `ProgramSession` already supplies those strings. Existing +integration callers can now see a rejected operation too. + +This draft is not an MCP server or JSON-RPC implementation. It does not add +program creation/selection, option editing, direct controller inputs, run IDs, +event subscriptions, an operation queue, or a new lifetime/ownership scheme. +The live adapter uses the existing tracker's locking and destruction contract; +concurrent session destruction still needs an explicit audit before production. +Multiple clients are coordinated only by the existing lab-channel mechanism. + +The terminal walkthrough demonstrates that the four-method interface can expose +rejection, accepted-but-still-stopping, and stale IDs. The remaining experiment +is to exercise the live adapter with a deliberately invalid configuration and +a suitable short program, then decide whether to add per-run results and events +before attaching MCP. No live-console validation is claimed for this draft. diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp b/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp new file mode 100644 index 0000000000..eabbf27052 --- /dev/null +++ b/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp @@ -0,0 +1,64 @@ +/* Program Session Agent - Live Prototype Adapter + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/Exceptions.h" +#include "CommonFramework/Globals.h" +#include "Integrations/ProgramTracker.h" +#include "SessionAgentPrototype.h" + +namespace PokemonAutomation{ +namespace SessionAgentPrototype{ +namespace{ + +const char* state_name(ProgramState state){ + switch (state){ + case ProgramState::NOT_READY: + return "NOT_READY"; + case ProgramState::STOPPED: + return "STOPPED"; + case ProgramState::RUNNING: + return "RUNNING"; + case ProgramState::STOPPING: + return "STOPPING"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown program state."); +} + +Json list_live_sessions(){ + Json sessions = Json::array(); + for (const auto& [id, state] : ProgramTracker::instance().all_programs()){ + sessions.push_back({ + {"session_id", id}, {"program", state.program_name}, + {"state", state_name(state.state)}, {"stats", state.stats}, + {"console_ids", state.console_ids} + }); + } + return sessions; +} + +} + +Json handle_live_request(const std::string& text){ + Bindings bindings{ + .list = list_live_sessions, + .start = [](uint64_t id){ return ProgramTracker::instance().start_program(id); }, + .stop = [](uint64_t id){ return ProgramTracker::instance().stop_program(id); }, + }; + try{ + const Json request = Json::parse(text); + return dispatch(bindings, request); + }catch (const Json::exception& e){ + return failure("invalid_request", e.what()); + }catch (Exception& e){ + return failure("operation_failed", e.message()); + }catch (const std::exception& e){ + return failure("operation_failed", e.what()); + } +} + +} +} diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h b/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h new file mode 100644 index 0000000000..8dcd028d80 --- /dev/null +++ b/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h @@ -0,0 +1,81 @@ +/* Program Session Agent - Throwaway Prototype + * + * From: https://github.com/PokemonAutomation/ + * + * Four JSON requests over existing live sessions. No transport or UI here. + * See README.md for the experiment and its limits. + */ + +#ifndef PokemonAutomation_SessionAgentPrototype_H +#define PokemonAutomation_SessionAgentPrototype_H + +#include +#include +#include +#include +#include "3rdParty-Core/nlohmann/json.hpp" + +namespace PokemonAutomation{ +namespace SessionAgentPrototype{ + +using Json = nlohmann::json; + +struct Bindings{ + std::function list; + std::function start; + std::function stop; +}; + +inline Json failure(const std::string& code, const std::string& message){ + return {{"ok", false}, {"error", {{"code", code}, {"message", message}}}}; +} + +inline Json status(Bindings& bindings, uint64_t id){ + for (const Json& session : bindings.list()){ + if (session.at("session_id").get() == id){ + return {{"ok", true}, {"result", session}}; + } + } + return failure("session_not_found", "The session is no longer registered."); +} + +// Returns an immediate reply; accepted start/stop is not run completion. +// Bindings own session lookup/lifetime protection. No session pointer escapes. +inline Json dispatch(Bindings& bindings, const Json& request){ + if (!request.is_object() || !request.contains("method") || !request["method"].is_string()){ + return failure("invalid_request", "Expected an object with a string method."); + } + const std::string method = request["method"].get(); + if (method == "sessions.list"){ + return {{"ok", true}, {"result", bindings.list()}}; + } + if (method != "sessions.status" && method != "sessions.start" && method != "sessions.stop"){ + return failure("unknown_method", "Use sessions.list, sessions.status, sessions.start or sessions.stop."); + } + if (!request.contains("session_id") || !request["session_id"].is_number_integer()){ + return failure("invalid_session_id", "Expected a positive integer session_id."); + } + if (!request["session_id"].is_number_unsigned() && request["session_id"].get() < 0){ + return failure("invalid_session_id", "Expected a positive integer session_id."); + } + const uint64_t id = request["session_id"].get(); + // ProgramTracker reserves UINT64_MAX for implicit selection. Never use it. + if (id == 0 || id == std::numeric_limits::max()){ + return failure("invalid_session_id", "Explicit live session_id required; implicit selection is disabled."); + } + if (method == "sessions.status"){ + return status(bindings, id); + } + const std::string error = method == "sessions.start" ? bindings.start(id) : bindings.stop(id); + if (!error.empty()){ + return failure("operation_rejected", error); + } + return {{"ok", true}, {"result", {{"session_id", id}, {"accepted", true}}}}; +} + +// Live app adapter. Input is JSON; exceptions become explicit error replies. +Json handle_live_request(const std::string& text); + +} +} +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 2809a35342..92cb4fe507 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -950,6 +950,8 @@ file(GLOB LIBRARY_SOURCES Source/Integrations/IntegrationsAPI.h Source/Integrations/ProgramTracker.cpp Source/Integrations/ProgramTracker.h + Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp + Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h Source/GameConsole/ConsoleHandle.cpp Source/GameConsole/ConsoleHandle.h Source/GameConsole/ConsolePanel.cpp diff --git a/devtools/prototypes/session-agent/run-demo.sh b/devtools/prototypes/session-agent/run-demo.sh new file mode 100644 index 0000000000..cde8eb73f7 --- /dev/null +++ b/devtools/prototypes/session-agent/run-demo.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Standalone native C++ demo; does not build or launch SerialPrograms. +set -eu +repo=$(CDPATH= cd -- "$(dirname -- "$0")/../../.." && pwd) +build_dir=$(mktemp -d "${TMPDIR:-/tmp}/pa-session-agent.XXXXXX") +"${CXX:-c++}" -std=c++23 -Wall -Wextra -Werror -I "$repo" \ + "$repo/SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp" \ + -o "$build_dir/demo" +exec "$build_dir/demo" "$@" From 47f27261b57b7055277eb44405bebfee37c71407 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 31 Oct 2026 23:25:21 -0700 Subject: [PATCH 02/16] Document concrete AutoStory examples for the session agent prototype --- .../SessionAgentPrototype/README.md | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md b/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md index a2ca555883..c818bf58da 100644 --- a/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md +++ b/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md @@ -96,3 +96,59 @@ rejection, accepted-but-still-stopping, and stale IDs. The remaining experiment is to exercise the live adapter with a deliberately invalid configuration and a suitable short program, then decide whether to add per-run results and events before attaching MCP. No live-console validation is claimed for this draft. + +## Concrete target: investigating the Sandgem failure + +In BDSP AutoStory run r109, the player reached Sandgem correctly, but OCR read +the location banner as `andgemltown`, so the location check failed. See the +[AutoStory lessons, section 13](https://github.com/dayfine/Arduino-Source/blob/bdsp/floaroma-mvp/docs/design/AutoStory-Lessons.md#13-floaroma-mvp-from-a-new-game-and-the-lab-around-it-2026-10-29-r107-to-r109). + +Lab can already start the program and report that it stopped. The agent must +then find the corresponding error and screenshot, associate them with this +attempt, and distinguish the original failure from later retries. The current +four-method prototype does not remove that work. + +The useful next slice would support this investigation: + +1. Start the configured checkpoint and receive a unique run ID. +2. Wait for that run's outcome, rather than polling whether the session is stopped. +3. Retrieve its error and failure evidence together. +4. Inspect the frame and log: the location is visibly correct, but its OCR + comparison rejected it. +5. Investigate the matcher with saved-image cases, including a different town + that must still be rejected, then rerun the checkpoint under a new run ID. + +An illustrative future result, **not implemented by this prototype**, is: + +```json +{ + "run_id": "run-109", + "outcome": "failed", + "checkpoint": "00.04", + "error": "Location verification failed", + "failure_screenshot": "...", + "log_excerpt": "..." +} +``` + +The error text above is illustrative too. The interface should preserve the +actual program error and capture the failure evidence before a retry changes +the screen. The matcher fix remains in AutoStory; the interface makes its +investigation dependable and automatable. Run identity, outcome and error +capture belong near `ProgramSession`. Checkpoint details require connecting +the program's progress reporting to that run. + +Other recorded failures show the boundary: + +| Failure | What an agent interface could contribute | What the program still must do | +|---|---|---| +| r84: selecting Water Gun at 0 PP for four minutes | Progress reports and recent captures let the agent recognize repetition and stop the identified run. | Read PP, skip exhausted moves and bound retries. A generic session cannot infer PP without observation or program reporting. | +| r105/r106: reported a pass after talking to the rock beside Roark | Expose a story-effect verification failure, once the program checks it. | Verify that Roark actually left. Lifecycle completion alone cannot establish story success. | +| r109a: the Start click missed after the window resized | Lab's existing explicit start/stop commands already address this. | No new benefit should be attributed to this prototype for removing GUI clicks. | + +The intended unlock is an agent that can run a bounded automation experiment, +identify its first failure, inspect the associated evidence, probe the cause, +change the code and repeat. Structured start/stop alone is groundwork, not +that outcome. A concrete next proof should demonstrate: **a checkpoint fails; +one request returns that run's outcome, error and failure screenshot**. MCP +can expose the same operations afterward; changing transport alone adds little. From 0a0bb99b394f9ac4207fb3d92170cd53facfb74d Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 31 Oct 2026 23:33:12 -0700 Subject: [PATCH 03/16] Design run results and failure evidence for the Lab backend --- .../SessionAgentPrototype/DESIGN.md | 234 ++++++++++++++++++ .../SessionAgentPrototype/README.md | 3 + 2 files changed, 237 insertions(+) create mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/DESIGN.md diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/DESIGN.md b/SerialPrograms/Source/Integrations/SessionAgentPrototype/DESIGN.md new file mode 100644 index 0000000000..a2d5166b89 --- /dev/null +++ b/SerialPrograms/Source/Integrations/SessionAgentPrototype/DESIGN.md @@ -0,0 +1,234 @@ +# Run results and failure evidence for agent-driven testing + +Status: proposed follow-up to the four-method prototype in [README.md](README.md). +The run API below is not implemented by #116. This document defines the next +experiment and the conditions for keeping it. + +## Goal + +An agent can start a configured program, identify that particular run, and +retrieve its terminal outcome and associated failure evidence even after a +later run starts. Build this into Lab's existing backend, using ProgramSession +as the lifecycle authority. File commands and a future MCP adapter should call +the same operations. + +The intended improvement is reliable investigation: no guessing which attempt +an error belongs to, no treating an accepted start as completion, and no using +a screenshot taken after a retry as evidence of the original failure. This +does not make gameplay logic correct or establish story success. + +The first proof is a failed checkpoint whose result includes the original +error and failure frame. Program selection, configuration editing, controller +inputs and interactive CFR investigation continue through existing Lab tools. + +## Existing seams and gaps + +- [ProgramSession](../../CommonFramework/ProgramSession.h) owns start/stop, + the worker and listeners. Its STOPPED state does not distinguish normal + completion, cancellation or failure. Listener errors are transient strings; + some, such as a stats-save error, are not a gameplay failure. +- [run_program()](../../CommonFramework/ProgramSession.cpp) already classifies + several exception types. Some setup work, prerequisite downloads and early + returns occur outside its main catch block. Observing state changes alone + cannot provide a complete run outcome. +- [ProgramTracker](../ProgramTracker.h) discovers existing sessions and consoles. + Reuse it; do not add another registry of live session pointers. Its current + locking does not by itself prove safety during derived-session destruction. +- [LabChannel](../../CommonFramework/Windows/LabChannel.cpp) supplies a command + transport, ownership checks and a journal. Preserve those controls and make + the run operations available through that channel. +- `OperationFailedExceptionWithScreenshot` and `FatalProgramException` already + expose an owned screenshot. Preserve that frame before the exception dies; + a later camera snapshot is weaker evidence. + +## Identity and result model + +| Identifier | Meaning | Lifetime | +|---|---|---| +| Program identifier | Descriptor/type, such as a particular AutoStory program | Across sessions | +| Session ID | One live ProgramSession | Until it is unloaded | +| Run ID | One accepted execution of that session | Until its record expires | +| Request ID | One client's start request, used for safe retries | Bounded deduplication window | + +Use opaque string run IDs containing a fresh app-instance identity and a +monotonic run counter. A restart cannot accidentally resolve an old ID to a +new run. Do not encode a mutable selection or an implicit "current program". + +Each record contains the run/session/program IDs, associated console IDs, +start/end timestamps, lifecycle status, terminal outcome, optional primary +error, bounded diagnostics, and evidence descriptors. Record the app revision +and the effective program configuration once at acceptance, excluding secrets; +identify any configuration fields that cannot be recorded. If a fingerprint +is used, version its canonicalization. Reproducibility should not depend on +whatever options happen to be selected when the agent later inspects a result. + +Lifecycle status is `running`, `stopping` or `terminal`. Terminal outcomes are: + +| Outcome | Meaning | +|---|---| +| `completed` | Normal return or ProgramFinishedException; the program declared completion | +| `cancelled` | Execution ended through the existing cancellation path | +| `failed` | A setup, connection or execution error prevented completion | + +`completed` does not mean a story effect was independently verified. Optional +program-reported progress and verification results are separate fields. The +initial implementation does not invent checkpoint IDs by parsing log text. + +A rejected start returns the existing rejection reason and creates no run. +An accepted start creates exactly one record before execution can emit events. +Even a program that finishes before the start response must remain queryable. +Record UI-initiated runs too, so an agent can inspect them without claiming +that its own request started them. + +## Shared operations + +These are proposed semantic operations; JSON field names can be settled during +implementation. They replace the prototype's parallel lifecycle dispatch once +the proof works. + +| Operation | Contract | +|---|---| +| `sessions.list/status` | Existing session snapshot, plus active run ID when present | +| `runs.start(session_id, request_id)` | Validate ownership and configuration; return accepted run ID or actual rejection | +| `runs.get(run_id)` | Return an immutable snapshot of this run, including terminal result and evidence availability | +| `runs.stop(run_id)` | Cancel only if this exact run is still active; never stop a later run of its session | +| `runs.evidence(run_id, evidence_id)` | Retrieve evidence belonging to this run, without accepting arbitrary filesystem paths | + +Initial clients poll `runs.get` with a deadline. A client timeout means the +client stopped waiting; it does not imply completion or trigger cancellation. +Do not put a blocking wait in Lab's serialized command queue: that could block +the stop request needed to end the run. A future long-poll or subscription must +release the command executor and handle disconnects independently. + +Within the bounded request-history window, retrying the same start request ID +returns the original acceptance/rejection; reusing it with different arguments +is an error. After that window, return an explicit expired-request error rather +than executing an ambiguous retry. A new intentional execution uses a new ID. +One bounded implementation uses app/owner-scoped increasing request numbers, +a retained high-water mark and the last 128 replies. A number below that mark +whose reply was evicted is expired, not new. New submissions must increase the +number; retries can reuse a retained one. Retiring an owner epoch invalidates +its requests. This avoids an unbounded set of previously seen UUIDs. + +Stop checks the run identity atomically with the lifecycle operation. Repeating +stop for a stopping or terminal run is harmless and returns that run's status. +No request may fall back to the selected session or the session's newest run. +Unknown, expired and foreign-app IDs have explicit errors. Preserve Lab's owner +checks for every operation; a run ID is not authorization or a new owner lease. +The first slice retains today's lease policy and does not add takeover. + +## Lifecycle publication + +Add a small typed run model and a bounded result store near the framework's +session lifecycle. ProgramSession publishes lifecycle facts; the store retains +value snapshots after a session is unloaded. Integrations serialize the model. +The framework must not depend on Lab command parsing, JSON transport or games. + +The implementation needs an outer execution boundary covering validation in +the worker, downloads, environment construction, execution, cancellation and +early returns. Every accepted run must reach exactly one terminal result on +all supported in-process exit paths. Task-dispatch failure must also finalize +an already accepted run, or roll back acceptance before it becomes observable. +A scope finalizer can enforce this, but its fallback must not invent success. + +Preserve existing notification behavior while recording outcomes at the actual +execution boundary. Do not translate every `report_error()` callback into a +terminal failure. Retain stats-save and evidence-write errors as diagnostics +without replacing the primary execution error. + +Publish terminal status after program execution and required cleanup cease, +before another run can replace the active-run slot. Required failure-frame +ownership must already be secured. If image encoding is asynchronous, expose +`pending`, `ready`, `unavailable` or `expired` explicitly; terminal execution +does not promise that disk encoding has completed. + +Cancellation requests do not overwrite an observed execution failure. A normal +return racing with a late stop remains completed; an actual cancellation exit +is cancelled. Serialize finalization so only one outcome wins. Ordinary setup +and download failures are failed; a deliberate cancellation during them is +cancelled. An app crash is outside the initial guarantee: in-memory history is +lost, and a missing app must never be reported as a completed run. + +## Failure evidence + +Prefer the screenshot carried by the exception, retaining its owning image +object rather than an ImageView or VideoStream pointer. Record its source, +console identity when known, dimensions and capture timestamp when available. +An unavailable timestamp stays unknown. Do not substitute the encoding time. + +For errors without a frame, a console-aware boundary may attempt a bounded +snapshot before recovery changes the screen. Label it `post_failure_capture`; +it is not necessarily the failing frame. A generic session may have no camera, +several consoles, or a disconnected stream: missing evidence is an explicit +result, and capture failure must not mask the original error. + +Keep a bounded run-scoped diagnostic tail at the logging boundary, tagging +entries while the run is active. Do not scrape the last lines of the global +log or include unrelated sessions. If propagation through background work is +not available, label that log coverage incomplete. The primary error and its +owned frame are the minimum first proof; broader log capture may follow. + +For the Sandgem example, publish the actual OCR error and exception frame. +The agent can compare the visible banner with the rejection, then investigate +the matcher through existing CFR and saved-image tests. If AutoStory catches +the error and retries internally, the session sees one continuing run. Capturing +every checkpoint attempt requires a later typed progress/failure hook before +recovery; it is not obtained for free from ProgramSession. + +## Lifetimes, locks and resource limits + +Before a live adapter is considered supported, audit registration, lookup, +shutdown and derived-session destruction together. Reject new operations once +shutdown begins, detach observers safely, cancel/join execution while derived +resources are valid, then unregister. Requests must hold a defined lifetime +guard or be marshalled through an owner that provides that guarantee. Keeping +a raw pointer after releasing the tracker lock is insufficient. + +Use short critical sections for run snapshots and transitions. Do not encode +images, write files, wait for workers, or invoke external callbacks while +holding the global tracker or result-store lock. Document lock ordering with +ProgramSession; existing listeners can run under its lock, so callbacks must +not re-enter it or acquire locks in the reverse order. + +Choose fixed initial limits: at most 32 retained terminal records, 64 KiB of +diagnostic text per run, 64 MiB of retained encoded evidence app-wide, and two +pending image encodes. Keep active records; evict oldest terminal evidence and +then terminal records. Bound queued raw images separately by bytes before +enqueueing. Oversized frames or a full queue yield explicit unavailable +evidence, never unbounded allocation or a blocked stop operation. These are +implementation defaults to measure during the proof, not new UI settings. + +Evidence is stored under an app-managed directory and retrieved by opaque ID. +Define deletion with eviction and shutdown; report expired evidence explicitly. +Do not return arbitrary paths as a general file-reading capability. Durable +history across restarts and crash recovery are deferred. + +## Implementation slices and validation + +1. **Typed lifecycle results.** Add run identity, bounded history and exactly-once + finalization at ProgramSession. Native C++ tests exercise rejection, immediate + completion, setup/download failure, cancellation, dispatch failure, and a + second run preserving the first result. Cover execution paths with scripted + actions rather than requiring a Switch. +2. **Lab operations.** Route start/get/stop through the shared service; preserve + existing commands and defaults. Test duplicate request IDs, timeout without + cancellation, stale-run stop while a later run is active, ownership rejection, + unload races and shutdown. Audit lifetime and lock ordering in this slice. +3. **Failure evidence.** Retain the exception image and expose evidence lookup. + Test missing frames, failed encoding, quota eviction and error preservation. + Prove the result remains associated with run A after run B begins. +4. **One live checkpoint demonstration.** Under separately authorized console + use, run a short checkpoint with a controlled failure and an owned frame. + Retrieve outcome/error/frame through Lab, start a second run, and retrieve + the first result again. Use Sandgem if the relevant setup is available; + do not require recreating the historical OCR failure to validate correlation. + +Keep each slice reviewable and run the applicable native tests and portable +builds. No live-console validation is claimed by this design or the current POC. + +The decision gate is the demonstrated run-A/run-B investigation, including a +stale stop that cannot affect B. If it works, absorb the prototype dispatcher +into the Lab service and remove duplicate lifecycle routes. Only then consider +MCP as another adapter, typed checkpoint/attempt events, bounded progress +history or optional durable storage. A second transport alone is not the +reliability improvement being evaluated. diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md b/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md index c818bf58da..c4711977e9 100644 --- a/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md +++ b/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md @@ -9,6 +9,9 @@ This prototype adds four JSON methods and a lab-channel adapter. It reuses dispatcher can be driven in a hardware-free terminal demo. Delete or absorb this prototype once the interface decision is made. +For the proposed next step, see [the run results and failure evidence design](DESIGN.md). +It adds per-run outcomes to Lab's shared backend; it is not implemented here. + ## Try the terminal demo From the repo root, with a C++23 compiler: From 76b5fcfc56f31e32bb0c091d711a1abf0903ad23 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 31 Oct 2026 23:40:58 -0700 Subject: [PATCH 04/16] Session agent design: scope the run results against the AutoStory runs Adds a scope review to DESIGN.md: what the console loop needs from a run record, how the proposal would have helped on the recorded failures, what to keep, add, drop and simplify in the first slice (typed program progress in, sessions.* and request-ID deduplication out, runs as files under Lab/), how CFR fits, and how the loop would use it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../SessionAgentPrototype/DESIGN.md | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/DESIGN.md b/SerialPrograms/Source/Integrations/SessionAgentPrototype/DESIGN.md index a2d5166b89..7850ea9154 100644 --- a/SerialPrograms/Source/Integrations/SessionAgentPrototype/DESIGN.md +++ b/SerialPrograms/Source/Integrations/SessionAgentPrototype/DESIGN.md @@ -21,6 +21,140 @@ The first proof is a failed checkpoint whose result includes the original error and failure frame. Program selection, configuration editing, controller inputs and interactive CFR investigation continue through existing Lab tools. +## Scope review against the AutoStory runs (2026-11-01) + +This section reads the proposal against what the BDSP AutoStory loop actually +spent its time on (runs r1 to r251, `docs/design/AutoStory-Lessons.md`, and the +`console-iteration` skill). It narrows the first slice and moves two pieces. + +### What we want out of it + +One iteration of the loop is: load the save, start one checkpoint with a known +configuration, wait, and on failure know **which step and attempt failed, with +what error, what the screen showed at that moment, and where the player was**; +then probe by hand with CFR, fix once, rerun. Today the agent gets this from +`run_watch.sh`: it remembers the log's line count, waits for a regex +(`Completed: 00.05`, `attempt N failed`, `Program State: STOPPED|ERROR`), prints +the `[NN.NN step i/n]`, `[Route]` and `[Event]` lines, and lists the newest files +under `Screenshots/AutoStory/failures/`. The verdict is PASS / FAIL / UNKNOWN, +and UNKNOWN happened whenever the program stopped without either line. + +So the deliverable is not a new API; it is **one place that says what a run did**: + +1. A run identity the agent can keep (`run_id`) and a terminal outcome that is + never inferred from log text or from `state=STOPPED`. +2. The failure, as a bundle: primary error, the frame the exception carried, + the step/attempt/event it belongs to, and the last lines of that run's log. +3. Per-checkpoint results inside one run. Run 3 (new game to Floaroma, every + batched map and route change verified together) is the next real use: its + value is a list "00.01 ok, 00.02 ok, ..., 01.08 failed at step 4 with ...", + not one outcome for the whole run. +4. The console left in a state the agent can probe: program stopped, game open + at the failing screen (memory `stop-and-close-game-on-failure`). CFR then + does the investigation. No new interactive surface is needed. + +### How it would have helped, run by run + +| Runs | What cost time | Would the proposal have helped? | +|---|---|---| +| r109 Sandgem `andgemltown` | Finding which screenshot matched the failing OCR; the banner was right, the matcher was narrow. | Yes: error + owned frame in one record. Small gain, the failures folder already had the frame; the gain is correlation, not the frame. | +| r84 Water Gun at 0 PP, four minutes | Nobody noticed repetition until the log was read. | Only with typed progress events the agent can watch; a terminal outcome arrives four minutes late. | +| r105/r106 "pass" after talking to a rock | The program declared completion; the story had not moved. | No. Completion stays program-reported; the fix was a story-effect check in the program. | +| r109a Start click missed | GUI clicks on a resized window. | Already solved by Lab `start`/`stop`/`state` (#107). | +| r178 to r215 Oreburgh 01.03/01.04 | What movement really is: tap counts, wall stops, stillness. Dozens of CFR `walk` probes and screenshots read by hand. | Not by the run API. The evidence that mattered was the walker's own view (blocked taps, tiles counted, stillness) per leg. That is a program event stream. | +| r216 to r251 Route 204 | Map rows wrong (image-traced grids); routes replanned; "where are we" after each leg. | Same: per-leg pose and leg end evidence from `StoryEvents`/the walker, drawn on the grid. Run-level evidence says only "01.08 failed". | +| Every run since r28 (lessons 8) | "Every run that failed in a new way was explained by one screenshot." | Yes, if that screenshot is the one the exception carried and it is named by run, step and attempt, so the agent opens the right one first. | + +Two conclusions. First, the frame-plus-error record is worth having but is the +smaller half; the half that paid for itself every day is **structured progress +from inside the program** (checkpoint, step, attempt, leg end, pose, recover +action). `StoryEvents` already produces these as `[Event]` log lines through +`EventLogger`; the walker logs `[Route]` lines. Second, AutoStory retries +inside the program, so a ProgramSession-level record sees one continuing run +and only the last failure. The design notes this; for us it is the common case, +not the edge case, so the progress hook belongs in the first slice, not after +the decision gate. + +### What to change + +- **Keep:** run identity; exactly one terminal outcome per accepted run + (`completed` / `cancelled` / `failed`); the primary error with its owned frame; + a run-tagged log tail; `runs.get` style retrieval that works after the next + run starts; UI-started runs recorded too; Lab's owner checks unchanged. +- **Add to the first slice:** a typed progress channel the program writes to + (`run.event(kind, checkpoint, step, attempt, text, optional frame, optional + pose)`), fed by `StoryEvents::EventLogger` and the walker, stored under the run. + The Sandgem proof then shows the failing attempt, not only the final error. + `completed` plus the list of checkpoint events is the run 3 report. +- **Drop from the first slice:** `sessions.*` as a separate method family. A lab + app runs one program; Lab's `select`/`start`/`stop`/`state` already address it. + Return `run_id` from `start` and show `run=` in the `state` file instead. +- **Drop:** the request ID with a 128-reply deduplication window and owner + epochs. Lab's inbox is a serialized queue of named request files; the request + file name is the request ID, and the journal already answers a resend. One + local client does not need idempotent retries with expiry semantics. +- **Simplify storage:** Lab is a file protocol. Write each run to + `Lab/runs//` (`result.json`, `events.jsonl`, `log.txt`, `frame-*.png`) + and keep the last N directories. This gives durability across app restarts + and crashes for free, which the in-memory store defers, and it removes the + opaque evidence IDs, byte quotas and encode queues from the first slice. The + rule "do not accept arbitrary paths as input" still holds; writing files into + the lab folder is what the channel already does for `shot`. +- **Keep deferred:** MCP, long-poll or subscriptions, multi-session. The agent + polls `result.json`; a `labctl.sh wait ` wrapper hides the polling. +- **Framework boundary:** the typed run model, the execution boundary with + exactly-once finalization and the owned-frame capture stay in + `CommonFramework` next to `ProgramSession`, as proposed. Event recording is a + small interface the program calls (`RunRecorder&` reachable from the + environment); `StoryEvents` and the walker depend on that interface, not on + Lab. Lab serializes it to files; a later MCP adapter reads the same files. + +### CFR + +CFR stays the hand-probing tool and already shares Lab's transport since #107 +(`press`, `dpad`, `walk`, `shot`, `detect`, `wait_for`, `expect`, game commands). +Two upgrades make sense, neither of them a new protocol: + +- `shot` and `detect` results taken while a run is active, or right after one + failed, are tagged with the `run_id`, so probing lands in the same run folder. +- A `bdsp_pose` / `frlg_pose` command that reports the program's best estimate + of the player's tile and facing (from the walker's count and the last + confirmed anchor), so "where are we" is one command instead of a screenshot + read against the grid. + +The old `CommandFiles/` folder can go once Lab carries every CFR command; that +is cleanup, not part of this design. + +### How it will be used + +In the `console-iteration` skill's loop, replacing `run_watch.sh`: + +```sh +sh labctl.sh send run15 <<'REQ' # select, set, bdsp_load_save, start +... +REQ +run=$(sh labctl.sh state | sed -n 's/^run=//p') +sh labctl.sh wait "$run" # polls Lab/runs/$run/result.json, prints it +# outcome=failed checkpoint=01.08 step=4/7 attempt=2 +# error: expect_failed: the Route 204 sign's box did not open +# frame: .../Lab/runs/run15-7/frame-01.08-s4-a2.png +# pose: route_204_south (12, 41) facing N, certainty inexact +# events: .../Lab/runs/run15-7/events.jsonl log: .../log.txt +``` + +The agent opens the frame, draws the pose on the grid, probes with CFR in the +same run folder, fixes once, and reruns under a new `run_id`. Run 3 reads +`events.jsonl` for the checkpoint list and the report writes itself. Verdict +UNKNOWN disappears because the outcome comes from the execution boundary. + +### Decision gate, restated + +Keep slice 4's live demonstration, with one addition: the failing attempt's +event (step, attempt, frame) must be in the record alongside the terminal +error, from the program's own hook, and a CFR `shot` sent after the failure +must land in the same run. If that works on one BDSP checkpoint, run 3 adopts +it; FRLG's Brock MVP uses the same recorder through its own events. + ## Existing seams and gaps - [ProgramSession](../../CommonFramework/ProgramSession.h) owns start/stop, From 8485907f0164049eaea90e0ed1fca04630627e13 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 00:31:50 -0700 Subject: [PATCH 05/16] Run records: a typed run model and a bounded store in CommonFramework RunRecord.h holds what one run of a program did: run ID, program, start and end time, status (running, stopping, terminal), exactly one result (completed, cancelled or failed, with the primary error and the owned failure frame), the events the program reported and a bounded log tail. RunRecordStore keeps the last 32 records of this app instance; SessionRunRecorder is one session's end of it, a RunRecorder for the active run and a Logger that copies lines into the run's tail. Nothing depends on the UI, JSON or a game. Native tests cover the IDs and capacity, the one-result rule, the event, frame and log bounds, frame release for old runs and the session recorder. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Source/CommonFramework/RunRecord.cpp | 227 ++++++++++++++++++ .../Source/CommonFramework/RunRecord.h | 178 ++++++++++++++ .../CommonFramework/RunRecord_Tests.cpp | 211 ++++++++++++++++ .../Source/CommonFramework/RunRecord_Tests.h | 20 ++ .../ComputerPrograms_UnitTestRunner.cpp | 2 + SerialPrograms/cmake/SourceFiles.cmake | 4 + 6 files changed, 642 insertions(+) create mode 100644 SerialPrograms/Source/CommonFramework/RunRecord.cpp create mode 100644 SerialPrograms/Source/CommonFramework/RunRecord.h create mode 100644 SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp create mode 100644 SerialPrograms/Source/CommonFramework/RunRecord_Tests.h diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.cpp b/SerialPrograms/Source/CommonFramework/RunRecord.cpp new file mode 100644 index 0000000000..17cffdca16 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord.cpp @@ -0,0 +1,227 @@ +/* Run Record + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/PrettyPrint.h" +#include "RunRecord.h" + +namespace PokemonAutomation{ + + + +std::string run_status_name(RunStatus status){ + switch (status){ + case RunStatus::RUNNING: + return "running"; + case RunStatus::STOPPING: + return "stopping"; + case RunStatus::TERMINAL: + return "terminal"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown run status."); +} +std::string run_outcome_name(RunOutcome outcome){ + switch (outcome){ + case RunOutcome::COMPLETED: + return "completed"; + case RunOutcome::CANCELLED: + return "cancelled"; + case RunOutcome::FAILED: + return "failed"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown run outcome."); +} + + + +RunRecordStore& RunRecordStore::instance(){ + static RunRecordStore store; + return store; +} + +RunRecordStore::RunRecordStore(size_t capacity) + // The app's start time, to the microsecond: two instances started in + // the same second still get different IDs. + : m_app_token(now_to_filestring()) + , m_capacity(std::max(capacity, 1)) +{} + +std::string RunRecordStore::begin(std::string program){ + std::lock_guard lg(m_lock); + while (m_records.size() >= m_capacity){ + m_records.pop_front(); + } + RunRecord& record = m_records.emplace_back(); + record.run_id = std::format("{}-{:03}", m_app_token, ++m_counter); + record.program = std::move(program); + record.start_time = current_time(); + return record.run_id; +} + +void RunRecordStore::stopping(const std::string& run_id){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record != nullptr && record->status == RunStatus::RUNNING){ + record->status = RunStatus::STOPPING; + } +} + +void RunRecordStore::event(const std::string& run_id, RunEvent event){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr){ + return; + } + if (record->events.size() >= MAX_EVENTS){ + record->dropped_events++; + return; + } + if (event.time == WallClock::min()){ + event.time = current_time(); + } + record->events.emplace_back(std::move(event)); + + // Keep the frames of the newest events only. + size_t frames = 0; + for (RunEvent& e : std::ranges::reverse_view(record->events)){ + if (e.frame == nullptr){ + continue; + } + if (++frames > MAX_EVENT_FRAMES){ + e.frame.reset(); + } + } +} + +void RunRecordStore::log_line(const std::string& run_id, const std::string& text){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr){ + return; + } + record->log_tail.emplace_back(text); + size_t bytes = 0; + for (const std::string& line : record->log_tail){ + bytes += line.size() + 1; + } + size_t drop = 0; + while (bytes > MAX_LOG_BYTES && drop + 1 < record->log_tail.size()){ + bytes -= record->log_tail[drop].size() + 1; + drop++; + } + if (drop > 0){ + record->log_tail.erase(record->log_tail.begin(), record->log_tail.begin() + drop); + record->log_truncated = true; + } +} + +bool RunRecordStore::finish(const std::string& run_id, RunResult result){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr || record->status == RunStatus::TERMINAL){ + return false; + } + record->status = RunStatus::TERMINAL; + record->end_time = current_time(); + record->result = std::move(result); + + // Frames stay with the newest terminal runs; older ones keep the rest + // of their record. + size_t terminal = 0; + for (RunRecord& r : std::ranges::reverse_view(m_records)){ + if (r.status == RunStatus::TERMINAL && ++terminal > RUNS_WITH_FRAMES){ + release_frames(r); + } + } + return true; +} + +std::vector RunRecordStore::run_ids() const{ + std::lock_guard lg(m_lock); + std::vector ids; + for (const RunRecord& record : m_records){ + ids.emplace_back(record.run_id); + } + return ids; +} + +std::optional RunRecordStore::get(const std::string& run_id) const{ + std::lock_guard lg(m_lock); + for (const RunRecord& record : m_records){ + if (record.run_id == run_id){ + return record; + } + } + return std::nullopt; +} + +RunRecord* RunRecordStore::find(const std::string& run_id){ + auto it = std::ranges::find_if(m_records, [&](const RunRecord& r){ return r.run_id == run_id; }); + return it == m_records.end() ? nullptr : &*it; +} + +void RunRecordStore::release_frames(RunRecord& record){ + for (RunEvent& event : record.events){ + event.frame.reset(); + } + if (record.result){ + record.result->frame.reset(); + } +} + + + +SessionRunRecorder::SessionRunRecorder(RunRecordStore& store, Logger& base_logger) + : m_store(store) + , m_base_logger(base_logger) +{} + +std::string SessionRunRecorder::begin(std::string program){ + std::lock_guard lg(m_lock); + m_run_id = m_store.begin(std::move(program)); + m_active = true; + return m_run_id; +} +void SessionRunRecorder::stopping(){ + std::lock_guard lg(m_lock); + if (m_active){ + m_store.stopping(m_run_id); + } +} +void SessionRunRecorder::finish(RunResult result){ + std::lock_guard lg(m_lock); + if (!m_active){ + return; + } + m_active = false; + m_store.finish(m_run_id, std::move(result)); +} + +std::string SessionRunRecorder::last_run_id() const{ + std::lock_guard lg(m_lock); + return m_run_id; +} + +void SessionRunRecorder::event(RunEvent event){ + std::lock_guard lg(m_lock); + if (m_active){ + m_store.event(m_run_id, std::move(event)); + } +} +void SessionRunRecorder::log(const std::string& msg, Color color){ + m_base_logger.log(msg, color); + std::lock_guard lg(m_lock); + if (m_active){ + m_store.log_line(m_run_id, msg); + } +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.h b/SerialPrograms/Source/CommonFramework/RunRecord.h new file mode 100644 index 0000000000..f9afd2c86d --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord.h @@ -0,0 +1,178 @@ +/* Run Record + * + * From: https://github.com/PokemonAutomation/ + * + * What one run of a program did, as the framework saw it: when it started, + * how it ended (exactly one outcome per accepted run), the error and the + * frame it died with, the events the program reported on the way, and the + * tail of its log. ProgramSession writes these at its execution boundary; + * RunRecordStore keeps the last few for whoever asks, such as the Lab + * channel, which writes them to files (Windows/Lab.md, "Run records"). + * Nothing here depends on the UI, on JSON or on a game. + * + */ + +#ifndef PokemonAutomation_CommonFramework_RunRecord_H +#define PokemonAutomation_CommonFramework_RunRecord_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" + +namespace PokemonAutomation{ + + +enum class RunStatus{ + RUNNING, + STOPPING, // A stop was requested; the outcome is not known yet. + TERMINAL, +}; +enum class RunOutcome{ + COMPLETED, // The program returned or threw ProgramFinishedException. + CANCELLED, // The program ended through the stop path. + FAILED, // A setup, connection or execution error ended it. +}; +std::string run_status_name(RunStatus status); // "running", "stopping", "terminal" +std::string run_outcome_name(RunOutcome outcome); // "completed", "cancelled", "failed" + + +// Something the program says about its own progress: a checkpoint reached, +// a step started, an attempt that failed, a pose. Every field but `kind` +// and `text` is optional; a game fills the ones it has. (The `= {}` let a +// designated initializer leave fields out on every compiler.) +struct RunEvent{ + WallClock time = WallClock::min(); // Set by the recorder when left at min(). + std::string kind = {}; // e.g. "checkpoint", "step", "attempt", "error" + std::optional checkpoint = {}; + std::optional step = {}; + std::optional attempt = {}; + std::string text = {}; + std::shared_ptr frame = {}; // The screen at that moment, if the program had it. + std::optional pose = {}; // Where the player is, as the program puts it. +}; + +// How a run ended. +struct RunResult{ + RunOutcome outcome = RunOutcome::FAILED; + std::optional error = {}; // The primary error of a failed run. + std::shared_ptr frame = {}; // The frame the exception carried, if any. +}; + +struct RunRecord{ + std::string run_id = {}; + std::string program = {}; // The program's identifier, ":". + WallClock start_time = WallClock::min(); + std::optional end_time = {}; + RunStatus status = RunStatus::RUNNING; + std::optional result = {}; // Set once, when the status turns TERMINAL. + std::vector events = {}; + size_t dropped_events = 0; // Events past RunRecordStore::MAX_EVENTS. + std::vector log_tail = {}; // The run's last log lines, oldest first. + bool log_truncated = false; // Whether older lines were dropped from the tail. +}; + + +// The program's progress hook, reachable as ProgramEnvironment::run_recorder(). +class RunRecorder{ +public: + virtual ~RunRecorder() = default; + virtual void event(RunEvent event) = 0; +}; + + +// The last few runs of this app instance. Thread-safe; every call is short +// and copies, so a reader never holds a lock while writing files. +// +// Run IDs are opaque strings unique to this app instance: a restart cannot +// resolve an old ID to a new run. They sort in the order the runs started. +// +// Memory: at most MAX_EVENTS events per run; the frames of a run's newest +// MAX_EVENT_FRAMES events; all frames only for the active run and the +// newest RUNS_WITH_FRAMES terminal runs; a log tail of MAX_LOG_BYTES. +// A consumer that wants a frame reads it soon after the event or the end. +class RunRecordStore{ +public: + static constexpr size_t DEFAULT_CAPACITY = 32; + static constexpr size_t MAX_EVENTS = 4096; + static constexpr size_t MAX_EVENT_FRAMES = 8; + static constexpr size_t RUNS_WITH_FRAMES = 4; + static constexpr size_t MAX_LOG_BYTES = 64 * 1024; + + static RunRecordStore& instance(); + + explicit RunRecordStore(size_t capacity = DEFAULT_CAPACITY); + RunRecordStore(const RunRecordStore&) = delete; + void operator=(const RunRecordStore&) = delete; + + // Opens a record for an accepted start and returns its run ID. The + // oldest record goes when the store is full. + std::string begin(std::string program); + + // Marks a running run as stopping. No effect once it is terminal. + void stopping(const std::string& run_id); + + // Appends to the run. Both are dropped for an unknown run. + void event(const std::string& run_id, RunEvent event); + void log_line(const std::string& run_id, const std::string& text); + + // Closes the run with its one result. Returns false, and changes + // nothing, when the run is unknown or already terminal. + bool finish(const std::string& run_id, RunResult result); + + // The IDs of the records held, oldest first. + std::vector run_ids() const; + + // A copy of the record, or nothing for an unknown run. + std::optional get(const std::string& run_id) const; + + +private: + RunRecord* find(const std::string& run_id); + void release_frames(RunRecord& record); + + mutable Mutex m_lock; + const std::string m_app_token; + const size_t m_capacity; + uint64_t m_counter = 0; + std::deque m_records; +}; + + +// One session's end of the store: a RunRecorder for the run active now and +// a Logger that copies what passes through it into that run's log tail. +// The session's loggers chain off `log()`, so a run's tail holds the lines +// the program and its consoles logged, and nothing from other sessions. +class SessionRunRecorder : public RunRecorder, public Logger{ +public: + SessionRunRecorder(RunRecordStore& store, Logger& base_logger); + + // Opens a run and makes it the active one. Returns its ID. + std::string begin(std::string program); + void stopping(); + // Closes the active run; nothing happens when none is active. + void finish(RunResult result); + + // The run the last begin() opened, active or finished; "" before the first. + std::string last_run_id() const; + + virtual void event(RunEvent event) override; + virtual void log(const std::string& msg, Color color = Color()) override; + +private: + RunRecordStore& m_store; + Logger& m_base_logger; + mutable Mutex m_lock; + std::string m_run_id; + bool m_active = false; +}; + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp new file mode 100644 index 0000000000..7c1c57c1e0 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp @@ -0,0 +1,211 @@ +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The store on its own. + * + */ + +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "RunRecord.h" +#include "RunRecord_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + +using namespace std::chrono_literals; + + +std::shared_ptr make_frame(size_t width){ + ImageRGB32 image(width, width); + image.fill(0xff000000); + return std::make_shared(std::move(image)); +} + +UnitTestResult test_store_ids(){ + RunRecordStore store(2); + const std::string a = store.begin("Test:A"); + const std::string b = store.begin("Test:B"); + if (a.empty() || a == b || !(a < b)){ + return std::format("Run IDs should be distinct and sort in start order: <{}>, <{}>", a, b); + } + if (store.run_ids() != std::vector{a, b}){ + return "run_ids() lists the records oldest first."; + } + const std::optional record = store.get(a); + if (!record || record->program != "Test:A" || record->status != RunStatus::RUNNING || record->result){ + return "A new record is running, with its program and no result."; + } + if (record->start_time == WallClock::min() || record->end_time){ + return "A new record has a start time and no end time."; + } + const std::string c = store.begin("Test:C"); + if (store.get(a) || store.run_ids() != std::vector{b, c}){ + return "A full store drops its oldest record."; + } + if (store.get("nobody")){ + return "An unknown run has no record."; + } + return true; +} + +UnitTestResult test_store_one_result(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.stopping(id); + if (store.get(id)->status != RunStatus::STOPPING){ + return "stopping() marks a running run."; + } + if (!store.finish(id, {.outcome = RunOutcome::FAILED, .error = "first"})){ + return "The first finish() takes."; + } + store.stopping(id); + if (store.finish(id, {.outcome = RunOutcome::COMPLETED})){ + return "A second finish() is refused."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || !record.end_time || !record.result){ + return "A finished record is terminal with an end time and a result."; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "first"){ + return std::format("The first result stands: got {}", run_outcome_name(record.result->outcome)); + } + if (store.finish("nobody", {})){ + return "Finishing an unknown run is refused."; + } + return true; +} + +UnitTestResult test_store_events_and_log(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.event(id, {.kind = "checkpoint", .checkpoint = "00.01", .text = "ok"}); + store.event("nobody", {.kind = "checkpoint"}); + for (size_t c = 0; c < RunRecordStore::MAX_EVENT_FRAMES + 2; c++){ + store.event(id, {.kind = "step", .step = std::to_string(c), .frame = make_frame(4)}); + } + RunRecord record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENT_FRAMES + 3){ + return std::format("Expected {} events, got {}.", RunRecordStore::MAX_EVENT_FRAMES + 3, record.events.size()); + } + if (record.events[0].time == WallClock::min() || record.events[0].checkpoint != "00.01"){ + return "An event gets the time it was recorded and keeps its fields."; + } + if (record.events[1].frame || record.events[2].frame || !record.events[3].frame || !record.events.back().frame){ + return "Only the newest MAX_EVENT_FRAMES events keep their frames."; + } + + for (size_t c = record.events.size(); c < RunRecordStore::MAX_EVENTS + 5; c++){ + store.event(id, {.kind = "step"}); + } + record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENTS || record.dropped_events != 5){ + return std::format("Events past the cap are counted, not kept: {} events, {} dropped.", record.events.size(), record.dropped_events); + } + + const std::string line(1000, 'x'); + for (size_t c = 0; c < 100; c++){ + store.log_line(id, std::format("{} {}", c, line)); + } + record = *store.get(id); + size_t bytes = 0; + for (const std::string& l : record.log_tail){ + bytes += l.size() + 1; + } + if (!record.log_truncated || bytes > RunRecordStore::MAX_LOG_BYTES || record.log_tail.back().rfind("99 ", 0) != 0){ + return std::format("The log tail keeps the newest {} bytes: {} lines, {} bytes.", RunRecordStore::MAX_LOG_BYTES, record.log_tail.size(), bytes); + } + return true; +} + +UnitTestResult test_store_releases_old_frames(){ + RunRecordStore store; + std::vector ids; + for (size_t c = 0; c < RunRecordStore::RUNS_WITH_FRAMES + 1; c++){ + ids.emplace_back(store.begin("Test:A")); + store.event(ids.back(), {.kind = "step", .frame = make_frame(4)}); + store.finish(ids.back(), {.outcome = RunOutcome::FAILED, .error = "e", .frame = make_frame(4)}); + } + const RunRecord oldest = *store.get(ids.front()); + if (oldest.events[0].frame || oldest.result->frame){ + return "A run older than the newest RUNS_WITH_FRAMES terminal runs keeps no frames."; + } + if (oldest.result->error != "e" || oldest.events.size() != 1){ + return "It keeps the rest of its record."; + } + const RunRecord newest = *store.get(ids.back()); + if (!newest.events[0].frame || !newest.result->frame){ + return "The newest terminal runs keep their frames."; + } + return true; +} + +UnitTestResult test_session_recorder(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + RunRecordStore store; + CountingLogger base; + SessionRunRecorder recorder(store, base); + recorder.log("before"); + recorder.event({.kind = "lost"}); + if (!recorder.last_run_id().empty() || !store.run_ids().empty()){ + return "Nothing is recorded before a run begins."; + } + const std::string id = recorder.begin("Test:A"); + if (recorder.last_run_id() != id){ + return "last_run_id() is the run begin() opened."; + } + recorder.log("during"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + recorder.log("after"); + recorder.event({.kind = "late"}); + recorder.finish({.outcome = RunOutcome::FAILED}); + const RunRecord record = *store.get(id); + if (base.lines != 3){ + return std::format("Every line reaches the base logger: {} of 3.", base.lines); + } + if (record.log_tail != std::vector{"during"} || record.events.size() != 1){ + return "Only lines and events during the run are recorded."; + } + if (record.result->outcome != RunOutcome::COMPLETED || recorder.last_run_id() != id){ + return "The result stands and the run stays known after finish()."; + } + return true; +} + + + +class Test_RunRecord : public UnitTest{ +public: + Test_RunRecord(std::string name, std::function body) + : UnitTest("CommonFramework::RunRecord - " + std::move(name)) + , m_body(std::move(body)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_body(); + } + +private: + std::function m_body; +}; + +} + + +void add_tests_RunRecord(UnitTestDatabase& database){ + database.add("store: ids and capacity", test_store_ids); + database.add("store: one result per run", test_store_one_result); + database.add("store: events, frames and log tail", test_store_events_and_log); + database.add("store: old runs release frames", test_store_releases_old_frames); + database.add("session recorder", test_session_recorder); +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h new file mode 100644 index 0000000000..9a3b88afcd --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h @@ -0,0 +1,20 @@ +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonFramework_RunRecord_Tests_H +#define PokemonAutomation_CommonFramework_RunRecord_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +void add_tests_RunRecord(UnitTestDatabase& database); + + + +} +#endif diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 77d3e4dea0..912037bb62 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -18,6 +18,7 @@ #include "CommonFramework/AudioPipeline/Spectrum/FFTStreamer.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ProgramStats/StatsTracking.h" +#include "CommonFramework/RunRecord_Tests.h" #include "CommonFramework/Tools/GlobalThreadPools.h" #include "CommonFramework/Windows/LabChannel_Tests.h" #include "CommonTools/VisualDetectors/BlackBorderDetector.h" @@ -59,6 +60,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ add_tests_SinkPipeline(ret); add_tests_AudioFileLoader(ret); add_tests_LabChannel(ret); + add_tests_RunRecord(ret); OCR::add_tests(ret); Kernels::add_tests(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 92cb4fe507..516647cfe0 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -652,6 +652,10 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/PersistentSettings.h Source/CommonFramework/ProgramSession.cpp Source/CommonFramework/ProgramSession.h + Source/CommonFramework/RunRecord.cpp + Source/CommonFramework/RunRecord.h + Source/CommonFramework/RunRecord_Tests.cpp + Source/CommonFramework/RunRecord_Tests.h Source/CommonFramework/ProgramStats/StatsDatabase.cpp Source/CommonFramework/ProgramStats/StatsDatabase.h Source/CommonFramework/ProgramStats/StatsTracking.cpp From a5e134b97109634574b613b078cc689b3ab2507e Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 00:32:14 -0700 Subject: [PATCH 06/16] ProgramSession: record every accepted run with exactly one result start_program() opens a run record once check_validity() passes, before the program thread exists; the thread closes it with one result whichever way it ends: a normal return or ProgramFinishedException is completed, the stop path is cancelled, and a setup, download, connection or execution error is failed with the exception's message and, for exceptions that carry one, its frame. A stop requested after a failure was observed does not overwrite it, and a normal return racing a stop stays completed. A dispatch failure closes the record too and leaves the session stopped instead of throwing out of start_program(). Errors reported through report_error() become "error" events, diagnostics rather than the outcome. The session's logger now chains through its SessionRunRecorder, and the console sessions take the logger they write through, so a run's log tail holds the program's and its consoles' lines and nothing else. ProgramEnvironment::run_recorder() is the program's progress hook. The lifecycle tests drive a ProgramSession whose program is a scripted fake with no console: rejected start, immediate completion, setup failure, failure with a frame, cancellation, a late stop, dispatch failure, and a second run preserving the first. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Source/CommonFramework/ProgramSession.cpp | 101 ++++- .../Source/CommonFramework/ProgramSession.h | 34 +- .../CommonFramework/RunRecord_Tests.cpp | 348 +++++++++++++++++- .../Tools/ProgramEnvironment.cpp | 4 + .../Tools/ProgramEnvironment.h | 7 + .../Framework/ConsolePanelSession.cpp | 3 +- .../Framework/ConsoleProgramSession.cpp | 1 + .../Framework/ConsoleSystemSession.cpp | 3 +- .../Framework/ConsoleSystemSession.h | 4 + .../Framework/MultiConsoleProgramSession.cpp | 2 +- .../Framework/MultiConsoleSystemSession.cpp | 4 + .../Framework/MultiConsoleSystemSession.h | 4 + 12 files changed, 490 insertions(+), 25 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp index f4176b9199..d091dc4b80 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp @@ -4,9 +4,11 @@ * */ +#include #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PanicDump.h" +#include "Common/Cpp/ScopeExit.h" #include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/Logging/Logger.h" @@ -64,11 +66,12 @@ void ProgramSession::remove_listener(Listener& listener){ -ProgramSession::ProgramSession(const ProgramDescriptor& descriptor) +ProgramSession::ProgramSession(const ProgramDescriptor& descriptor, RunRecordStore& run_records) : m_descriptor(descriptor) , m_instance_id(ProgramTracker::instance().add_program(*this)) + , m_run_recorder(run_records, global_logger_raw()) // , m_logger(global_logger_raw(), "Program:" + std::to_string(m_instance_id)) - , m_logger(global_logger_raw(), "Program") + , m_logger(m_run_recorder, "Program") , m_last_state_change(current_time()) , m_state(ProgramState::STOPPED) , m_scope(nullptr) @@ -186,6 +189,9 @@ void ProgramSession::push_stats(){ } void ProgramSession::push_error(const std::string& message){ m_listeners.run_method(&Listener::error, message); + // A diagnostic in the run's record, not its outcome: the stats-save + // error and the download warnings come through here too. + m_run_recorder.event({.kind = "error", .text = message}); } void ProgramSession::push_download_error(const std::string& message){ @@ -247,26 +253,24 @@ std::string ProgramSession::start_program(){ // Wait for previous program thread to finish. join_program_thread(); - // Now start the program. + // Now start the program. The run is accepted here: its record + // exists before the program thread can report anything. m_logger.log("Starting program..."); m_last_state_change.store(current_time(), std::memory_order_relaxed); + m_run_recorder.begin(identifier()); set_state(ProgramState::RUNNING); - m_program_thread = GlobalThreadPools::unlimited_realtime().dispatch_now_blocking( - [this]{ + try{ + m_program_thread = dispatch_program_thread([this]{ run_with_catch( "ProgramSession::start_program()", - [this]{ run_program(); } + [this]{ run_program_and_record(); } ); - } - ); -#if 0 - m_thread = Thread([this]{ - run_with_catch( - "ProgramSession::start_program()", - [this]{ run_program(); } - ); - }); -#endif + }); + }catch (Exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.message())); + }catch (std::exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.what())); + } return ""; } case ProgramState::RUNNING: @@ -300,6 +304,7 @@ std::string ProgramSession::stop_program(){ case ProgramState::RUNNING: m_logger.log("Stopping program..."); set_state(ProgramState::STOPPING); + m_run_recorder.stopping(); break; case ProgramState::STOPPING: m_logger.log("Program is already stopping.", COLOR_RED); @@ -319,13 +324,51 @@ std::string ProgramSession::stop_program(){ -void ProgramSession::run_program(){ +// No thread, so no run: closes the record and leaves the session stopped, +// so the next start can try again. Returns `message` for start_program(). +std::string ProgramSession::abandon_run(std::string message){ + m_logger.log(message, COLOR_RED); + m_run_recorder.finish({.outcome = RunOutcome::FAILED, .error = message}); + set_state(ProgramState::STOPPED); + return message; +} + +AsyncTask ProgramSession::dispatch_program_thread(std::function body){ + return GlobalThreadPools::unlimited_realtime().dispatch_now_blocking(std::move(body)); +} + +void ProgramSession::run_program_and_record(){ + // Exactly one result per run: whichever way the thread leaves, the + // record is closed, and a path that set none is a failure, never a + // success. An exception that escapes run_program() (as today it does + // for a validity check that fails after the start) is its result. + std::optional result; + ScopeExit finish_run([&]{ + m_run_recorder.finish(result.value_or(RunResult{ + .outcome = RunOutcome::FAILED, + .error = "The program thread ended without a result.", + })); + }); + try{ + result = run_program(); + }catch (Exception& e){ + result = RunResult{.outcome = RunOutcome::FAILED, .error = e.message().empty() ? e.name() : e.message()}; + throw; + }catch (std::exception& e){ + result = RunResult{.outcome = RunOutcome::FAILED, .error = e.what()}; + throw; + } +} + +RunResult ProgramSession::run_program(){ SleepSuppressScope sleep_scope(GlobalSettings::instance().SLEEP_SUPPRESS->PROGRAM_RUNNING); + RunResult result; { std::lock_guard lg(m_lock); if (current_state() != ProgramState::RUNNING){ - return; + // Stopped before the thread got going. + return {.outcome = RunOutcome::CANCELLED}; } std::string error = check_validity(); @@ -350,7 +393,12 @@ void ProgramSession::run_program(){ { RunningProgramScope running_scope(*this); if (!download_prereqs(*m_scope)){ - return; + // The details were reported as "error" events. A stop during + // the download is a cancellation, not a failure. + if (current_state() == ProgramState::STOPPING){ + return {.outcome = RunOutcome::CANCELLED}; + } + return {.outcome = RunOutcome::FAILED, .error = "A required resource could not be downloaded."}; } std::unique_ptr env = make_env(program_info); try{ @@ -360,10 +408,13 @@ void ProgramSession::run_program(){ internal_run_program(*env); env->log_to_ui("- Program Finished -"); logger().log("Program finished normally!", COLOR_BLUE); + result.outcome = RunOutcome::COMPLETED; }catch (OperationCancelledException&){ env->log_to_ui("- Program Stopped -"); + result.outcome = RunOutcome::CANCELLED; }catch (ProgramCancelledException&){ env->log_to_ui("- Program Stopped -"); + result.outcome = RunOutcome::CANCELLED; }catch (ProgramFinishedException& e){ logger().log("Program finished early!", COLOR_BLUE); env->log_to_ui("- Program Finished -"); @@ -372,6 +423,8 @@ void ProgramSession::run_program(){ e.message(), e.screenshot_view() ); + result.outcome = RunOutcome::COMPLETED; + result.frame = e.screenshot(); }catch (InvalidConnectionStateException& e){ logger().log("Program stopped due to connection issue.", COLOR_RED); env->log_to_ui("- Invalid Connection -", COLOR_RED); @@ -380,6 +433,7 @@ void ProgramSession::run_program(){ message = e.name(); } report_error(message); + result.error = message; }catch (FatalProgramException& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -390,6 +444,8 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; + result.frame = e.screenshot(); }catch (OperationFailedExceptionWithScreenshot& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -400,6 +456,8 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; + result.frame = e.screenshot(); }catch (OperationFailedException& e){ // no screenshot logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -410,6 +468,7 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; }catch (Exception& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -422,6 +481,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, message ); + result.error = message; }catch (std::exception& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -434,6 +494,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, message ); + result.error = message; }catch (...){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Unknown Error -", COLOR_RED); @@ -442,6 +503,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, "Unknown error." ); + result.error = "Unknown error."; } } @@ -451,6 +513,7 @@ void ProgramSession::run_program(){ update_historical_stats_with_current(); set_state(ProgramState::STOPPED); } + return result; } diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.h b/SerialPrograms/Source/CommonFramework/ProgramSession.h index 664709cd3e..36889c5c30 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.h +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.h @@ -17,6 +17,7 @@ #define PokemonAutomation_CommonFramework_ProgramSession_H #include +#include #include #include #include "Common/Cpp/Logging/TaggedLogger.h" @@ -29,6 +30,7 @@ #include "CommonFramework/Globals.h" //#include "CommonFramework/Logging/Logger.h" #include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/RunRecord.h" #include "CommonFramework/Tools/ProgramEnvironment.h" // #include "CommonFramework/ResourceDownload/ProgramMissingResourceTracker.h" #include "Integrations/ProgramTrackerInterfaces.h" @@ -66,7 +68,12 @@ class ProgramSession : public TrackableProgram{ public: // Setup - ProgramSession(const ProgramDescriptor& descriptor); + // `run_records` is where this session's runs are recorded (RunRecord.h); + // a test passes its own store. + ProgramSession( + const ProgramDescriptor& descriptor, + RunRecordStore& run_records = RunRecordStore::instance() + ); // Before the child-most class begins destruction, you must first: // 1. Stop the program. @@ -104,6 +111,21 @@ class ProgramSession : public TrackableProgram{ Mutex& program_lock() const{ return m_lock; } +public: + // Run records (RunRecord.h) + + // The run the last accepted start opened, active or finished; "" before + // the first start. + std::string last_run_id() const{ return m_run_recorder.last_run_id(); } + + // The program's progress hook. Events go to the active run's record. + RunRecorder& run_recorder(){ return m_run_recorder; } + + // The logger the session's consoles chain off, so that their lines land + // in the active run's log tail. + Logger& run_logger(){ return m_run_recorder; } + + public: // Program Control @@ -123,6 +145,9 @@ class ProgramSession : public TrackableProgram{ virtual void internal_run_program(ProgramEnvironment& env) = 0; virtual void internal_stop_program(); + // Starts the program thread. A test fake makes this fail. + virtual AsyncTask dispatch_program_thread(std::function body); + // virtual void restore_defaults(){ return; } public: @@ -153,13 +178,18 @@ class ProgramSession : public TrackableProgram{ private: - void run_program(); + // The program thread: runs the program and closes its run record with + // exactly one result, whichever way run_program() ends. + void run_program_and_record(); + RunResult run_program(); + std::string abandon_run(std::string message); private: const ProgramDescriptor& m_descriptor; uint64_t m_instance_id = 0; + SessionRunRecorder m_run_recorder; // Before m_logger, which chains off it. TaggedLogger m_logger; diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp index 7c1c57c1e0..aedd954fa0 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp @@ -2,13 +2,24 @@ * * From: https://github.com/PokemonAutomation/ * - * The store on its own. + * The store on its own, and the run lifecycle through a ProgramSession + * whose program is a scripted fake: no console, no window. * */ +#include #include #include +#include +#include +#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Notifications/ProgramInfo.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/ProgramSession.h" +#include "CommonFramework/Tools/ProgramEnvironment.h" #include "RunRecord.h" #include "RunRecord_Tests.h" @@ -25,6 +36,20 @@ std::shared_ptr make_frame(size_t width){ return std::make_shared(std::move(image)); } +// Polls `done` every few milliseconds until it holds or `timeout` passes. +bool wait_until(const std::function& done, Milliseconds timeout = 5000ms){ + const WallClock deadline = current_time() + timeout; + while (!done()){ + if (current_time() > deadline){ + return false; + } + std::this_thread::sleep_for(5ms); + } + return true; +} + + + UnitTestResult test_store_ids(){ RunRecordStore store(2); const std::string a = store.begin("Test:A"); @@ -181,6 +206,319 @@ UnitTestResult test_session_recorder(){ +// A program with no console: what it does is scripted per test. +struct FakeScript{ + std::string validity = {}; // check_validity(): "" accepts the start. + bool fail_dispatch = false; // The program thread cannot be started. + std::function start_checks = {}; + std::function body = {}; +}; + +class FakeDescriptor : public ProgramDescriptor{ +public: + FakeDescriptor() + : ProgramDescriptor( + "Test:RunRecordFake", "Test", "Run Record Fake", + "", "", PanelQualityStanding::DEVELOPER, false, {} + ) + {} + virtual std::unique_ptr make_panel() const override{ return nullptr; } +}; + +class FakeInstance : public ProgramInstance{ +public: + explicit FakeInstance(std::function start_checks) + : m_start_checks(std::move(start_checks)) + {} + virtual void run_start_program_checks(const ProgramDescriptor&, ProgramEnvironment& env) override{ + if (m_start_checks){ + m_start_checks(env); + } + } +private: + std::function m_start_checks; +}; + +class FakeProgram : public ProgramSession{ +public: + FakeProgram(const FakeDescriptor& descriptor, RunRecordStore& store, FakeScript script) + : ProgramSession(descriptor, store) + , m_script(std::move(script)) + { + m_instance = std::make_unique(m_script.start_checks); + } + virtual ~FakeProgram(){ + internal_stop_program(); + join_program_thread(); + } + + // Whether the body has started running. + bool running() const{ return m_running.load(std::memory_order_acquire); } + + // Waits for the program thread to end, so the run's record is closed. + bool wait_until_stopped(){ + if (!wait_until([this]{ return current_state() == ProgramState::STOPPED; })){ + return false; + } + join_program_thread(); + return true; + } + + virtual std::string check_validity() const override{ return m_script.validity; } + +protected: + virtual std::unique_ptr make_env(const ProgramInfo& program_info) override{ + return std::make_unique(program_info, *this, current_stats_tracker(), historical_stats_tracker()); + } + virtual void internal_run_program(ProgramEnvironment& env) override{ + m_running.store(true, std::memory_order_release); + if (m_script.body){ + m_script.body(*this, env, *m_scope); + } + } + virtual AsyncTask dispatch_program_thread(std::function body) override{ + if (m_script.fail_dispatch){ + throw std::runtime_error("no thread"); + } + return ProgramSession::dispatch_program_thread(std::move(body)); + } + +private: + FakeScript m_script; + std::atomic m_running{false}; +}; + +// Starts the program, waits for it to end, and returns its record. +UnitTestResult run_to_end(FakeProgram& program, RunRecordStore& store, RunRecord& record){ + const std::string error = program.start_program(); + if (!error.empty()){ + return "The start was refused: " + error; + } + if (!program.wait_until_stopped()){ + return "The program did not stop."; + } + const std::optional found = store.get(program.last_run_id()); + if (!found){ + return "The run has no record."; + } + record = *found; + if (record.status != RunStatus::TERMINAL || !record.result){ + return "The record is not terminal after the program stopped."; + } + return true; +} + +UnitTestResult test_rejection(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.validity = "Pick a save."}); + if (program.start_program() != "Pick a save."){ + return "A start refused by check_validity() returns its message."; + } + if (!store.run_ids().empty() || !program.last_run_id().empty()){ + return "A refused start opens no run."; + } + return true; +} + +UnitTestResult test_immediate_completion(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment& env, CancellableScope&){ + env.run_recorder().event({.kind = "checkpoint", .checkpoint = "00.01", .step = "3/3", .attempt = 1, .text = "ok", .pose = "home (2, 4) N"}); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::COMPLETED || record.result->error || record.result->frame){ + return "A program that returns completed its run, with no error and no frame."; + } + if (record.program != "Test:RunRecordFake" || record.start_time > *record.end_time){ + return "The record names the program and its times."; + } + if (record.events.size() != 1 || record.events[0].checkpoint != "00.01" || record.events[0].attempt != size_t(1) || record.events[0].pose != "home (2, 4) N"){ + return std::format("The program's event is in the record: {} events.", record.events.size()); + } + bool stopped_logged = false; + for (const std::string& line : record.log_tail){ + stopped_logged |= line.find("Program State: STOPPED") != std::string::npos; + } + if (!stopped_logged){ + return "The run's log tail runs through the session's last state change."; + } + return true; +} + +UnitTestResult test_setup_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .start_checks = [](ProgramEnvironment& env){ + throw UserSetupError(env.logger(), "The controller is not ready."); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The controller is not ready."){ + return "A setup error fails the run with its message."; + } + if (record.events.size() != 1 || record.events[0].kind != "error" || record.events[0].text != "The controller is not ready."){ + return "The reported error is an \"error\" event."; + } + return true; +} + +UnitTestResult test_failure_with_frame(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + ImageRGB32 screen(8, 8); + screen.fill(0xffff0000); + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "The sign's box did not open.", nullptr, std::move(screen) + ); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The sign's box did not open."){ + return "The exception's message is the primary error."; + } + if (!record.result->frame || record.result->frame->width() != 8){ + return "The exception's frame is kept with the result."; + } + return true; +} + +UnitTestResult test_cancellation(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope& scope){ + while (true){ + scope.throw_if_cancelled(); + std::this_thread::sleep_for(5ms); + } + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = program.last_run_id(); + if (!program.stop_program().empty() || !program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || record.result->outcome != RunOutcome::CANCELLED || record.result->error){ + return "A stopped program's run is cancelled, with no error."; + } + return true; +} + +// The body waits for the stop request and then ends its own way. +UnitTestResult run_racing_a_stop(const std::function& ending, RunOutcome expected, const char* what){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [&](ProgramSession& self, ProgramEnvironment&, CancellableScope&){ + wait_until([&]{ return self.current_state() == ProgramState::STOPPING; }); + ending(); + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = program.last_run_id(); + program.stop_program(); + if (!program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.result->outcome != expected){ + return std::format("{}: expected {}, got {}.", what, run_outcome_name(expected), run_outcome_name(record.result->outcome)); + } + return true; +} +UnitTestResult test_late_stop(){ + UnitTestResult result = run_racing_a_stop( + []{ throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "Failed anyway."); }, + RunOutcome::FAILED, "A failure observed after the stop request stays a failure" + ); + if (result.result != UnitTestResult::PASSED){ + return result; + } + return run_racing_a_stop( + []{}, RunOutcome::COMPLETED, "A normal return racing the stop request stays completed" + ); +} + +UnitTestResult test_dispatch_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.fail_dispatch = true}); + const std::string error = program.start_program(); + if (error.find("could not be started") == std::string::npos){ + return "A start whose thread cannot be dispatched returns an error: <" + error + ">"; + } + const std::optional record = store.get(program.last_run_id()); + if (!record || record->status != RunStatus::TERMINAL || record->result->outcome != RunOutcome::FAILED){ + return "The accepted run is closed as failed."; + } + if (record->result->error != error){ + return "The run's error is the start's error."; + } + if (program.current_state() != ProgramState::STOPPED){ + return "The session is stopped again, so the next start can try."; + } + return true; +} + +UnitTestResult test_second_run_preserves_first(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram failing(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "first", nullptr, ImageRGB32(8, 8) + ); + }, + }); + RunRecord first; + UnitTestResult result = run_to_end(failing, store, first); + if (result.result != UnitTestResult::PASSED){ + return result; + } + FakeProgram completing(descriptor, store, {}); + RunRecord second; + result = run_to_end(completing, store, second); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (first.run_id == second.run_id || !(first.run_id < second.run_id)){ + return "Each run has its own ID, sorting after the earlier one."; + } + const std::optional kept = store.get(first.run_id); + if (!kept || kept->result->outcome != RunOutcome::FAILED || kept->result->error != "first" || !kept->result->frame){ + return "The first run's result and frame are still there after the second run."; + } + if (second.result->outcome != RunOutcome::COMPLETED || store.run_ids() != std::vector{first.run_id, second.run_id}){ + return "The second run has its own, completed, record."; + } + return true; +} + + class Test_RunRecord : public UnitTest{ public: Test_RunRecord(std::string name, std::function body) @@ -204,6 +542,14 @@ void add_tests_RunRecord(UnitTestDatabase& database){ database.add("store: events, frames and log tail", test_store_events_and_log); database.add("store: old runs release frames", test_store_releases_old_frames); database.add("session recorder", test_session_recorder); + database.add("lifecycle: rejected start", test_rejection); + database.add("lifecycle: immediate completion", test_immediate_completion); + database.add("lifecycle: setup failure", test_setup_failure); + database.add("lifecycle: failure with frame", test_failure_with_frame); + database.add("lifecycle: cancellation", test_cancellation); + database.add("lifecycle: late stop", test_late_stop); + database.add("lifecycle: dispatch failure", test_dispatch_failure); + database.add("lifecycle: second run preserves the first", test_second_run_preserves_first); } diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp index 07e95a1a16..5814e7aaab 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp @@ -33,6 +33,10 @@ const ProgramInfo& ProgramEnvironment::program_info() const{ return m_program_info; } +RunRecorder& ProgramEnvironment::run_recorder(){ + return m_session.run_recorder(); +} + void ProgramEnvironment::update_stats(){ m_session.report_stats_changed(); diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h index 72d6d812df..45068ec33c 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h @@ -9,6 +9,7 @@ #include "Common/Compiler.h" #include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/RunRecord.h" namespace PokemonAutomation{ @@ -42,6 +43,12 @@ class ProgramEnvironment{ virtual void log_to_ui(const std::string& msg, Color color = Color()){} +public: + // Progress reporting (RunRecord.h): what the program says about its + // checkpoints, steps and attempts goes to this run's record. + RunRecorder& run_recorder(); + + public: // Stats Management diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp index 2cf4ceb24d..d56f6c79c4 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp @@ -7,6 +7,7 @@ #include "Common/Cpp/EarlyShutdown.h" #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Logging/GlobalLogger.h" #include "ConsolePanelSession.h" namespace PokemonAutomation{ @@ -28,7 +29,7 @@ ConsolePanelSession::ConsolePanelSession(const ConsolePanelDescriptor& descripto : UiState(descriptor) , m_descriptor(descriptor) , m_system_option(descriptor.num_controllers()) - , m_system(m_system_option, true, 0) + , m_system(global_logger_raw(), m_system_option, true, 0) , m_instance(descriptor.make_instance(m_system)) {} diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp index f8c81bbd7a..8b95f560c7 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp @@ -39,6 +39,7 @@ ConsoleProgramSession::ConsoleProgramSession( , m_descriptor(descriptor) , m_system_option(1, std::move(extra_option)) , m_system( + run_logger(), m_system_option, descriptor.allow_commands_while_running(), 0, diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp index 0e9e08a0a2..cb14619341 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp @@ -50,6 +50,7 @@ ConsoleSystemSession::~ConsoleSystemSession(){ ); } ConsoleSystemSession::ConsoleSystemSession( + Logger& base_logger, ConsoleSystemOption& option, bool allow_commands_while_locked, size_t console_number, @@ -57,7 +58,7 @@ ConsoleSystemSession::ConsoleSystemSession( ) : m_allow_commands_while_locked(allow_commands_while_locked) , m_console_number(console_number) - , m_logger(global_logger_raw(), "Console " + std::to_string(console_number)) + , m_logger(base_logger, "Console " + std::to_string(console_number)) , m_option(option) , m_video(m_logger, option.m_video) , m_audio(m_logger, option.m_audio) diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h index 62ec876c71..8c08e359a8 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h @@ -66,7 +66,11 @@ class ConsoleSystemSession final virtual bool try_shutdown() noexcept override; ~ConsoleSystemSession(); + // `base_logger` is what this console's logger writes through: a + // program session passes its run logger (ProgramSession::run_logger()), + // so the console's lines land in the run's record. ConsoleSystemSession( + Logger& base_logger, ConsoleSystemOption& option, bool allow_commands_while_locked, size_t console_number, diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp index 3f6cbb6e18..6ed9c67747 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp @@ -47,7 +47,7 @@ MultiConsoleProgramSession::MultiConsoleProgramSession( descriptor.default_consoles() ) ) - , m_system(*m_system_option, descriptor.allow_commands_while_running(), instance_id()) + , m_system(run_logger(), *m_system_option, descriptor.allow_commands_while_running(), instance_id()) { auto instance = descriptor.make_instance(m_system); instance->update_active_consoles(m_system_option->active_consoles()); diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp index dde05b293c..613d62dabf 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp @@ -40,11 +40,13 @@ MultiConsoleSystemSession::~MultiConsoleSystemSession(){ } MultiConsoleSystemSession::MultiConsoleSystemSession( + Logger& base_logger, MultiConsoleSystemOption& option, bool allow_commands_while_locked, std::optional program_tracking_id ) : m_logger(global_logger_tagged()) + , m_base_logger(base_logger) , m_option(option) , m_allow_commands_while_locked(allow_commands_while_locked) , m_program_tracking_id(program_tracking_id) @@ -54,6 +56,7 @@ MultiConsoleSystemSession::MultiConsoleSystemSession( for (size_t c = 0; c < count; c++){ m_consoles.emplace_back( std::make_unique( + m_base_logger, option[c], allow_commands_while_locked, c, @@ -121,6 +124,7 @@ void MultiConsoleSystemSession::set_active_consoles(size_t count){ for (size_t c = 0; c < count; c++){ m_consoles.emplace_back( std::make_unique( + m_base_logger, m_option[c], m_allow_commands_while_locked, c, diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h index e31786be68..117ccf260d 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h @@ -47,7 +47,10 @@ class MultiConsoleSystemSession // Construct the multi-console session. // By default it constructs the generic console. But you can inject your // own factory for your own custom subclass. + // `base_logger` is what the consoles' loggers write through; see + // ConsoleSystemSession. MultiConsoleSystemSession( + Logger& base_logger, MultiConsoleSystemOption& option, bool allow_commands_while_locked, std::optional program_tracking_id = {} @@ -85,6 +88,7 @@ class MultiConsoleSystemSession private: Logger& m_logger; + Logger& m_base_logger; // What the consoles' loggers write through. MultiConsoleSystemOption& m_option; bool m_allow_commands_while_locked; From ca345ad2b3f3464f9d58cf62bc40eed58f977b57 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 00:44:41 -0700 Subject: [PATCH 07/16] Lab channel: write each run to Lab/runs// and drop the sessions.* prototype LabRunWriter copies the run records into files on the channel's poll: result.json (status, outcome, error, times, frame, last event; final once "status" is "terminal"), events.jsonl (one line per event, appended as the program reports them), log.txt (the run's log tail) and frame-*.png, with the newest 32 run folders kept. "start" prints run= and the state file gains a run= line, so a script can wait on that run's result.json. A shot or detect taken during a run, or within two minutes after it ended, is appended to the run's events.jsonl as a probe and its screenshot copied in. The session_agent command, the SessionAgentPrototype sources and the terminal demo go; nothing of sessions.* remains. LabChannel tests cover result.json's shape at the indents labctl.sh greps, events.jsonl lines and frame names, the writer with probes and pruning, and run= in the state text. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../CommonFramework/Windows/LabChannel.cpp | 95 +++--- .../CommonFramework/Windows/LabChannel.h | 17 +- .../Windows/LabChannel_Tests.cpp | 209 +++++++++++- .../CommonFramework/Windows/LabRequest.cpp | 1 + .../CommonFramework/Windows/LabRequest.h | 11 +- .../CommonFramework/Windows/LabRunFiles.cpp | 312 ++++++++++++++++++ .../CommonFramework/Windows/LabRunFiles.h | 109 ++++++ .../SessionAgentPrototype/PrototypeDemo.cpp | 104 ------ .../SessionAgentPrototype/README.md | 157 --------- .../SessionAgentPrototype.cpp | 64 ---- .../SessionAgentPrototype.h | 81 ----- SerialPrograms/cmake/SourceFiles.cmake | 4 +- devtools/prototypes/session-agent/run-demo.sh | 9 - 13 files changed, 699 insertions(+), 474 deletions(-) create mode 100644 SerialPrograms/Source/CommonFramework/Windows/LabRunFiles.cpp create mode 100644 SerialPrograms/Source/CommonFramework/Windows/LabRunFiles.h delete mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp delete mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md delete mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp delete mode 100644 SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h delete mode 100644 devtools/prototypes/session-agent/run-demo.sh diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabChannel.cpp b/SerialPrograms/Source/CommonFramework/Windows/LabChannel.cpp index b4f386a6dd..a77a2cb775 100644 --- a/SerialPrograms/Source/CommonFramework/Windows/LabChannel.cpp +++ b/SerialPrograms/Source/CommonFramework/Windows/LabChannel.cpp @@ -17,7 +17,6 @@ #include "CommonFramework/Logging/Logger.h" #include "CommonFramework/ProgramSession.h" #include "CommonFramework/Tools/GlobalThreadPools.h" -#include "Integrations/SessionAgentPrototype/SessionAgentPrototype.h" #include "LabChannel.h" namespace PokemonAutomation{ @@ -34,31 +33,6 @@ constexpr Milliseconds POLL_INTERVAL = 500ms; constexpr Milliseconds UI_WAIT_SLICE = 100ms; -void write_file(const Filesystem::Path& path, const std::string& text){ - FileIO file; - if (file.open(path, FileMode::WRITE | FileMode::BINARY)){ - file.write(text); - } -} - -// Writes `text` to `path` in one step, so a reader never sees a partial file. -void write_file_atomically(const Filesystem::Path& path, const std::string& text){ - const Filesystem::Path temp = Filesystem::unique_temp_path(path); - write_file(temp, text); - std::error_code ec; - Filesystem::rename(temp, path, ec); -} - -// The prototype runs through the same owner check as other channel writes. -std::string run_session_agent_request(const LabRequestLine& line, LabCommandOutput& output){ - const size_t begin = line.text.find_first_not_of(" \t"); - const size_t space = line.text.find_first_of(" \t", begin); - const std::string text = space == std::string::npos ? "" : line.text.substr(space + 1); - const auto reply = SessionAgentPrototype::handle_live_request(text); - output.out << reply.dump() << "\n"; - return reply.at("ok").get() ? "" : reply.at("error").at("message").get(); -} - // Returns the error message of the start or stop, or empty on success. // Unlike the Start/Stop button, a refused start (e.g. an option that fails // check_validity()) goes to the journal, so the request stops there. @@ -185,6 +159,7 @@ std::unique_ptr LabChannel::start_if_enabled( LabChannel::LabChannel(LabChannelCallbacks callbacks, std::unique_ptr console) : m_callbacks(std::move(callbacks)) , m_console(std::move(console)) + , m_runs(folder() + "runs/") { // A pool task rather than a Thread: Thread::join() does not wait under // the Qt thread-adoption workaround, and the worker must be gone before @@ -263,6 +238,7 @@ void LabChannel::poll(){ handle_request(*file); } keep_awake(); + m_runs.sync(RunRecordStore::instance()); write_state(); } @@ -286,7 +262,9 @@ void LabChannel::handle_request(const Filesystem::Path& file){ for (const LabRequestLine& line : request.lines){ const WallClock line_start = current_time(); output.line = line.number; + const size_t journal_before = out.str().size(); error = run_line(request, line, output); + record_probe(line, out.str().substr(journal_before), error); if (!error.empty()){ out << journal_error(line.number, line.text, error); break; @@ -296,10 +274,10 @@ void LabChannel::handle_request(const Filesystem::Path& file){ std::chrono::duration_cast(line_start - start), std::chrono::duration_cast(current_time() - start) ); - write_file(tmp_path, out.str()); + write_lab_file(tmp_path, out.str()); } out << "end\n"; - write_file(tmp_path, out.str()); + write_lab_file(tmp_path, out.str()); Filesystem::remove(out_path); Filesystem::rename(tmp_path, out_path); @@ -353,10 +331,25 @@ std::string LabChannel::run_line(const LabRequest& request, const LabRequestLine if (command == "keepawake"){ return m_keep_awake.set(words, current_time()); } - if (kind == LabConsoleCommands::Kind::UNKNOWN && command != "session_agent"){ + if (kind == LabConsoleCommands::Kind::UNKNOWN){ return "Unknown command: " + command + " (try \"help\")"; } - return command == "session_agent" ? run_session_agent_request(line, output) : run_console_command(words, output); + return run_console_command(words, output); +} + +// A screen read (shot, detect, ...) taken during a run, or right after one, +// goes into that run's folder too, so a probe of a failure sits beside the +// failure. +void LabChannel::record_probe(const LabRequestLine& line, const std::string& output, const std::string& error){ + if (m_console->kind(line.words[0]) != LabConsoleCommands::Kind::READS_STREAM){ + return; + } + m_runs.sync(RunRecordStore::instance()); + const std::optional run = m_runs.run_for_probe(current_time()); + if (!run){ + return; + } + m_runs.attach_probe(*run, line.text, error.empty() ? output : output + "error: " + error + "\n"); } std::string LabChannel::run_console_command(const std::vector& words, LabCommandOutput& output){ @@ -444,9 +437,20 @@ std::string LabChannel::run_ui_command(const LabRequestLine& line, std::ostream& return error; } const bool start = command == "start"; - return run_on_ui([&]{ - return start_or_stop(m_callbacks.current_panel(), start); + std::string run_id; + const std::string error = run_on_ui([&]{ + PanelSession* panel = m_callbacks.current_panel(); + std::string error = start_or_stop(panel, start); + if (error.empty() && start){ + run_id = dynamic_cast(panel)->last_run_id(); + } + return error; }); + if (error.empty() && start){ + // The run's ID, for Lab/runs// and "labctl.sh wait". + out << "run=" << run_id << "\n"; + } + return error; } // Runs `body` on the UI thread and returns what it returns: "" or an error. @@ -494,8 +498,7 @@ void LabChannel::keep_awake(){ bool running = false; { std::lock_guard lg(m_panel_lock); - std::string state, program; - read_program_state(state, program, running); + running = read_program_state().running; } if (!m_keep_awake.due(current_time(), running)){ return; @@ -508,30 +511,34 @@ void LabChannel::keep_awake(){ } } -void LabChannel::read_program_state(std::string& state, std::string& program, bool& running) const{ +LabChannel::SelectedProgram LabChannel::read_program_state() const{ const ProgramSession* session = dynamic_cast(m_panel); - program = m_panel == nullptr ? "" : m_panel->descriptor().identifier(); + SelectedProgram selected; + selected.program = m_panel == nullptr ? "" : m_panel->descriptor().identifier(); if (session == nullptr){ - state = "NO_PROGRAM"; - running = false; - return; + selected.state = "NO_PROGRAM"; + return selected; } const ProgramState current = session->current_state(); - state = program_state_name(current); - running = current != ProgramState::STOPPED; + selected.state = program_state_name(current); + selected.run = session->last_run_id(); + selected.running = current != ProgramState::STOPPED; + return selected; } void LabChannel::write_state(){ { std::lock_guard lg(m_panel_lock); - bool running; - read_program_state(m_state.state, m_state.program, running); + const SelectedProgram selected = read_program_state(); + m_state.state = selected.state; + m_state.program = selected.program; + m_state.run = selected.run; } m_state.owner = m_owner.token(); std::string text = lab_state_text(m_state); const Filesystem::Path path(folder() + "state"); if (text != m_written_state || !Filesystem::exists(path)){ - write_file_atomically(path, text); + write_lab_file_atomically(path, text); m_written_state = std::move(text); } } diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabChannel.h b/SerialPrograms/Source/CommonFramework/Windows/LabChannel.h index c94e171035..47ee9c5d5f 100644 --- a/SerialPrograms/Source/CommonFramework/Windows/LabChannel.h +++ b/SerialPrograms/Source/CommonFramework/Windows/LabChannel.h @@ -12,7 +12,8 @@ * order from a worker thread and then moved to "Lab/done/". Its journal * "Lab/outbox/.out" has one line per command and ends with "end". * "Lab/state" is rewritten after every request and whenever the program - * state changes. + * state changes. Each run of the program is written to "Lab/runs//" + * as it goes (LabRunFiles.h). * * The channel owns the worker and the files. Anything that touches the * window (select, start, stop, set) runs on the UI thread through the @@ -38,6 +39,7 @@ #include "CommonFramework/Globals.h" #include "CommonFramework/Panels/PanelSession.h" #include "LabRequest.h" +#include "LabRunFiles.h" namespace PokemonAutomation{ @@ -120,6 +122,7 @@ class LabChannel{ void poll(); void handle_request(const Filesystem::Path& file); std::string run_line(const LabRequest& request, const LabRequestLine& line, LabCommandOutput& output); + void record_probe(const LabRequestLine& line, const std::string& output, const std::string& error); std::string run_console_command(const std::vector& words, LabCommandOutput& output); std::string run_on_console(const std::function& body); std::string run_ui_command(const LabRequestLine& line, std::ostream& out); @@ -127,8 +130,15 @@ class LabChannel{ void keep_awake(); void write_state(); - // The state of the selected program. Call under m_panel_lock. - void read_program_state(std::string& state, std::string& program, bool& running) const; + // The state of the selected program, as the state file shows it. + struct SelectedProgram{ + std::string state = {}; + std::string program = {}; + std::string run = {}; + bool running = false; + }; + // Call under m_panel_lock. + SelectedProgram read_program_state() const; private: @@ -146,6 +156,7 @@ class LabChannel{ // Owned by the worker thread. LabOwner m_owner; KeepAwakeSchedule m_keep_awake; + LabRunWriter m_runs; LabState m_state; std::string m_written_state; bool m_folder_ready = false; diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabChannel_Tests.cpp b/SerialPrograms/Source/CommonFramework/Windows/LabChannel_Tests.cpp index 7d11dcf8f2..cf61b46d29 100644 --- a/SerialPrograms/Source/CommonFramework/Windows/LabChannel_Tests.cpp +++ b/SerialPrograms/Source/CommonFramework/Windows/LabChannel_Tests.cpp @@ -3,7 +3,7 @@ * From: https://github.com/PokemonAutomation/ * * The channel's text formats, with no app: request files, the journal, the - * "state" file, the owner lock and the keep-awake schedule. + * "state" file, the owner lock, the keep-awake schedule and the run files. * */ @@ -11,8 +11,11 @@ #include #include #include "Common/Cpp/Filesystem/FileIO.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/TestRunners/UnitTest.h" #include "LabRequest.h" +#include "LabRunFiles.h" #include "LabChannel_Tests.h" namespace PokemonAutomation{ @@ -139,12 +142,15 @@ UnitTestResult test_journal(){ } UnitTestResult test_state_text(){ - LabState state{"r1", "RUNNING", "PokemonBDSP:AutoStory", "lab-B", ""}; - if (lab_state_text(state) != "seq=r1\nstate=RUNNING\nprogram=PokemonBDSP:AutoStory\nowner=lab-B\nerror=\n"){ + LabState state{ + .seq = "r1", .state = "RUNNING", .program = "PokemonBDSP:AutoStory", + .run = "20261101-153012123456-007", .owner = "lab-B", .error = "", + }; + if (lab_state_text(state) != "seq=r1\nstate=RUNNING\nprogram=PokemonBDSP:AutoStory\nrun=20261101-153012123456-007\nowner=lab-B\nerror=\n"){ return "Unexpected state text: " + lab_state_text(state); } - state = {"", "NO_PROGRAM", "", "", "The selected panel is not a program."}; - if (lab_state_text(state) != "seq=\nstate=NO_PROGRAM\nprogram=\nowner=\nerror=The selected panel is not a program.\n"){ + state = {.state = "NO_PROGRAM", .error = "The selected panel is not a program."}; + if (lab_state_text(state) != "seq=\nstate=NO_PROGRAM\nprogram=\nrun=\nowner=\nerror=The selected panel is not a program.\n"){ return "Unexpected state text: " + lab_state_text(state); } return true; @@ -227,6 +233,195 @@ UnitTestResult test_keep_awake(){ } +std::shared_ptr test_frame(){ + ImageRGB32 image(4, 4); + image.fill(0xff0000ff); + return std::make_shared(std::move(image)); +} + +// Whether `text` has the line `"key": value,` at `indent` spaces, as the +// 4-space result.json puts a top-level (4) or last_event (8) field, which is +// what labctl.sh's "wait" greps for. +bool has_json_line(const std::string& text, size_t indent, const std::string& key, const std::string& value){ + const std::string line = std::format("\n{}\"{}\": {}", std::string(indent, ' '), key, value); + return text.find(line + ",\n") != std::string::npos || text.find(line + "\n") != std::string::npos; +} + +UnitTestResult test_time_text(){ + const std::string text = lab_time_text(current_time()); + if (text.size() != 23 || text[4] != '-' || text[7] != '-' || text[10] != ' ' || text[13] != ':' || text[16] != ':' || text[19] != '.'){ + return "Expected \"YYYY-MM-DD HH:MM:SS.mmm\", got " + text; + } + return true; +} + +UnitTestResult test_event_json(){ + const WallClock t0 = current_time(); + RunEvent event{ + .time = t0, .kind = "attempt", .checkpoint = "01.08", .step = "4/7", .attempt = 2, + .text = "the sign's box did not open", .frame = test_frame(), .pose = "route_204 (12, 41) N", + }; + const std::string frame = event_frame_file(4, event); + if (frame != "frame-004-01.08-s4_7-a2.png"){ + return "Unexpected frame file name: " + frame; + } + const std::string expected = std::format( + "{{\"attempt\":2,\"checkpoint\":\"01.08\",\"frame\":\"{}\",\"kind\":\"attempt\",\"pose\":\"route_204 (12, 41) N\"," + "\"step\":\"4/7\",\"text\":\"the sign's box did not open\",\"time\":\"{}\"}}", + frame, lab_time_text(t0) + ); + if (run_event_json_line(event, frame) != expected){ + return "Unexpected event line: " + run_event_json_line(event, frame); + } + const RunEvent bare{.time = t0, .kind = "checkpoint", .text = "00.01 ok"}; + if (event_frame_file(0, bare) != "frame-000.png"){ + return "A bare event's frame is named by its index alone: " + event_frame_file(0, bare); + } + const std::string line = run_event_json_line(bare, std::nullopt); + if (line != std::format("{{\"kind\":\"checkpoint\",\"text\":\"00.01 ok\",\"time\":\"{}\"}}", lab_time_text(t0))){ + return "Fields an event does not have are left out: " + line; + } + return true; +} + +UnitTestResult test_result_json(){ + const WallClock t0 = current_time(); + RunRecord record{ + .run_id = "20261101-153012123456-007", .program = "PokemonBDSP:AutoStory", + .start_time = t0, .status = RunStatus::RUNNING, + }; + std::string text = run_result_json(record, std::nullopt, {}); + if (!has_json_line(text, 4, "status", "\"running\"") || !has_json_line(text, 4, "event_count", "0") || + text.find("outcome") != std::string::npos || text.find("last_event") != std::string::npos + ){ + return "A running record has a status and no outcome or last event:\n" + text; + } + + record.events.push_back({.time = t0, .kind = "checkpoint", .checkpoint = "01.07", .text = "ok"}); + record.events.push_back({.time = t0, .kind = "attempt", .checkpoint = "01.08", .step = "4/7", .attempt = 2, .text = "failed"}); + record.status = RunStatus::TERMINAL; + record.end_time = t0 + std::chrono::seconds(90); + record.result = RunResult{.outcome = RunOutcome::FAILED, .error = "expect_failed: the Route 204 sign's box did not open"}; + record.log_truncated = true; + text = run_result_json(record, "frame-failure.png", {{1, "frame-001-01.08-s4_7-a2.png"}}); + const JsonValue json = parse_json(text); + const JsonObject* object = json.to_object(); + if (object == nullptr){ + return "result.json should parse:\n" + text; + } + if (!has_json_line(text, 4, "status", "\"terminal\"") || + !has_json_line(text, 4, "outcome", "\"failed\"") || + !has_json_line(text, 4, "error", "\"expect_failed: the Route 204 sign's box did not open\"") || + !has_json_line(text, 4, "frame", "\"frame-failure.png\"") || + !has_json_line(text, 4, "duration_ms", "90000") || + !has_json_line(text, 4, "event_count", "2") || + !has_json_line(text, 4, "log_truncated", "true") || + !has_json_line(text, 4, "run_id", "\"20261101-153012123456-007\"") + ){ + return "Unexpected top-level fields:\n" + text; + } + if (!has_json_line(text, 8, "checkpoint", "\"01.08\"") || + !has_json_line(text, 8, "step", "\"4/7\"") || + !has_json_line(text, 8, "attempt", "2") || + !has_json_line(text, 8, "frame", "\"frame-001-01.08-s4_7-a2.png\"") + ){ + return "last_event should be the newest event, with its frame, at 8 spaces:\n" + text; + } + if (object->get_string_throw("end_time") != lab_time_text(*record.end_time)){ + return "end_time is the record's end."; + } + return true; +} + +UnitTestResult test_run_writer(){ + const Filesystem::Path root = Filesystem::unique_temp_path( + Filesystem::application_scratch_path() / "LabRunWriterTest" + ); + const std::string runs = root.string() + "/runs/"; + Filesystem::create_directories(root); + std::string error; + { + RunRecordStore store; + LabRunWriter writer(runs, 2); + const WallClock t0 = current_time(); + + if (writer.run_for_probe(t0)){ + error = "No run, no probe target."; + } + const std::string a = store.begin("Test:A"); + store.event(a, {.kind = "checkpoint", .checkpoint = "00.01", .text = "ok", .frame = test_frame()}); + writer.sync(store); + const std::string folder = writer.run_folder(a); + std::string text; + if (error.empty() && (!file_to_string(folder + "result.json", text) || !has_json_line(text, 4, "status", "\"running\""))){ + error = "After the first sync the run has a folder with a running result.json: " + text; + } + if (error.empty() && !Filesystem::is_regular_file(Filesystem::Path(folder + "frame-000-00.01.png"))){ + error = "The event's frame is saved under its event name."; + } + if (error.empty() && writer.run_for_probe(t0) != a){ + error = "A probe during the run belongs to it."; + } + + // A probe: its text goes into events.jsonl and its shot is copied. + const std::string shot = root.string() + "/probe_after_a.png"; + test_frame()->save(shot); + writer.attach_probe(a, "shot after_a", "shot " + shot + "\n"); + store.event(a, {.kind = "step", .step = "2/3", .text = "walk"}); + store.log_line(a, "line 1"); + store.log_line(a, "line 2"); + store.finish(a, {.outcome = RunOutcome::FAILED, .error = "the box did not open", .frame = test_frame()}); + writer.sync(store); + if (error.empty() && (!file_to_string(folder + "result.json", text) || !has_json_line(text, 4, "outcome", "\"failed\"") || !has_json_line(text, 4, "frame", "\"frame-failure.png\""))){ + error = "After the run ends result.json is terminal with the outcome and the frame: " + text; + } + if (error.empty() && !Filesystem::is_regular_file(Filesystem::Path(folder + "frame-failure.png"))){ + error = "The failure frame is saved."; + } + if (error.empty() && (!file_to_string(folder + "log.txt", text) || text != "line 1\nline 2\n")){ + error = "log.txt is the run's log tail: " + text; + } + if (error.empty() && file_to_string(folder + "events.jsonl", text)){ + size_t lines = 0; + for (char ch : text){ + lines += ch == '\n'; + } + if (lines != 3 || text.find("\"kind\":\"probe\"") == std::string::npos || text.find("\"frame\":\"probe-probe_after_a.png\"") == std::string::npos){ + error = "events.jsonl has the two events and the probe between them, naming the copied shot:\n" + text; + } + }else if (error.empty()){ + error = "events.jsonl should exist."; + } + if (error.empty() && !Filesystem::is_regular_file(Filesystem::Path(folder + "probe-probe_after_a.png"))){ + error = "The probe's shot is copied into the run's folder."; + } + if (error.empty() && writer.run_for_probe(current_time()) != a){ + error = "A probe right after the run ended still belongs to it."; + } + if (error.empty() && writer.run_for_probe(current_time() + LabRunWriter::PROBE_WINDOW + std::chrono::seconds(1))){ + error = "A probe long after the run ended belongs to no run."; + } + + // The newest `keep` folders stay. + const std::string b = store.begin("Test:B"); + writer.sync(store); + const std::string c = store.begin("Test:C"); + writer.sync(store); + if (error.empty() && (Filesystem::is_directory(Filesystem::Path(folder)) || !Filesystem::is_directory(Filesystem::Path(writer.run_folder(b))) || !Filesystem::is_directory(Filesystem::Path(writer.run_folder(c))))){ + error = "With keep=2, the third run removes the first run's folder and keeps the other two."; + } + if (error.empty() && writer.run_for_probe(current_time()) != c){ + error = "A probe belongs to the newest run."; + } + } + Filesystem::remove_all(root); + if (!error.empty()){ + return error; + } + return true; +} + + class Test_LabChannel : public UnitTest{ public: Test_LabChannel(std::string name, std::function body) @@ -253,6 +448,10 @@ void add_tests_LabChannel(UnitTestDatabase& database){ database.add("state text", test_state_text); database.add("owner lock", test_owner); database.add("keep-awake schedule", test_keep_awake); + database.add("run files: time text", test_time_text); + database.add("run files: event line and frame name", test_event_json); + database.add("run files: result.json", test_result_json); + database.add("run files: writer, probes and pruning", test_run_writer); } diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabRequest.cpp b/SerialPrograms/Source/CommonFramework/Windows/LabRequest.cpp index c8dac21a11..fb2124a44c 100644 --- a/SerialPrograms/Source/CommonFramework/Windows/LabRequest.cpp +++ b/SerialPrograms/Source/CommonFramework/Windows/LabRequest.cpp @@ -118,6 +118,7 @@ std::string lab_state_text(const LabState& state){ "seq=" + state.seq + "\n" + "state=" + state.state + "\n" + "program=" + state.program + "\n" + + "run=" + state.run + "\n" + "owner=" + state.owner + "\n" + "error=" + state.error + "\n"; } diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabRequest.h b/SerialPrograms/Source/CommonFramework/Windows/LabRequest.h index 52958b859c..9cf0feff64 100644 --- a/SerialPrograms/Source/CommonFramework/Windows/LabRequest.h +++ b/SerialPrograms/Source/CommonFramework/Windows/LabRequest.h @@ -60,11 +60,12 @@ std::string journal_error(size_t number, const std::string& command, const std:: // What the "state" file says. struct LabState{ - std::string seq; // Name of the last request handled. - std::string state; // STOPPED, RUNNING, STOPPING, NOT_READY or NO_PROGRAM. - std::string program; // The selected program's identifier, or "". - std::string owner; // The owner token, or "". - std::string error; // The last error; "" once a request succeeds. + std::string seq = {}; // Name of the last request handled. + std::string state = {}; // STOPPED, RUNNING, STOPPING, NOT_READY or NO_PROGRAM. + std::string program = {}; // The selected program's identifier, or "". + std::string run = {}; // The selected program's last run (Lab/runs//), or "". + std::string owner = {}; // The owner token, or "". + std::string error = {}; // The last error; "" once a request succeeds. }; std::string lab_state_text(const LabState& state); diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabRunFiles.cpp b/SerialPrograms/Source/CommonFramework/Windows/LabRunFiles.cpp new file mode 100644 index 0000000000..ad769c82ca --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Windows/LabRunFiles.cpp @@ -0,0 +1,312 @@ +/* Lab Run Files + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/FileIO.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "LabRunFiles.h" + +namespace PokemonAutomation{ + +namespace{ + +// A file name from a checkpoint or step label: anything but letters, +// digits, '.', '_' and '-' becomes '_'. +std::string file_name_part(const std::string& text){ + std::string part = text; + for (char& ch : part){ + const bool keep = std::isalnum(static_cast(ch)) || ch == '.' || ch == '_' || ch == '-'; + if (!keep){ + ch = '_'; + } + } + return part; +} + +JsonObject event_json(const RunEvent& event, const std::optional& frame_file){ + JsonObject json; + json["time"] = lab_time_text(event.time); + json["kind"] = event.kind; + if (event.checkpoint){ + json["checkpoint"] = *event.checkpoint; + } + if (event.step){ + json["step"] = *event.step; + } + if (event.attempt){ + json["attempt"] = *event.attempt; + } + json["text"] = event.text; + if (frame_file){ + json["frame"] = *frame_file; + } + if (event.pose){ + json["pose"] = *event.pose; + } + return json; +} + +std::string probe_json_line( + WallClock time, const std::string& command, const std::string& output, + const std::optional& frame_file +){ + JsonObject json; + json["time"] = lab_time_text(time); + json["kind"] = "probe"; + json["text"] = command; + json["output"] = output; + if (frame_file){ + json["frame"] = *frame_file; + } + return JsonValue(std::move(json)).dump(-1); +} + +void append_file(const std::string& path, const std::string& text){ + FileIO file; + if (file.open(Filesystem::Path(path), FileMode::APPEND | FileMode::BINARY)){ + file.write(text); + } +} + +} + + + +std::string lab_time_text(WallClock time){ +#if _WIN32 && _MSC_VER +#pragma warning(disable:4996) +#endif + using namespace std::chrono; + const auto since_epoch = time.time_since_epoch(); + const auto millis = duration_cast(since_epoch - duration_cast(since_epoch)).count(); + const std::time_t tt = system_clock::to_time_t(time); + const std::tm local = *std::localtime(&tt); + return std::format( + "{}-{:02}-{:02} {:02}:{:02}:{:02}.{:03}", + local.tm_year + 1900, local.tm_mon + 1, local.tm_mday, + local.tm_hour, local.tm_min, local.tm_sec, millis + ); +} + + +std::string run_result_json( + const RunRecord& record, + const std::optional& frame_file, + const std::map& event_frames +){ + JsonObject json; + json["run_id"] = record.run_id; + json["program"] = record.program; + json["status"] = run_status_name(record.status); + json["start_time"] = lab_time_text(record.start_time); + if (record.end_time){ + json["end_time"] = lab_time_text(*record.end_time); + json["duration_ms"] = std::chrono::duration_cast(*record.end_time - record.start_time).count(); + } + if (record.result){ + json["outcome"] = run_outcome_name(record.result->outcome); + if (record.result->error){ + json["error"] = *record.result->error; + } + } + if (frame_file){ + json["frame"] = *frame_file; + } + json["event_count"] = record.events.size(); + json["dropped_events"] = record.dropped_events; + json["log_truncated"] = record.log_truncated; + if (!record.events.empty()){ + const size_t last = record.events.size() - 1; + std::optional last_frame; + if (auto it = event_frames.find(last); it != event_frames.end()){ + last_frame = it->second; + } + json["last_event"] = event_json(record.events[last], last_frame); + } + return JsonValue(std::move(json)).dump(4) + "\n"; +} + +std::string run_event_json_line(const RunEvent& event, const std::optional& frame_file){ + return JsonValue(event_json(event, frame_file)).dump(-1); +} + +std::string event_frame_file(size_t index, const RunEvent& event){ + std::string name = std::format("frame-{:03}", index); + if (event.checkpoint){ + name += "-" + file_name_part(*event.checkpoint); + } + if (event.step){ + name += "-s" + file_name_part(*event.step); + } + if (event.attempt){ + name += std::format("-a{}", *event.attempt); + } + return name + ".png"; +} + + +void write_lab_file(const Filesystem::Path& path, const std::string& text){ + FileIO file; + if (file.open(path, FileMode::WRITE | FileMode::BINARY)){ + file.write(text); + } +} +void write_lab_file_atomically(const Filesystem::Path& path, const std::string& text){ + const Filesystem::Path temp = Filesystem::unique_temp_path(path); + write_lab_file(temp, text); + std::error_code ec; + Filesystem::rename(temp, path, ec); +} + + + +LabRunWriter::LabRunWriter(std::string folder, size_t keep) + : m_folder(std::move(folder)) + , m_keep(std::max(keep, 1)) +{} + +void LabRunWriter::sync(const RunRecordStore& store){ + const std::vector ids = store.run_ids(); + for (const std::string& id : ids){ + Written& written = m_written[id]; + if (written.finished){ + continue; + } + const std::optional record = store.get(id); + if (!record){ + continue; + } + if (m_last_run < id){ + m_last_run = id; + m_last_end.reset(); + } + write_run(*record, written); + } + // Forget the runs the store has dropped. + for (auto it = m_written.begin(); it != m_written.end();){ + it = std::ranges::find(ids, it->first) == ids.end() ? m_written.erase(it) : std::next(it); + } +} + +void LabRunWriter::write_run(const RunRecord& record, Written& written){ + const std::string folder = run_folder(record.run_id); + if (written.result_text.empty()){ + Filesystem::create_directories(Filesystem::Path(folder)); + prune(); + } + + std::string lines; + for (size_t i = written.events; i < record.events.size(); i++){ + const RunEvent& event = record.events[i]; + std::optional frame_file; + if (event.frame){ + frame_file = event_frame_file(i, event); + if (event.frame->save(folder + *frame_file)){ + written.event_frames[i] = *frame_file; + }else{ + frame_file.reset(); + } + } + lines += run_event_json_line(event, frame_file) + "\n"; + } + if (!lines.empty()){ + append_file(folder + "events.jsonl", lines); + } + written.events = record.events.size(); + + std::optional frame_file; + if (record.status == RunStatus::TERMINAL){ + if (record.result && record.result->frame){ + frame_file = record.result->outcome == RunOutcome::FAILED ? "frame-failure.png" : "frame-end.png"; + if (!record.result->frame->save(folder + *frame_file)){ + frame_file.reset(); + } + } + std::string log; + if (record.log_truncated){ + log += "[older lines dropped]\n"; + } + for (const std::string& line : record.log_tail){ + log += line + "\n"; + } + write_lab_file_atomically(Filesystem::Path(folder + "log.txt"), log); + } + + const std::string text = run_result_json(record, frame_file, written.event_frames); + if (text != written.result_text){ + write_lab_file_atomically(Filesystem::Path(folder + "result.json"), text); + written.result_text = text; + } + if (record.status == RunStatus::TERMINAL){ + written.finished = true; + if (record.run_id == m_last_run){ + m_last_end = record.end_time; + } + } +} + +void LabRunWriter::prune(){ + std::vector names; + for (const Filesystem::Path& path : Filesystem::list_directory(Filesystem::Path(m_folder))){ + if (Filesystem::is_directory(path)){ + names.emplace_back(path.filename().string()); + } + } + std::ranges::sort(names); + for (size_t i = 0; i + m_keep < names.size(); i++){ + Filesystem::remove_all(Filesystem::Path(m_folder + names[i])); + m_written.erase(names[i]); + } +} + +std::optional LabRunWriter::run_for_probe(WallClock now) const{ + if (m_last_run.empty()){ + return std::nullopt; + } + if (m_last_end && now - *m_last_end > PROBE_WINDOW){ + return std::nullopt; + } + return m_last_run; +} + +void LabRunWriter::attach_probe(const std::string& run_id, const std::string& command, const std::string& output){ + const std::string folder = run_folder(run_id); + if (!Filesystem::is_directory(Filesystem::Path(folder))){ + return; + } + std::optional frame_file; + std::istringstream lines(output); + std::string line; + while (std::getline(lines, line)){ + if (line.rfind("shot ", 0) != 0){ + continue; + } + const Filesystem::Path source(line.substr(5)); + const std::string copy = "probe-" + source.filename().string(); + try{ + Filesystem::remove(Filesystem::Path(folder + copy)); + if (Filesystem::copy_file(source, Filesystem::Path(folder + copy)) && !frame_file){ + frame_file = copy; + } + }catch (std::exception&){ + // A probe that printed a path it did not save: nothing to copy. + } + } + append_file(folder + "events.jsonl", probe_json_line(current_time(), command, output, frame_file) + "\n"); +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabRunFiles.h b/SerialPrograms/Source/CommonFramework/Windows/LabRunFiles.h new file mode 100644 index 0000000000..6623d6ba4a --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Windows/LabRunFiles.h @@ -0,0 +1,109 @@ +/* Lab Run Files + * + * From: https://github.com/PokemonAutomation/ + * + * The lab channel's copy of the run records (CommonFramework/RunRecord.h) + * as files, one folder per run under "Lab/runs//": + * + * result.json The record without its events and log: status, + * outcome, error, times, the frame's file name and the + * last event. Rewritten as the run goes; final once + * "status" is "terminal". + * events.jsonl One JSON object per line, appended as the program + * reports events; console probes (shot, detect) taken + * during or right after the run are appended too. + * log.txt The run's log tail, written when the run ends. + * frame-*.png The frames of events and of the result. + * probe-*.png Screenshots of probes attached to the run. + * + * The newest KEEP folders stay; older ones are removed when a run starts. + * Lab.md documents the shapes. + * + */ + +#ifndef PokemonAutomation_LabRunFiles_H +#define PokemonAutomation_LabRunFiles_H + +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/RunRecord.h" + +namespace PokemonAutomation{ + + +// "2026-11-01 15:30:12.123", local time, as the log writes it. +std::string lab_time_text(WallClock time); + +// The text of result.json. `frame_file` is the result frame's file name in +// the run's folder, if it was written; `event_frames` maps an event's index +// to its frame's file name. +std::string run_result_json( + const RunRecord& record, + const std::optional& frame_file, + const std::map& event_frames +); + +// One line of events.jsonl, without the newline. +std::string run_event_json_line(const RunEvent& event, const std::optional& frame_file); + +// The file name of an event's frame: "frame-[-][-s][-a].png". +std::string event_frame_file(size_t index, const RunEvent& event); + +// Writes `text` to `path`; the atomic one in one step, so a reader never +// sees a partial file. +void write_lab_file(const Filesystem::Path& path, const std::string& text); +void write_lab_file_atomically(const Filesystem::Path& path, const std::string& text); + + +class LabRunWriter{ +public: + static constexpr size_t DEFAULT_KEEP = 32; + + // Probes taken this long after a run ended still belong to it. + static constexpr Seconds PROBE_WINDOW = Seconds(120); + + // `folder` is "Lab/runs/", ending in '/'. It is created on the first run. + explicit LabRunWriter(std::string folder, size_t keep = DEFAULT_KEEP); + + const std::string& folder() const{ return m_folder; } + std::string run_folder(const std::string& run_id) const{ return m_folder + run_id + "/"; } + + // Writes what `store` has that the files don't yet. Call from one + // thread, now and then; nothing is held between calls but counters. + void sync(const RunRecordStore& store); + + // The run a probe taken now belongs to: the active run, or the last + // one if it ended less than PROBE_WINDOW ago. + std::optional run_for_probe(WallClock now) const; + + // Appends a console command and what it printed to the run's + // events.jsonl as a "probe" event, and copies each file a "shot " + // line names into the run's folder as "probe-". + void attach_probe(const std::string& run_id, const std::string& command, const std::string& output); + + +private: + struct Written{ + size_t events = 0; // Events written to events.jsonl so far. + std::map event_frames = {}; + std::string result_text = {}; // The last result.json written. + bool finished = false; + }; + + void write_run(const RunRecord& record, Written& written); + void prune(); + + const std::string m_folder; + const size_t m_keep; + std::map m_written; + std::string m_last_run; // The newest run seen, "" before the first. + std::optional m_last_end; // When it ended, once it has. +}; + + + +} +#endif diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp b/SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp deleted file mode 100644 index 152f04a120..0000000000 --- a/SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* Program Session Agent - Hardware-Free Terminal Demo - * - * From: https://github.com/PokemonAutomation/ - * - * Simulated lifecycle only. The JSON dispatcher is shared with the live adapter. - */ - -#include -#include -#include -#include "SessionAgentPrototype.h" - -using namespace PokemonAutomation::SessionAgentPrototype; - -namespace{ - -struct DemoSession{ - std::string state = "STOPPED"; - std::string start_error; -}; - -std::string operate(std::map& sessions, uint64_t id, const std::string& action){ - auto it = sessions.find(id); - if (it == sessions.end()){ - return "Session ID not found."; - } - DemoSession& session = it->second; - if (action == "start"){ - if (session.state != "STOPPED"){ - return "Program is " + session.state + "."; - } - if (!session.start_error.empty()){ - return session.start_error; - } - session.state = "RUNNING"; - }else{ - if (session.state != "RUNNING"){ - return "Program is " + session.state + "."; - } - session.state = "STOPPING"; - } - return ""; -} - -Json frame(const std::map& sessions){ - Json result = Json::array(); - for (const auto& [id, session] : sessions){ - result.push_back({ - {"session_id", id}, {"program", "Prototype:Session"}, - {"state", session.state}, {"start_error", session.start_error}, - {"console_ids", Json::array()}, {"stats", ""} - }); - } - return result; -} - -Json demo_action(std::map& sessions, Bindings& bindings, const std::string& line){ - if (line == "finish"){ - for (auto& [id, session] : sessions){ - if (session.state == "STOPPING"){ - session.state = "STOPPED"; - } - } - return {{"demo", "Stop completion simulated."}}; - } - if (line == "unload"){ - sessions.erase(1); - return {{"demo", "Session 1 unloaded; its ID is now stale."}}; - } - try{ - return dispatch(bindings, Json::parse(line)); - }catch (const Json::exception& e){ - return failure("invalid_request", e.what()); - } -} - -} - -int main(int argc, char** argv){ - const bool plain = argc > 1 && std::string(argv[1]) == "--plain"; - std::map sessions{ - {1, {.state = "STOPPED", .start_error = ""}}, - {2, {.state = "STOPPED", .start_error = "Example configuration is invalid."}}, - }; - Bindings bindings{ - .list = [&]{ return frame(sessions); }, - .start = [&](uint64_t id){ return operate(sessions, id, "start"); }, - .stop = [&](uint64_t id){ return operate(sessions, id, "stop"); }, - }; - Json reply; - std::string line; - while (true){ - if (!plain){ - std::cout << "\033[2J\033[H\033[1mTHROWAWAY PROTOTYPE: simulated sessions, no console\033[0m\n"; - } - std::cout << "sessions: " << frame(sessions).dump(2) << "\nreply: " << reply.dump(2) << "\n"; - std::cout << "JSON method: sessions.list/status/start/stop; session_id for all except list\n" - << "demo controls: finish | unload | quit\n" << std::flush; - if (!std::getline(std::cin, line) || line == "quit"){ - break; - } - reply = demo_action(sessions, bindings, line); - } -} diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md b/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md deleted file mode 100644 index c4711977e9..0000000000 --- a/SerialPrograms/Source/Integrations/SessionAgentPrototype/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# ProgramSession agent interface — throwaway proof of concept - -Question: can an agent list existing sessions and request start/stop by explicit -session ID, receiving the real rejection reason and distinguishing acceptance -from completion, without owning a special automation program? - -This prototype adds four JSON methods and a lab-channel adapter. It reuses -`ProgramTracker`; it does not introduce a second session registry. The same -dispatcher can be driven in a hardware-free terminal demo. Delete or absorb -this prototype once the interface decision is made. - -For the proposed next step, see [the run results and failure evidence design](DESIGN.md). -It adds per-run outcomes to Lab's shared backend; it is not implemented here. - -## Try the terminal demo - -From the repo root, with a C++23 compiler: - -```sh -sh devtools/prototypes/session-agent/run-demo.sh -``` - -This compiles only the small native demo, into a temporary directory. It does -not build SerialPrograms, connect hardware, or control a running app. Session 1 -can start; session 2 has a simulated configuration error. Enter one request per line: - -```json -{"method":"sessions.list"} -{"method":"sessions.start","session_id":2} -{"method":"sessions.start","session_id":1} -{"method":"sessions.status","session_id":1} -{"method":"sessions.stop","session_id":1} -{"method":"sessions.status","session_id":1} -``` - -The rejected start reports `ok:false`. The accepted stop leaves session 1 in -`STOPPING`; the demo-only command `finish` simulates its eventual completion. -`unload` removes session 1, so requests using that ID fail. `quit` exits; -`--plain` disables terminal clearing for a piped walkthrough. Full simulated -state and the last reply are displayed after each action. - -## Live app adapter - -A build of this branch accepts `session_agent ` through the existing -lab channel. Select and configure a suitable program in the app first. Within -the lab's existing console lock and owner session: - -```sh -printf '%s\n' 'session_agent {"method":"sessions.list"}' | - sh devtools/lab/labctl.sh send sessions-list -``` - -The journal contains a JSON result, followed by its normal `ok`/`end` lines. -Take the `session_id` from that result, then use it explicitly: - -```sh -printf '%s\n' 'session_agent {"method":"sessions.status","session_id":1}' | - sh devtools/lab/labctl.sh send session-status -``` - -Use `sessions.start`/`sessions.stop` with the same shape only for the program -under test: these invoke its actual lifecycle and may produce game/save effects. -When the channel is claimed, all prototype requests require `LAB_OWNER`, -including observations. A failed request emits both the JSON error and the -channel's `error` line, so the batch stops and `labctl.sh` exits unsuccessfully. - -## Reply contract - -- `sessions.list`: current live session snapshots, each with `session_id`, - `program` (descriptor identifier), `state`, `stats`, and `console_ids`. -- `sessions.status`: one snapshot or `session_not_found`. -- `sessions.start` / `sessions.stop`: `operation_rejected` with the session's - actual error, or `{"ok":true,"result":{"session_id":1,"accepted":true}}`. -- Poll status to observe the outcome; `accepted` is not completion. A program - may also finish between polls. There is no per-run history in this prototype. -- IDs belong to this app process. Rediscover after restart. IDs 0, UINT64_MAX, - negative numbers, fractions and booleans are rejected; no implicit selection. - -## What this proves, and what it does not - -`SessionAgentPrototype.h` contains the transport-independent dispatcher; -`SessionAgentPrototype.cpp` binds it to the live tracker. `PrototypeDemo.cpp` -supplies simulated bindings and is excluded from the app build. - -One prerequisite fixes an existing information loss: `TrackableProgram`'s -`async_start`/`async_stop` now return their error strings, and the tracker -propagates them. `ProgramSession` already supplies those strings. Existing -integration callers can now see a rejected operation too. - -This draft is not an MCP server or JSON-RPC implementation. It does not add -program creation/selection, option editing, direct controller inputs, run IDs, -event subscriptions, an operation queue, or a new lifetime/ownership scheme. -The live adapter uses the existing tracker's locking and destruction contract; -concurrent session destruction still needs an explicit audit before production. -Multiple clients are coordinated only by the existing lab-channel mechanism. - -The terminal walkthrough demonstrates that the four-method interface can expose -rejection, accepted-but-still-stopping, and stale IDs. The remaining experiment -is to exercise the live adapter with a deliberately invalid configuration and -a suitable short program, then decide whether to add per-run results and events -before attaching MCP. No live-console validation is claimed for this draft. - -## Concrete target: investigating the Sandgem failure - -In BDSP AutoStory run r109, the player reached Sandgem correctly, but OCR read -the location banner as `andgemltown`, so the location check failed. See the -[AutoStory lessons, section 13](https://github.com/dayfine/Arduino-Source/blob/bdsp/floaroma-mvp/docs/design/AutoStory-Lessons.md#13-floaroma-mvp-from-a-new-game-and-the-lab-around-it-2026-10-29-r107-to-r109). - -Lab can already start the program and report that it stopped. The agent must -then find the corresponding error and screenshot, associate them with this -attempt, and distinguish the original failure from later retries. The current -four-method prototype does not remove that work. - -The useful next slice would support this investigation: - -1. Start the configured checkpoint and receive a unique run ID. -2. Wait for that run's outcome, rather than polling whether the session is stopped. -3. Retrieve its error and failure evidence together. -4. Inspect the frame and log: the location is visibly correct, but its OCR - comparison rejected it. -5. Investigate the matcher with saved-image cases, including a different town - that must still be rejected, then rerun the checkpoint under a new run ID. - -An illustrative future result, **not implemented by this prototype**, is: - -```json -{ - "run_id": "run-109", - "outcome": "failed", - "checkpoint": "00.04", - "error": "Location verification failed", - "failure_screenshot": "...", - "log_excerpt": "..." -} -``` - -The error text above is illustrative too. The interface should preserve the -actual program error and capture the failure evidence before a retry changes -the screen. The matcher fix remains in AutoStory; the interface makes its -investigation dependable and automatable. Run identity, outcome and error -capture belong near `ProgramSession`. Checkpoint details require connecting -the program's progress reporting to that run. - -Other recorded failures show the boundary: - -| Failure | What an agent interface could contribute | What the program still must do | -|---|---|---| -| r84: selecting Water Gun at 0 PP for four minutes | Progress reports and recent captures let the agent recognize repetition and stop the identified run. | Read PP, skip exhausted moves and bound retries. A generic session cannot infer PP without observation or program reporting. | -| r105/r106: reported a pass after talking to the rock beside Roark | Expose a story-effect verification failure, once the program checks it. | Verify that Roark actually left. Lifecycle completion alone cannot establish story success. | -| r109a: the Start click missed after the window resized | Lab's existing explicit start/stop commands already address this. | No new benefit should be attributed to this prototype for removing GUI clicks. | - -The intended unlock is an agent that can run a bounded automation experiment, -identify its first failure, inspect the associated evidence, probe the cause, -change the code and repeat. Structured start/stop alone is groundwork, not -that outcome. A concrete next proof should demonstrate: **a checkpoint fails; -one request returns that run's outcome, error and failure screenshot**. MCP -can expose the same operations afterward; changing transport alone adds little. diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp b/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp deleted file mode 100644 index eabbf27052..0000000000 --- a/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* Program Session Agent - Live Prototype Adapter - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include "Common/Cpp/Exceptions.h" -#include "CommonFramework/Globals.h" -#include "Integrations/ProgramTracker.h" -#include "SessionAgentPrototype.h" - -namespace PokemonAutomation{ -namespace SessionAgentPrototype{ -namespace{ - -const char* state_name(ProgramState state){ - switch (state){ - case ProgramState::NOT_READY: - return "NOT_READY"; - case ProgramState::STOPPED: - return "STOPPED"; - case ProgramState::RUNNING: - return "RUNNING"; - case ProgramState::STOPPING: - return "STOPPING"; - } - throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown program state."); -} - -Json list_live_sessions(){ - Json sessions = Json::array(); - for (const auto& [id, state] : ProgramTracker::instance().all_programs()){ - sessions.push_back({ - {"session_id", id}, {"program", state.program_name}, - {"state", state_name(state.state)}, {"stats", state.stats}, - {"console_ids", state.console_ids} - }); - } - return sessions; -} - -} - -Json handle_live_request(const std::string& text){ - Bindings bindings{ - .list = list_live_sessions, - .start = [](uint64_t id){ return ProgramTracker::instance().start_program(id); }, - .stop = [](uint64_t id){ return ProgramTracker::instance().stop_program(id); }, - }; - try{ - const Json request = Json::parse(text); - return dispatch(bindings, request); - }catch (const Json::exception& e){ - return failure("invalid_request", e.what()); - }catch (Exception& e){ - return failure("operation_failed", e.message()); - }catch (const std::exception& e){ - return failure("operation_failed", e.what()); - } -} - -} -} diff --git a/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h b/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h deleted file mode 100644 index 8dcd028d80..0000000000 --- a/SerialPrograms/Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h +++ /dev/null @@ -1,81 +0,0 @@ -/* Program Session Agent - Throwaway Prototype - * - * From: https://github.com/PokemonAutomation/ - * - * Four JSON requests over existing live sessions. No transport or UI here. - * See README.md for the experiment and its limits. - */ - -#ifndef PokemonAutomation_SessionAgentPrototype_H -#define PokemonAutomation_SessionAgentPrototype_H - -#include -#include -#include -#include -#include "3rdParty-Core/nlohmann/json.hpp" - -namespace PokemonAutomation{ -namespace SessionAgentPrototype{ - -using Json = nlohmann::json; - -struct Bindings{ - std::function list; - std::function start; - std::function stop; -}; - -inline Json failure(const std::string& code, const std::string& message){ - return {{"ok", false}, {"error", {{"code", code}, {"message", message}}}}; -} - -inline Json status(Bindings& bindings, uint64_t id){ - for (const Json& session : bindings.list()){ - if (session.at("session_id").get() == id){ - return {{"ok", true}, {"result", session}}; - } - } - return failure("session_not_found", "The session is no longer registered."); -} - -// Returns an immediate reply; accepted start/stop is not run completion. -// Bindings own session lookup/lifetime protection. No session pointer escapes. -inline Json dispatch(Bindings& bindings, const Json& request){ - if (!request.is_object() || !request.contains("method") || !request["method"].is_string()){ - return failure("invalid_request", "Expected an object with a string method."); - } - const std::string method = request["method"].get(); - if (method == "sessions.list"){ - return {{"ok", true}, {"result", bindings.list()}}; - } - if (method != "sessions.status" && method != "sessions.start" && method != "sessions.stop"){ - return failure("unknown_method", "Use sessions.list, sessions.status, sessions.start or sessions.stop."); - } - if (!request.contains("session_id") || !request["session_id"].is_number_integer()){ - return failure("invalid_session_id", "Expected a positive integer session_id."); - } - if (!request["session_id"].is_number_unsigned() && request["session_id"].get() < 0){ - return failure("invalid_session_id", "Expected a positive integer session_id."); - } - const uint64_t id = request["session_id"].get(); - // ProgramTracker reserves UINT64_MAX for implicit selection. Never use it. - if (id == 0 || id == std::numeric_limits::max()){ - return failure("invalid_session_id", "Explicit live session_id required; implicit selection is disabled."); - } - if (method == "sessions.status"){ - return status(bindings, id); - } - const std::string error = method == "sessions.start" ? bindings.start(id) : bindings.stop(id); - if (!error.empty()){ - return failure("operation_rejected", error); - } - return {{"ok", true}, {"result", {{"session_id", id}, {"accepted", true}}}}; -} - -// Live app adapter. Input is JSON; exceptions become explicit error replies. -Json handle_live_request(const std::string& text); - -} -} -#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 516647cfe0..054fdff8d1 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -776,6 +776,8 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/Windows/LabChannel_Tests.h Source/CommonFramework/Windows/LabRequest.cpp Source/CommonFramework/Windows/LabRequest.h + Source/CommonFramework/Windows/LabRunFiles.cpp + Source/CommonFramework/Windows/LabRunFiles.h Source/CommonFramework/Windows/MainWindow.cpp Source/CommonFramework/Windows/MainWindow.h Source/CommonFramework/Windows/WindowTracker.cpp @@ -954,8 +956,6 @@ file(GLOB LIBRARY_SOURCES Source/Integrations/IntegrationsAPI.h Source/Integrations/ProgramTracker.cpp Source/Integrations/ProgramTracker.h - Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.cpp - Source/Integrations/SessionAgentPrototype/SessionAgentPrototype.h Source/GameConsole/ConsoleHandle.cpp Source/GameConsole/ConsoleHandle.h Source/GameConsole/ConsolePanel.cpp diff --git a/devtools/prototypes/session-agent/run-demo.sh b/devtools/prototypes/session-agent/run-demo.sh deleted file mode 100644 index cde8eb73f7..0000000000 --- a/devtools/prototypes/session-agent/run-demo.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -# Standalone native C++ demo; does not build or launch SerialPrograms. -set -eu -repo=$(CDPATH= cd -- "$(dirname -- "$0")/../../.." && pwd) -build_dir=$(mktemp -d "${TMPDIR:-/tmp}/pa-session-agent.XXXXXX") -"${CXX:-c++}" -std=c++23 -Wall -Wextra -Werror -I "$repo" \ - "$repo/SerialPrograms/Source/Integrations/SessionAgentPrototype/PrototypeDemo.cpp" \ - -o "$build_dir/demo" -exec "$build_dir/demo" "$@" From 68eafa970bee8e9b1af2bdeda957fc52812bc682 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 00:45:05 -0700 Subject: [PATCH 08/16] Lab scripts and docs: labctl.sh wait, and the run records in Lab.md and the README "labctl.sh wait RUN_ID" polls Lab/runs/RUN_ID/result.json until it is terminal (LABCTL_RUN_WAIT, default two hours) and prints the outcome, the last event's checkpoint, step and attempt, the error, and the frame, events and log paths; exit 0 for completed, 1 for failed or cancelled, 2 on a timeout. "start" now echoes the run= line of its journal. labctl_wait_test.sh checks it against hand-written result files. Lab.md documents the run folder, result.json and events.jsonl, the run= state line and start's reply; the README gets a "Run records" section with the console-iteration loop that replaces run_watch.sh. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Source/CommonFramework/Windows/Lab.md | 109 ++++++++++++++++-- devtools/lab/README.md | 49 +++++++- devtools/lab/labctl.sh | 47 +++++++- devtools/lab/labctl_wait_test.sh | 72 ++++++++++++ 4 files changed, 262 insertions(+), 15 deletions(-) create mode 100755 devtools/lab/labctl_wait_test.sh diff --git a/SerialPrograms/Source/CommonFramework/Windows/Lab.md b/SerialPrograms/Source/CommonFramework/Windows/Lab.md index 86172c1fde..ea2609aa9e 100644 --- a/SerialPrograms/Source/CommonFramework/Windows/Lab.md +++ b/SerialPrograms/Source/CommonFramework/Windows/Lab.md @@ -28,6 +28,7 @@ takes effect at the next launch. | `Lab/outbox/.out` | app writes | The request's journal (below). It appears only once the request has finished; `.tmp` shows the progress so far. | | `Lab/outbox/_