From b29d8c5ac585f7b13d1204ccff7a92a5cad7b77e Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 13:01:28 -0800 Subject: [PATCH 1/4] Run records: a typed run model, a bounded store and the session boundary The framework half of fork PR #116 (5ffec9d1b), unchanged: RunRecord.h with RunRecordStore and SessionRunRecorder, ProgramSession recording every accepted run with exactly one result, ProgramEnvironment:: run_recorder(), the console sessions' loggers chained through the run logger, and the RunRecord unit tests. Nothing here depends on the lab channel. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Source/CommonFramework/ProgramSession.cpp | 101 +++- .../Source/CommonFramework/ProgramSession.h | 38 +- .../Source/CommonFramework/RunRecord.cpp | 227 +++++++ .../Source/CommonFramework/RunRecord.h | 178 ++++++ .../CommonFramework/RunRecord_Tests.cpp | 557 ++++++++++++++++++ .../Source/CommonFramework/RunRecord_Tests.h | 20 + .../Tools/ProgramEnvironment.cpp | 4 + .../Tools/ProgramEnvironment.h | 7 + .../ComputerPrograms_UnitTestRunner.cpp | 2 + .../Framework/ConsolePanelSession.cpp | 3 +- .../Framework/ConsoleProgramSession.cpp | 1 + .../Framework/ConsoleSystemSession.cpp | 3 +- .../Framework/ConsoleSystemSession.h | 4 + .../Framework/MultiConsolePanelSession.cpp | 3 +- .../Framework/MultiConsoleProgramSession.cpp | 2 +- .../Framework/MultiConsoleSystemSession.cpp | 4 + .../Framework/MultiConsoleSystemSession.h | 4 + .../Source/Integrations/ProgramTracker.cpp | 6 +- .../Integrations/ProgramTrackerInterfaces.h | 4 +- .../NintendoSwitch_UpdatePopupDetector.cpp | 3 +- .../Sounds/PokemonLA_ShinySoundDetector.cpp | 3 +- SerialPrograms/cmake/SourceFiles.cmake | 4 + 22 files changed, 1143 insertions(+), 35 deletions(-) create mode 100644 SerialPrograms/Source/CommonFramework/RunRecord.cpp create mode 100644 SerialPrograms/Source/CommonFramework/RunRecord.h create mode 100644 SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp create mode 100644 SerialPrograms/Source/CommonFramework/RunRecord_Tests.h diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp index f4176b9199..d091dc4b80 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp @@ -4,9 +4,11 @@ * */ +#include #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PanicDump.h" +#include "Common/Cpp/ScopeExit.h" #include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/Logging/Logger.h" @@ -64,11 +66,12 @@ void ProgramSession::remove_listener(Listener& listener){ -ProgramSession::ProgramSession(const ProgramDescriptor& descriptor) +ProgramSession::ProgramSession(const ProgramDescriptor& descriptor, RunRecordStore& run_records) : m_descriptor(descriptor) , m_instance_id(ProgramTracker::instance().add_program(*this)) + , m_run_recorder(run_records, global_logger_raw()) // , m_logger(global_logger_raw(), "Program:" + std::to_string(m_instance_id)) - , m_logger(global_logger_raw(), "Program") + , m_logger(m_run_recorder, "Program") , m_last_state_change(current_time()) , m_state(ProgramState::STOPPED) , m_scope(nullptr) @@ -186,6 +189,9 @@ void ProgramSession::push_stats(){ } void ProgramSession::push_error(const std::string& message){ m_listeners.run_method(&Listener::error, message); + // A diagnostic in the run's record, not its outcome: the stats-save + // error and the download warnings come through here too. + m_run_recorder.event({.kind = "error", .text = message}); } void ProgramSession::push_download_error(const std::string& message){ @@ -247,26 +253,24 @@ std::string ProgramSession::start_program(){ // Wait for previous program thread to finish. join_program_thread(); - // Now start the program. + // Now start the program. The run is accepted here: its record + // exists before the program thread can report anything. m_logger.log("Starting program..."); m_last_state_change.store(current_time(), std::memory_order_relaxed); + m_run_recorder.begin(identifier()); set_state(ProgramState::RUNNING); - m_program_thread = GlobalThreadPools::unlimited_realtime().dispatch_now_blocking( - [this]{ + try{ + m_program_thread = dispatch_program_thread([this]{ run_with_catch( "ProgramSession::start_program()", - [this]{ run_program(); } + [this]{ run_program_and_record(); } ); - } - ); -#if 0 - m_thread = Thread([this]{ - run_with_catch( - "ProgramSession::start_program()", - [this]{ run_program(); } - ); - }); -#endif + }); + }catch (Exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.message())); + }catch (std::exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.what())); + } return ""; } case ProgramState::RUNNING: @@ -300,6 +304,7 @@ std::string ProgramSession::stop_program(){ case ProgramState::RUNNING: m_logger.log("Stopping program..."); set_state(ProgramState::STOPPING); + m_run_recorder.stopping(); break; case ProgramState::STOPPING: m_logger.log("Program is already stopping.", COLOR_RED); @@ -319,13 +324,51 @@ std::string ProgramSession::stop_program(){ -void ProgramSession::run_program(){ +// No thread, so no run: closes the record and leaves the session stopped, +// so the next start can try again. Returns `message` for start_program(). +std::string ProgramSession::abandon_run(std::string message){ + m_logger.log(message, COLOR_RED); + m_run_recorder.finish({.outcome = RunOutcome::FAILED, .error = message}); + set_state(ProgramState::STOPPED); + return message; +} + +AsyncTask ProgramSession::dispatch_program_thread(std::function body){ + return GlobalThreadPools::unlimited_realtime().dispatch_now_blocking(std::move(body)); +} + +void ProgramSession::run_program_and_record(){ + // Exactly one result per run: whichever way the thread leaves, the + // record is closed, and a path that set none is a failure, never a + // success. An exception that escapes run_program() (as today it does + // for a validity check that fails after the start) is its result. + std::optional result; + ScopeExit finish_run([&]{ + m_run_recorder.finish(result.value_or(RunResult{ + .outcome = RunOutcome::FAILED, + .error = "The program thread ended without a result.", + })); + }); + try{ + result = run_program(); + }catch (Exception& e){ + result = RunResult{.outcome = RunOutcome::FAILED, .error = e.message().empty() ? e.name() : e.message()}; + throw; + }catch (std::exception& e){ + result = RunResult{.outcome = RunOutcome::FAILED, .error = e.what()}; + throw; + } +} + +RunResult ProgramSession::run_program(){ SleepSuppressScope sleep_scope(GlobalSettings::instance().SLEEP_SUPPRESS->PROGRAM_RUNNING); + RunResult result; { std::lock_guard lg(m_lock); if (current_state() != ProgramState::RUNNING){ - return; + // Stopped before the thread got going. + return {.outcome = RunOutcome::CANCELLED}; } std::string error = check_validity(); @@ -350,7 +393,12 @@ void ProgramSession::run_program(){ { RunningProgramScope running_scope(*this); if (!download_prereqs(*m_scope)){ - return; + // The details were reported as "error" events. A stop during + // the download is a cancellation, not a failure. + if (current_state() == ProgramState::STOPPING){ + return {.outcome = RunOutcome::CANCELLED}; + } + return {.outcome = RunOutcome::FAILED, .error = "A required resource could not be downloaded."}; } std::unique_ptr env = make_env(program_info); try{ @@ -360,10 +408,13 @@ void ProgramSession::run_program(){ internal_run_program(*env); env->log_to_ui("- Program Finished -"); logger().log("Program finished normally!", COLOR_BLUE); + result.outcome = RunOutcome::COMPLETED; }catch (OperationCancelledException&){ env->log_to_ui("- Program Stopped -"); + result.outcome = RunOutcome::CANCELLED; }catch (ProgramCancelledException&){ env->log_to_ui("- Program Stopped -"); + result.outcome = RunOutcome::CANCELLED; }catch (ProgramFinishedException& e){ logger().log("Program finished early!", COLOR_BLUE); env->log_to_ui("- Program Finished -"); @@ -372,6 +423,8 @@ void ProgramSession::run_program(){ e.message(), e.screenshot_view() ); + result.outcome = RunOutcome::COMPLETED; + result.frame = e.screenshot(); }catch (InvalidConnectionStateException& e){ logger().log("Program stopped due to connection issue.", COLOR_RED); env->log_to_ui("- Invalid Connection -", COLOR_RED); @@ -380,6 +433,7 @@ void ProgramSession::run_program(){ message = e.name(); } report_error(message); + result.error = message; }catch (FatalProgramException& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -390,6 +444,8 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; + result.frame = e.screenshot(); }catch (OperationFailedExceptionWithScreenshot& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -400,6 +456,8 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; + result.frame = e.screenshot(); }catch (OperationFailedException& e){ // no screenshot logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -410,6 +468,7 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; }catch (Exception& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -422,6 +481,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, message ); + result.error = message; }catch (std::exception& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -434,6 +494,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, message ); + result.error = message; }catch (...){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Unknown Error -", COLOR_RED); @@ -442,6 +503,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, "Unknown error." ); + result.error = "Unknown error."; } } @@ -451,6 +513,7 @@ void ProgramSession::run_program(){ update_historical_stats_with_current(); set_state(ProgramState::STOPPED); } + return result; } diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.h b/SerialPrograms/Source/CommonFramework/ProgramSession.h index af12d8c820..36889c5c30 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.h +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.h @@ -17,6 +17,7 @@ #define PokemonAutomation_CommonFramework_ProgramSession_H #include +#include #include #include #include "Common/Cpp/Logging/TaggedLogger.h" @@ -29,6 +30,7 @@ #include "CommonFramework/Globals.h" //#include "CommonFramework/Logging/Logger.h" #include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/RunRecord.h" #include "CommonFramework/Tools/ProgramEnvironment.h" // #include "CommonFramework/ResourceDownload/ProgramMissingResourceTracker.h" #include "Integrations/ProgramTrackerInterfaces.h" @@ -66,7 +68,12 @@ class ProgramSession : public TrackableProgram{ public: // Setup - ProgramSession(const ProgramDescriptor& descriptor); + // `run_records` is where this session's runs are recorded (RunRecord.h); + // a test passes its own store. + ProgramSession( + const ProgramDescriptor& descriptor, + RunRecordStore& run_records = RunRecordStore::instance() + ); // Before the child-most class begins destruction, you must first: // 1. Stop the program. @@ -104,6 +111,21 @@ class ProgramSession : public TrackableProgram{ Mutex& program_lock() const{ return m_lock; } +public: + // Run records (RunRecord.h) + + // The run the last accepted start opened, active or finished; "" before + // the first start. + std::string last_run_id() const{ return m_run_recorder.last_run_id(); } + + // The program's progress hook. Events go to the active run's record. + RunRecorder& run_recorder(){ return m_run_recorder; } + + // The logger the session's consoles chain off, so that their lines land + // in the active run's log tail. + Logger& run_logger(){ return m_run_recorder; } + + public: // Program Control @@ -114,8 +136,8 @@ class ProgramSession : public TrackableProgram{ std::string stop_program(); // Returns before the program is completely stopped. // Temporary - virtual void async_start() override{ start_program(); } - virtual void async_stop() override{ stop_program(); } + virtual std::string async_start() override{ return start_program(); } + virtual std::string async_stop() override{ return stop_program(); } protected: @@ -123,6 +145,9 @@ class ProgramSession : public TrackableProgram{ virtual void internal_run_program(ProgramEnvironment& env) = 0; virtual void internal_stop_program(); + // Starts the program thread. A test fake makes this fail. + virtual AsyncTask dispatch_program_thread(std::function body); + // virtual void restore_defaults(){ return; } public: @@ -153,13 +178,18 @@ class ProgramSession : public TrackableProgram{ private: - void run_program(); + // The program thread: runs the program and closes its run record with + // exactly one result, whichever way run_program() ends. + void run_program_and_record(); + RunResult run_program(); + std::string abandon_run(std::string message); private: const ProgramDescriptor& m_descriptor; uint64_t m_instance_id = 0; + SessionRunRecorder m_run_recorder; // Before m_logger, which chains off it. TaggedLogger m_logger; diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.cpp b/SerialPrograms/Source/CommonFramework/RunRecord.cpp new file mode 100644 index 0000000000..17cffdca16 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord.cpp @@ -0,0 +1,227 @@ +/* Run Record + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/PrettyPrint.h" +#include "RunRecord.h" + +namespace PokemonAutomation{ + + + +std::string run_status_name(RunStatus status){ + switch (status){ + case RunStatus::RUNNING: + return "running"; + case RunStatus::STOPPING: + return "stopping"; + case RunStatus::TERMINAL: + return "terminal"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown run status."); +} +std::string run_outcome_name(RunOutcome outcome){ + switch (outcome){ + case RunOutcome::COMPLETED: + return "completed"; + case RunOutcome::CANCELLED: + return "cancelled"; + case RunOutcome::FAILED: + return "failed"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown run outcome."); +} + + + +RunRecordStore& RunRecordStore::instance(){ + static RunRecordStore store; + return store; +} + +RunRecordStore::RunRecordStore(size_t capacity) + // The app's start time, to the microsecond: two instances started in + // the same second still get different IDs. + : m_app_token(now_to_filestring()) + , m_capacity(std::max(capacity, 1)) +{} + +std::string RunRecordStore::begin(std::string program){ + std::lock_guard lg(m_lock); + while (m_records.size() >= m_capacity){ + m_records.pop_front(); + } + RunRecord& record = m_records.emplace_back(); + record.run_id = std::format("{}-{:03}", m_app_token, ++m_counter); + record.program = std::move(program); + record.start_time = current_time(); + return record.run_id; +} + +void RunRecordStore::stopping(const std::string& run_id){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record != nullptr && record->status == RunStatus::RUNNING){ + record->status = RunStatus::STOPPING; + } +} + +void RunRecordStore::event(const std::string& run_id, RunEvent event){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr){ + return; + } + if (record->events.size() >= MAX_EVENTS){ + record->dropped_events++; + return; + } + if (event.time == WallClock::min()){ + event.time = current_time(); + } + record->events.emplace_back(std::move(event)); + + // Keep the frames of the newest events only. + size_t frames = 0; + for (RunEvent& e : std::ranges::reverse_view(record->events)){ + if (e.frame == nullptr){ + continue; + } + if (++frames > MAX_EVENT_FRAMES){ + e.frame.reset(); + } + } +} + +void RunRecordStore::log_line(const std::string& run_id, const std::string& text){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr){ + return; + } + record->log_tail.emplace_back(text); + size_t bytes = 0; + for (const std::string& line : record->log_tail){ + bytes += line.size() + 1; + } + size_t drop = 0; + while (bytes > MAX_LOG_BYTES && drop + 1 < record->log_tail.size()){ + bytes -= record->log_tail[drop].size() + 1; + drop++; + } + if (drop > 0){ + record->log_tail.erase(record->log_tail.begin(), record->log_tail.begin() + drop); + record->log_truncated = true; + } +} + +bool RunRecordStore::finish(const std::string& run_id, RunResult result){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr || record->status == RunStatus::TERMINAL){ + return false; + } + record->status = RunStatus::TERMINAL; + record->end_time = current_time(); + record->result = std::move(result); + + // Frames stay with the newest terminal runs; older ones keep the rest + // of their record. + size_t terminal = 0; + for (RunRecord& r : std::ranges::reverse_view(m_records)){ + if (r.status == RunStatus::TERMINAL && ++terminal > RUNS_WITH_FRAMES){ + release_frames(r); + } + } + return true; +} + +std::vector RunRecordStore::run_ids() const{ + std::lock_guard lg(m_lock); + std::vector ids; + for (const RunRecord& record : m_records){ + ids.emplace_back(record.run_id); + } + return ids; +} + +std::optional RunRecordStore::get(const std::string& run_id) const{ + std::lock_guard lg(m_lock); + for (const RunRecord& record : m_records){ + if (record.run_id == run_id){ + return record; + } + } + return std::nullopt; +} + +RunRecord* RunRecordStore::find(const std::string& run_id){ + auto it = std::ranges::find_if(m_records, [&](const RunRecord& r){ return r.run_id == run_id; }); + return it == m_records.end() ? nullptr : &*it; +} + +void RunRecordStore::release_frames(RunRecord& record){ + for (RunEvent& event : record.events){ + event.frame.reset(); + } + if (record.result){ + record.result->frame.reset(); + } +} + + + +SessionRunRecorder::SessionRunRecorder(RunRecordStore& store, Logger& base_logger) + : m_store(store) + , m_base_logger(base_logger) +{} + +std::string SessionRunRecorder::begin(std::string program){ + std::lock_guard lg(m_lock); + m_run_id = m_store.begin(std::move(program)); + m_active = true; + return m_run_id; +} +void SessionRunRecorder::stopping(){ + std::lock_guard lg(m_lock); + if (m_active){ + m_store.stopping(m_run_id); + } +} +void SessionRunRecorder::finish(RunResult result){ + std::lock_guard lg(m_lock); + if (!m_active){ + return; + } + m_active = false; + m_store.finish(m_run_id, std::move(result)); +} + +std::string SessionRunRecorder::last_run_id() const{ + std::lock_guard lg(m_lock); + return m_run_id; +} + +void SessionRunRecorder::event(RunEvent event){ + std::lock_guard lg(m_lock); + if (m_active){ + m_store.event(m_run_id, std::move(event)); + } +} +void SessionRunRecorder::log(const std::string& msg, Color color){ + m_base_logger.log(msg, color); + std::lock_guard lg(m_lock); + if (m_active){ + m_store.log_line(m_run_id, msg); + } +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.h b/SerialPrograms/Source/CommonFramework/RunRecord.h new file mode 100644 index 0000000000..f9afd2c86d --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord.h @@ -0,0 +1,178 @@ +/* Run Record + * + * From: https://github.com/PokemonAutomation/ + * + * What one run of a program did, as the framework saw it: when it started, + * how it ended (exactly one outcome per accepted run), the error and the + * frame it died with, the events the program reported on the way, and the + * tail of its log. ProgramSession writes these at its execution boundary; + * RunRecordStore keeps the last few for whoever asks, such as the Lab + * channel, which writes them to files (Windows/Lab.md, "Run records"). + * Nothing here depends on the UI, on JSON or on a game. + * + */ + +#ifndef PokemonAutomation_CommonFramework_RunRecord_H +#define PokemonAutomation_CommonFramework_RunRecord_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" + +namespace PokemonAutomation{ + + +enum class RunStatus{ + RUNNING, + STOPPING, // A stop was requested; the outcome is not known yet. + TERMINAL, +}; +enum class RunOutcome{ + COMPLETED, // The program returned or threw ProgramFinishedException. + CANCELLED, // The program ended through the stop path. + FAILED, // A setup, connection or execution error ended it. +}; +std::string run_status_name(RunStatus status); // "running", "stopping", "terminal" +std::string run_outcome_name(RunOutcome outcome); // "completed", "cancelled", "failed" + + +// Something the program says about its own progress: a checkpoint reached, +// a step started, an attempt that failed, a pose. Every field but `kind` +// and `text` is optional; a game fills the ones it has. (The `= {}` let a +// designated initializer leave fields out on every compiler.) +struct RunEvent{ + WallClock time = WallClock::min(); // Set by the recorder when left at min(). + std::string kind = {}; // e.g. "checkpoint", "step", "attempt", "error" + std::optional checkpoint = {}; + std::optional step = {}; + std::optional attempt = {}; + std::string text = {}; + std::shared_ptr frame = {}; // The screen at that moment, if the program had it. + std::optional pose = {}; // Where the player is, as the program puts it. +}; + +// How a run ended. +struct RunResult{ + RunOutcome outcome = RunOutcome::FAILED; + std::optional error = {}; // The primary error of a failed run. + std::shared_ptr frame = {}; // The frame the exception carried, if any. +}; + +struct RunRecord{ + std::string run_id = {}; + std::string program = {}; // The program's identifier, ":". + WallClock start_time = WallClock::min(); + std::optional end_time = {}; + RunStatus status = RunStatus::RUNNING; + std::optional result = {}; // Set once, when the status turns TERMINAL. + std::vector events = {}; + size_t dropped_events = 0; // Events past RunRecordStore::MAX_EVENTS. + std::vector log_tail = {}; // The run's last log lines, oldest first. + bool log_truncated = false; // Whether older lines were dropped from the tail. +}; + + +// The program's progress hook, reachable as ProgramEnvironment::run_recorder(). +class RunRecorder{ +public: + virtual ~RunRecorder() = default; + virtual void event(RunEvent event) = 0; +}; + + +// The last few runs of this app instance. Thread-safe; every call is short +// and copies, so a reader never holds a lock while writing files. +// +// Run IDs are opaque strings unique to this app instance: a restart cannot +// resolve an old ID to a new run. They sort in the order the runs started. +// +// Memory: at most MAX_EVENTS events per run; the frames of a run's newest +// MAX_EVENT_FRAMES events; all frames only for the active run and the +// newest RUNS_WITH_FRAMES terminal runs; a log tail of MAX_LOG_BYTES. +// A consumer that wants a frame reads it soon after the event or the end. +class RunRecordStore{ +public: + static constexpr size_t DEFAULT_CAPACITY = 32; + static constexpr size_t MAX_EVENTS = 4096; + static constexpr size_t MAX_EVENT_FRAMES = 8; + static constexpr size_t RUNS_WITH_FRAMES = 4; + static constexpr size_t MAX_LOG_BYTES = 64 * 1024; + + static RunRecordStore& instance(); + + explicit RunRecordStore(size_t capacity = DEFAULT_CAPACITY); + RunRecordStore(const RunRecordStore&) = delete; + void operator=(const RunRecordStore&) = delete; + + // Opens a record for an accepted start and returns its run ID. The + // oldest record goes when the store is full. + std::string begin(std::string program); + + // Marks a running run as stopping. No effect once it is terminal. + void stopping(const std::string& run_id); + + // Appends to the run. Both are dropped for an unknown run. + void event(const std::string& run_id, RunEvent event); + void log_line(const std::string& run_id, const std::string& text); + + // Closes the run with its one result. Returns false, and changes + // nothing, when the run is unknown or already terminal. + bool finish(const std::string& run_id, RunResult result); + + // The IDs of the records held, oldest first. + std::vector run_ids() const; + + // A copy of the record, or nothing for an unknown run. + std::optional get(const std::string& run_id) const; + + +private: + RunRecord* find(const std::string& run_id); + void release_frames(RunRecord& record); + + mutable Mutex m_lock; + const std::string m_app_token; + const size_t m_capacity; + uint64_t m_counter = 0; + std::deque m_records; +}; + + +// One session's end of the store: a RunRecorder for the run active now and +// a Logger that copies what passes through it into that run's log tail. +// The session's loggers chain off `log()`, so a run's tail holds the lines +// the program and its consoles logged, and nothing from other sessions. +class SessionRunRecorder : public RunRecorder, public Logger{ +public: + SessionRunRecorder(RunRecordStore& store, Logger& base_logger); + + // Opens a run and makes it the active one. Returns its ID. + std::string begin(std::string program); + void stopping(); + // Closes the active run; nothing happens when none is active. + void finish(RunResult result); + + // The run the last begin() opened, active or finished; "" before the first. + std::string last_run_id() const; + + virtual void event(RunEvent event) override; + virtual void log(const std::string& msg, Color color = Color()) override; + +private: + RunRecordStore& m_store; + Logger& m_base_logger; + mutable Mutex m_lock; + std::string m_run_id; + bool m_active = false; +}; + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp new file mode 100644 index 0000000000..aedd954fa0 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp @@ -0,0 +1,557 @@ +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The store on its own, and the run lifecycle through a ProgramSession + * whose program is a scripted fake: no console, no window. + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Notifications/ProgramInfo.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/ProgramSession.h" +#include "CommonFramework/Tools/ProgramEnvironment.h" +#include "RunRecord.h" +#include "RunRecord_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + +using namespace std::chrono_literals; + + +std::shared_ptr make_frame(size_t width){ + ImageRGB32 image(width, width); + image.fill(0xff000000); + return std::make_shared(std::move(image)); +} + +// Polls `done` every few milliseconds until it holds or `timeout` passes. +bool wait_until(const std::function& done, Milliseconds timeout = 5000ms){ + const WallClock deadline = current_time() + timeout; + while (!done()){ + if (current_time() > deadline){ + return false; + } + std::this_thread::sleep_for(5ms); + } + return true; +} + + + +UnitTestResult test_store_ids(){ + RunRecordStore store(2); + const std::string a = store.begin("Test:A"); + const std::string b = store.begin("Test:B"); + if (a.empty() || a == b || !(a < b)){ + return std::format("Run IDs should be distinct and sort in start order: <{}>, <{}>", a, b); + } + if (store.run_ids() != std::vector{a, b}){ + return "run_ids() lists the records oldest first."; + } + const std::optional record = store.get(a); + if (!record || record->program != "Test:A" || record->status != RunStatus::RUNNING || record->result){ + return "A new record is running, with its program and no result."; + } + if (record->start_time == WallClock::min() || record->end_time){ + return "A new record has a start time and no end time."; + } + const std::string c = store.begin("Test:C"); + if (store.get(a) || store.run_ids() != std::vector{b, c}){ + return "A full store drops its oldest record."; + } + if (store.get("nobody")){ + return "An unknown run has no record."; + } + return true; +} + +UnitTestResult test_store_one_result(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.stopping(id); + if (store.get(id)->status != RunStatus::STOPPING){ + return "stopping() marks a running run."; + } + if (!store.finish(id, {.outcome = RunOutcome::FAILED, .error = "first"})){ + return "The first finish() takes."; + } + store.stopping(id); + if (store.finish(id, {.outcome = RunOutcome::COMPLETED})){ + return "A second finish() is refused."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || !record.end_time || !record.result){ + return "A finished record is terminal with an end time and a result."; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "first"){ + return std::format("The first result stands: got {}", run_outcome_name(record.result->outcome)); + } + if (store.finish("nobody", {})){ + return "Finishing an unknown run is refused."; + } + return true; +} + +UnitTestResult test_store_events_and_log(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.event(id, {.kind = "checkpoint", .checkpoint = "00.01", .text = "ok"}); + store.event("nobody", {.kind = "checkpoint"}); + for (size_t c = 0; c < RunRecordStore::MAX_EVENT_FRAMES + 2; c++){ + store.event(id, {.kind = "step", .step = std::to_string(c), .frame = make_frame(4)}); + } + RunRecord record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENT_FRAMES + 3){ + return std::format("Expected {} events, got {}.", RunRecordStore::MAX_EVENT_FRAMES + 3, record.events.size()); + } + if (record.events[0].time == WallClock::min() || record.events[0].checkpoint != "00.01"){ + return "An event gets the time it was recorded and keeps its fields."; + } + if (record.events[1].frame || record.events[2].frame || !record.events[3].frame || !record.events.back().frame){ + return "Only the newest MAX_EVENT_FRAMES events keep their frames."; + } + + for (size_t c = record.events.size(); c < RunRecordStore::MAX_EVENTS + 5; c++){ + store.event(id, {.kind = "step"}); + } + record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENTS || record.dropped_events != 5){ + return std::format("Events past the cap are counted, not kept: {} events, {} dropped.", record.events.size(), record.dropped_events); + } + + const std::string line(1000, 'x'); + for (size_t c = 0; c < 100; c++){ + store.log_line(id, std::format("{} {}", c, line)); + } + record = *store.get(id); + size_t bytes = 0; + for (const std::string& l : record.log_tail){ + bytes += l.size() + 1; + } + if (!record.log_truncated || bytes > RunRecordStore::MAX_LOG_BYTES || record.log_tail.back().rfind("99 ", 0) != 0){ + return std::format("The log tail keeps the newest {} bytes: {} lines, {} bytes.", RunRecordStore::MAX_LOG_BYTES, record.log_tail.size(), bytes); + } + return true; +} + +UnitTestResult test_store_releases_old_frames(){ + RunRecordStore store; + std::vector ids; + for (size_t c = 0; c < RunRecordStore::RUNS_WITH_FRAMES + 1; c++){ + ids.emplace_back(store.begin("Test:A")); + store.event(ids.back(), {.kind = "step", .frame = make_frame(4)}); + store.finish(ids.back(), {.outcome = RunOutcome::FAILED, .error = "e", .frame = make_frame(4)}); + } + const RunRecord oldest = *store.get(ids.front()); + if (oldest.events[0].frame || oldest.result->frame){ + return "A run older than the newest RUNS_WITH_FRAMES terminal runs keeps no frames."; + } + if (oldest.result->error != "e" || oldest.events.size() != 1){ + return "It keeps the rest of its record."; + } + const RunRecord newest = *store.get(ids.back()); + if (!newest.events[0].frame || !newest.result->frame){ + return "The newest terminal runs keep their frames."; + } + return true; +} + +UnitTestResult test_session_recorder(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + RunRecordStore store; + CountingLogger base; + SessionRunRecorder recorder(store, base); + recorder.log("before"); + recorder.event({.kind = "lost"}); + if (!recorder.last_run_id().empty() || !store.run_ids().empty()){ + return "Nothing is recorded before a run begins."; + } + const std::string id = recorder.begin("Test:A"); + if (recorder.last_run_id() != id){ + return "last_run_id() is the run begin() opened."; + } + recorder.log("during"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + recorder.log("after"); + recorder.event({.kind = "late"}); + recorder.finish({.outcome = RunOutcome::FAILED}); + const RunRecord record = *store.get(id); + if (base.lines != 3){ + return std::format("Every line reaches the base logger: {} of 3.", base.lines); + } + if (record.log_tail != std::vector{"during"} || record.events.size() != 1){ + return "Only lines and events during the run are recorded."; + } + if (record.result->outcome != RunOutcome::COMPLETED || recorder.last_run_id() != id){ + return "The result stands and the run stays known after finish()."; + } + return true; +} + + + +// A program with no console: what it does is scripted per test. +struct FakeScript{ + std::string validity = {}; // check_validity(): "" accepts the start. + bool fail_dispatch = false; // The program thread cannot be started. + std::function start_checks = {}; + std::function body = {}; +}; + +class FakeDescriptor : public ProgramDescriptor{ +public: + FakeDescriptor() + : ProgramDescriptor( + "Test:RunRecordFake", "Test", "Run Record Fake", + "", "", PanelQualityStanding::DEVELOPER, false, {} + ) + {} + virtual std::unique_ptr make_panel() const override{ return nullptr; } +}; + +class FakeInstance : public ProgramInstance{ +public: + explicit FakeInstance(std::function start_checks) + : m_start_checks(std::move(start_checks)) + {} + virtual void run_start_program_checks(const ProgramDescriptor&, ProgramEnvironment& env) override{ + if (m_start_checks){ + m_start_checks(env); + } + } +private: + std::function m_start_checks; +}; + +class FakeProgram : public ProgramSession{ +public: + FakeProgram(const FakeDescriptor& descriptor, RunRecordStore& store, FakeScript script) + : ProgramSession(descriptor, store) + , m_script(std::move(script)) + { + m_instance = std::make_unique(m_script.start_checks); + } + virtual ~FakeProgram(){ + internal_stop_program(); + join_program_thread(); + } + + // Whether the body has started running. + bool running() const{ return m_running.load(std::memory_order_acquire); } + + // Waits for the program thread to end, so the run's record is closed. + bool wait_until_stopped(){ + if (!wait_until([this]{ return current_state() == ProgramState::STOPPED; })){ + return false; + } + join_program_thread(); + return true; + } + + virtual std::string check_validity() const override{ return m_script.validity; } + +protected: + virtual std::unique_ptr make_env(const ProgramInfo& program_info) override{ + return std::make_unique(program_info, *this, current_stats_tracker(), historical_stats_tracker()); + } + virtual void internal_run_program(ProgramEnvironment& env) override{ + m_running.store(true, std::memory_order_release); + if (m_script.body){ + m_script.body(*this, env, *m_scope); + } + } + virtual AsyncTask dispatch_program_thread(std::function body) override{ + if (m_script.fail_dispatch){ + throw std::runtime_error("no thread"); + } + return ProgramSession::dispatch_program_thread(std::move(body)); + } + +private: + FakeScript m_script; + std::atomic m_running{false}; +}; + +// Starts the program, waits for it to end, and returns its record. +UnitTestResult run_to_end(FakeProgram& program, RunRecordStore& store, RunRecord& record){ + const std::string error = program.start_program(); + if (!error.empty()){ + return "The start was refused: " + error; + } + if (!program.wait_until_stopped()){ + return "The program did not stop."; + } + const std::optional found = store.get(program.last_run_id()); + if (!found){ + return "The run has no record."; + } + record = *found; + if (record.status != RunStatus::TERMINAL || !record.result){ + return "The record is not terminal after the program stopped."; + } + return true; +} + +UnitTestResult test_rejection(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.validity = "Pick a save."}); + if (program.start_program() != "Pick a save."){ + return "A start refused by check_validity() returns its message."; + } + if (!store.run_ids().empty() || !program.last_run_id().empty()){ + return "A refused start opens no run."; + } + return true; +} + +UnitTestResult test_immediate_completion(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment& env, CancellableScope&){ + env.run_recorder().event({.kind = "checkpoint", .checkpoint = "00.01", .step = "3/3", .attempt = 1, .text = "ok", .pose = "home (2, 4) N"}); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::COMPLETED || record.result->error || record.result->frame){ + return "A program that returns completed its run, with no error and no frame."; + } + if (record.program != "Test:RunRecordFake" || record.start_time > *record.end_time){ + return "The record names the program and its times."; + } + if (record.events.size() != 1 || record.events[0].checkpoint != "00.01" || record.events[0].attempt != size_t(1) || record.events[0].pose != "home (2, 4) N"){ + return std::format("The program's event is in the record: {} events.", record.events.size()); + } + bool stopped_logged = false; + for (const std::string& line : record.log_tail){ + stopped_logged |= line.find("Program State: STOPPED") != std::string::npos; + } + if (!stopped_logged){ + return "The run's log tail runs through the session's last state change."; + } + return true; +} + +UnitTestResult test_setup_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .start_checks = [](ProgramEnvironment& env){ + throw UserSetupError(env.logger(), "The controller is not ready."); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The controller is not ready."){ + return "A setup error fails the run with its message."; + } + if (record.events.size() != 1 || record.events[0].kind != "error" || record.events[0].text != "The controller is not ready."){ + return "The reported error is an \"error\" event."; + } + return true; +} + +UnitTestResult test_failure_with_frame(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + ImageRGB32 screen(8, 8); + screen.fill(0xffff0000); + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "The sign's box did not open.", nullptr, std::move(screen) + ); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The sign's box did not open."){ + return "The exception's message is the primary error."; + } + if (!record.result->frame || record.result->frame->width() != 8){ + return "The exception's frame is kept with the result."; + } + return true; +} + +UnitTestResult test_cancellation(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope& scope){ + while (true){ + scope.throw_if_cancelled(); + std::this_thread::sleep_for(5ms); + } + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = program.last_run_id(); + if (!program.stop_program().empty() || !program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || record.result->outcome != RunOutcome::CANCELLED || record.result->error){ + return "A stopped program's run is cancelled, with no error."; + } + return true; +} + +// The body waits for the stop request and then ends its own way. +UnitTestResult run_racing_a_stop(const std::function& ending, RunOutcome expected, const char* what){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [&](ProgramSession& self, ProgramEnvironment&, CancellableScope&){ + wait_until([&]{ return self.current_state() == ProgramState::STOPPING; }); + ending(); + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = program.last_run_id(); + program.stop_program(); + if (!program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.result->outcome != expected){ + return std::format("{}: expected {}, got {}.", what, run_outcome_name(expected), run_outcome_name(record.result->outcome)); + } + return true; +} +UnitTestResult test_late_stop(){ + UnitTestResult result = run_racing_a_stop( + []{ throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "Failed anyway."); }, + RunOutcome::FAILED, "A failure observed after the stop request stays a failure" + ); + if (result.result != UnitTestResult::PASSED){ + return result; + } + return run_racing_a_stop( + []{}, RunOutcome::COMPLETED, "A normal return racing the stop request stays completed" + ); +} + +UnitTestResult test_dispatch_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.fail_dispatch = true}); + const std::string error = program.start_program(); + if (error.find("could not be started") == std::string::npos){ + return "A start whose thread cannot be dispatched returns an error: <" + error + ">"; + } + const std::optional record = store.get(program.last_run_id()); + if (!record || record->status != RunStatus::TERMINAL || record->result->outcome != RunOutcome::FAILED){ + return "The accepted run is closed as failed."; + } + if (record->result->error != error){ + return "The run's error is the start's error."; + } + if (program.current_state() != ProgramState::STOPPED){ + return "The session is stopped again, so the next start can try."; + } + return true; +} + +UnitTestResult test_second_run_preserves_first(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram failing(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "first", nullptr, ImageRGB32(8, 8) + ); + }, + }); + RunRecord first; + UnitTestResult result = run_to_end(failing, store, first); + if (result.result != UnitTestResult::PASSED){ + return result; + } + FakeProgram completing(descriptor, store, {}); + RunRecord second; + result = run_to_end(completing, store, second); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (first.run_id == second.run_id || !(first.run_id < second.run_id)){ + return "Each run has its own ID, sorting after the earlier one."; + } + const std::optional kept = store.get(first.run_id); + if (!kept || kept->result->outcome != RunOutcome::FAILED || kept->result->error != "first" || !kept->result->frame){ + return "The first run's result and frame are still there after the second run."; + } + if (second.result->outcome != RunOutcome::COMPLETED || store.run_ids() != std::vector{first.run_id, second.run_id}){ + return "The second run has its own, completed, record."; + } + return true; +} + + +class Test_RunRecord : public UnitTest{ +public: + Test_RunRecord(std::string name, std::function body) + : UnitTest("CommonFramework::RunRecord - " + std::move(name)) + , m_body(std::move(body)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_body(); + } + +private: + std::function m_body; +}; + +} + + +void add_tests_RunRecord(UnitTestDatabase& database){ + database.add("store: ids and capacity", test_store_ids); + database.add("store: one result per run", test_store_one_result); + database.add("store: events, frames and log tail", test_store_events_and_log); + database.add("store: old runs release frames", test_store_releases_old_frames); + database.add("session recorder", test_session_recorder); + database.add("lifecycle: rejected start", test_rejection); + database.add("lifecycle: immediate completion", test_immediate_completion); + database.add("lifecycle: setup failure", test_setup_failure); + database.add("lifecycle: failure with frame", test_failure_with_frame); + database.add("lifecycle: cancellation", test_cancellation); + database.add("lifecycle: late stop", test_late_stop); + database.add("lifecycle: dispatch failure", test_dispatch_failure); + database.add("lifecycle: second run preserves the first", test_second_run_preserves_first); +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h new file mode 100644 index 0000000000..9a3b88afcd --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h @@ -0,0 +1,20 @@ +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonFramework_RunRecord_Tests_H +#define PokemonAutomation_CommonFramework_RunRecord_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +void add_tests_RunRecord(UnitTestDatabase& database); + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp index 07e95a1a16..5814e7aaab 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp @@ -33,6 +33,10 @@ const ProgramInfo& ProgramEnvironment::program_info() const{ return m_program_info; } +RunRecorder& ProgramEnvironment::run_recorder(){ + return m_session.run_recorder(); +} + void ProgramEnvironment::update_stats(){ m_session.report_stats_changed(); diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h index 72d6d812df..45068ec33c 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h @@ -9,6 +9,7 @@ #include "Common/Compiler.h" #include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/RunRecord.h" namespace PokemonAutomation{ @@ -42,6 +43,12 @@ class ProgramEnvironment{ virtual void log_to_ui(const std::string& msg, Color color = Color()){} +public: + // Progress reporting (RunRecord.h): what the program says about its + // checkpoints, steps and attempts goes to this run's record. + RunRecorder& run_recorder(); + + public: // Stats Management diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 5b3e421dfd..7b7d97a61f 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -20,6 +20,7 @@ #include "CommonFramework/AudioPipeline/Spectrum/FFTStreamer.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ProgramStats/StatsTracking.h" +#include "CommonFramework/RunRecord_Tests.h" #include "CommonFramework/Tools/GlobalThreadPools.h" #include "CommonFramework/Windows/LabControl.h" #include "CommonTools/VisualDetectors/BlackBorderDetector.h" @@ -62,6 +63,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ add_tests_SinkPipeline(ret); add_tests_AudioFileLoader(ret); add_tests_LabControl(ret); + add_tests_RunRecord(ret); OCR::add_tests(ret); Kernels::add_tests(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp index 2cf4ceb24d..d56f6c79c4 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp @@ -7,6 +7,7 @@ #include "Common/Cpp/EarlyShutdown.h" #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Logging/GlobalLogger.h" #include "ConsolePanelSession.h" namespace PokemonAutomation{ @@ -28,7 +29,7 @@ ConsolePanelSession::ConsolePanelSession(const ConsolePanelDescriptor& descripto : UiState(descriptor) , m_descriptor(descriptor) , m_system_option(descriptor.num_controllers()) - , m_system(m_system_option, true, 0) + , m_system(global_logger_raw(), m_system_option, true, 0) , m_instance(descriptor.make_instance(m_system)) {} diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp index f8c81bbd7a..8b95f560c7 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp @@ -39,6 +39,7 @@ ConsoleProgramSession::ConsoleProgramSession( , m_descriptor(descriptor) , m_system_option(1, std::move(extra_option)) , m_system( + run_logger(), m_system_option, descriptor.allow_commands_while_running(), 0, diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp index 0e9e08a0a2..cb14619341 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp @@ -50,6 +50,7 @@ ConsoleSystemSession::~ConsoleSystemSession(){ ); } ConsoleSystemSession::ConsoleSystemSession( + Logger& base_logger, ConsoleSystemOption& option, bool allow_commands_while_locked, size_t console_number, @@ -57,7 +58,7 @@ ConsoleSystemSession::ConsoleSystemSession( ) : m_allow_commands_while_locked(allow_commands_while_locked) , m_console_number(console_number) - , m_logger(global_logger_raw(), "Console " + std::to_string(console_number)) + , m_logger(base_logger, "Console " + std::to_string(console_number)) , m_option(option) , m_video(m_logger, option.m_video) , m_audio(m_logger, option.m_audio) diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h index 62ec876c71..8c08e359a8 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h @@ -66,7 +66,11 @@ class ConsoleSystemSession final virtual bool try_shutdown() noexcept override; ~ConsoleSystemSession(); + // `base_logger` is what this console's logger writes through: a + // program session passes its run logger (ProgramSession::run_logger()), + // so the console's lines land in the run's record. ConsoleSystemSession( + Logger& base_logger, ConsoleSystemOption& option, bool allow_commands_while_locked, size_t console_number, diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsolePanelSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsolePanelSession.cpp index 0b42002c1c..2de1af248c 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsolePanelSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsolePanelSession.cpp @@ -7,6 +7,7 @@ #include "Common/Cpp/EarlyShutdown.h" #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/Logging/Logger.h" #include "MultiConsolePanelSession.h" @@ -34,7 +35,7 @@ MultiConsolePanelSession::MultiConsolePanelSession(const MultiConsolePanelDescri descriptor.default_consoles(), descriptor.option_factory() ) - , m_system(m_system_option, true) + , m_system(global_logger_raw(), m_system_option, true) , m_instance(descriptor.make_instance(m_system)) {} diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp index 3f6cbb6e18..6ed9c67747 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp @@ -47,7 +47,7 @@ MultiConsoleProgramSession::MultiConsoleProgramSession( descriptor.default_consoles() ) ) - , m_system(*m_system_option, descriptor.allow_commands_while_running(), instance_id()) + , m_system(run_logger(), *m_system_option, descriptor.allow_commands_while_running(), instance_id()) { auto instance = descriptor.make_instance(m_system); instance->update_active_consoles(m_system_option->active_consoles()); diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp index dde05b293c..613d62dabf 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp @@ -40,11 +40,13 @@ MultiConsoleSystemSession::~MultiConsoleSystemSession(){ } MultiConsoleSystemSession::MultiConsoleSystemSession( + Logger& base_logger, MultiConsoleSystemOption& option, bool allow_commands_while_locked, std::optional program_tracking_id ) : m_logger(global_logger_tagged()) + , m_base_logger(base_logger) , m_option(option) , m_allow_commands_while_locked(allow_commands_while_locked) , m_program_tracking_id(program_tracking_id) @@ -54,6 +56,7 @@ MultiConsoleSystemSession::MultiConsoleSystemSession( for (size_t c = 0; c < count; c++){ m_consoles.emplace_back( std::make_unique( + m_base_logger, option[c], allow_commands_while_locked, c, @@ -121,6 +124,7 @@ void MultiConsoleSystemSession::set_active_consoles(size_t count){ for (size_t c = 0; c < count; c++){ m_consoles.emplace_back( std::make_unique( + m_base_logger, m_option[c], m_allow_commands_while_locked, c, diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h index e31786be68..117ccf260d 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h @@ -47,7 +47,10 @@ class MultiConsoleSystemSession // Construct the multi-console session. // By default it constructs the generic console. But you can inject your // own factory for your own custom subclass. + // `base_logger` is what the consoles' loggers write through; see + // ConsoleSystemSession. MultiConsoleSystemSession( + Logger& base_logger, MultiConsoleSystemOption& option, bool allow_commands_while_locked, std::optional program_tracking_id = {} @@ -85,6 +88,7 @@ class MultiConsoleSystemSession private: Logger& m_logger; + Logger& m_base_logger; // What the consoles' loggers write through. MultiConsoleSystemOption& m_option; bool m_allow_commands_while_locked; diff --git a/SerialPrograms/Source/Integrations/ProgramTracker.cpp b/SerialPrograms/Source/Integrations/ProgramTracker.cpp index ca3c242416..b7734cb55c 100644 --- a/SerialPrograms/Source/Integrations/ProgramTracker.cpp +++ b/SerialPrograms/Source/Integrations/ProgramTracker.cpp @@ -173,8 +173,7 @@ std::string ProgramTracker::start_program(uint64_t program_id){ if (program == nullptr){ return error; } - program->async_start(); - return ""; + return program->async_start(); } std::string ProgramTracker::stop_program(uint64_t program_id){ std::string header = make_header("stop_program", program_id); @@ -184,8 +183,7 @@ std::string ProgramTracker::stop_program(uint64_t program_id){ if (program == nullptr){ return error; } - program->async_stop(); - return ""; + return program->async_stop(); } std::string ProgramTracker::run_controller_command( uint64_t console_id, uint64_t controller_index, diff --git a/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h b/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h index 9d4f4c3125..e771bb80cb 100644 --- a/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h +++ b/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h @@ -36,8 +36,8 @@ class TrackableProgram{ virtual ProgramState current_state() const = 0; virtual std::string current_stats() const = 0; - virtual void async_start() = 0; - virtual void async_stop() = 0; + virtual std::string async_start() = 0; + virtual std::string async_stop() = 0; }; diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp index 2d107709b9..07b85fbec2 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp @@ -6,6 +6,7 @@ #include "Common/Compiler.h" #include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" @@ -274,7 +275,7 @@ class Test_UpdatePopupDetector : public ImageUnitTest{ const std::vector& images ) const override{ GameConsole::ConsoleSystemOption option(1); - GameConsole::ConsoleSystemSession session(option, false, 0, {}); + GameConsole::ConsoleSystemSession session(global_logger_raw(), option, false, 0, {}); ConsoleHandle console(session); UpdatePopupDetector detector(console); diff --git a/SerialPrograms/Source/VideoGames/PokemonLA/Inference/Sounds/PokemonLA_ShinySoundDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLA/Inference/Sounds/PokemonLA_ShinySoundDetector.cpp index b58b5f526e..40103a79c1 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLA/Inference/Sounds/PokemonLA_ShinySoundDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLA/Inference/Sounds/PokemonLA_ShinySoundDetector.cpp @@ -4,6 +4,7 @@ * */ +#include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonTools/Audio/SpectrogramMatcher.h" #include "CommonTools/Audio/AudioTemplateCache.h" #include "Tests/TestUtils.h" @@ -55,7 +56,7 @@ class Test_ShinySoundDetector : public UnitTest{ virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ GameConsole::ConsoleSystemOption option(1); - GameConsole::ConsoleSystemSession session(option, false, 0, {}); + GameConsole::ConsoleSystemSession session(global_logger_raw(), option, false, 0, {}); ConsoleHandle console(session); ShinySoundDetector detector(console, [&](float error_coefficient) -> bool{ return true; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 79b6848df0..48ddcd5849 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -672,6 +672,10 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/PersistentSettings.h Source/CommonFramework/ProgramSession.cpp Source/CommonFramework/ProgramSession.h + Source/CommonFramework/RunRecord.cpp + Source/CommonFramework/RunRecord.h + Source/CommonFramework/RunRecord_Tests.cpp + Source/CommonFramework/RunRecord_Tests.h Source/CommonFramework/ProgramStats/StatsDatabase.cpp Source/CommonFramework/ProgramStats/StatsDatabase.h Source/CommonFramework/ProgramStats/StatsTracking.cpp From 74dd0fbccbe199fb1516331bbc40d86b5deb456a Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 13:08:24 -0800 Subject: [PATCH 2/4] Run records: the build, the settings, a log up to 8 MiB, and off by default - Each record names the app's version, the git commit it was built from (PROGRAM_COMMIT, written into BuildCommit.cpp by cmake/BuildCommit.cmake on every build and rewritten only when the commit changes) and the program's settings as the settings file stores them. - The log is numbered line by line, so a reader can copy it out as it grows: the store keeps a 256 KiB tail and records at most 8 MiB per run, counting the lines past that. - The app's store records nothing until it is enabled; whoever reads the records enables it. Tests build their own, enabled stores. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- SerialPrograms/CMakeLists.txt | 18 +++++ .../Source/CommonFramework/BuildCommit.h | 24 ++++++ .../Source/CommonFramework/ProgramSession.cpp | 2 +- .../Source/CommonFramework/ProgramSession.h | 4 + .../Source/CommonFramework/RunRecord.cpp | 44 ++++++----- .../Source/CommonFramework/RunRecord.h | 56 ++++++++++---- .../CommonFramework/RunRecord_Tests.cpp | 77 +++++++++++++++++-- .../Framework/ConsoleProgramSession.cpp | 3 + .../Framework/ConsoleProgramSession.h | 1 + .../Framework/MultiConsoleProgramSession.cpp | 3 + .../Framework/MultiConsoleProgramSession.h | 1 + SerialPrograms/cmake/BuildCommit.cmake | 34 ++++++++ SerialPrograms/cmake/SourceFiles.cmake | 1 + 13 files changed, 227 insertions(+), 41 deletions(-) create mode 100644 SerialPrograms/Source/CommonFramework/BuildCommit.h create mode 100644 SerialPrograms/cmake/BuildCommit.cmake diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 35bb597e32..b0a02a7243 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -193,6 +193,24 @@ include(cmake/CoreLib.cmake) # Create the GUI library add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES}) +# PROGRAM_COMMIT (Source/CommonFramework/BuildCommit.h): the git commit the app is +# built from. BuildCommit.cpp is written at configure time and checked on every +# build; it changes, and recompiles, only when the commit does. +set(PA_BUILD_COMMIT_CPP "${CMAKE_CURRENT_BINARY_DIR}/generated/BuildCommit.cpp") +set(PA_BUILD_COMMIT_COMMAND + "${CMAKE_COMMAND}" "-DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}" "-DOUTPUT=${PA_BUILD_COMMIT_CPP}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/BuildCommit.cmake" +) +execute_process(COMMAND ${PA_BUILD_COMMIT_COMMAND}) +add_custom_target(BuildCommit + COMMAND ${PA_BUILD_COMMIT_COMMAND} + BYPRODUCTS "${PA_BUILD_COMMIT_CPP}" + COMMENT "Checking the git commit for PROGRAM_COMMIT" + VERBATIM +) +target_sources(SerialProgramsLib PRIVATE "${PA_BUILD_COMMIT_CPP}") +add_dependencies(SerialProgramsLib BuildCommit) + # Create binary target if (APPLE) diff --git a/SerialPrograms/Source/CommonFramework/BuildCommit.h b/SerialPrograms/Source/CommonFramework/BuildCommit.h new file mode 100644 index 0000000000..3e0568cd4f --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/BuildCommit.h @@ -0,0 +1,24 @@ +/* Build Commit + * + * From: https://github.com/PokemonAutomation/ + * + * The git commit the app was built from. The build writes its definition + * (cmake/BuildCommit.cmake) into BuildCommit.cpp in the build folder, and + * rewrites it only when the commit changes, so a new commit recompiles + * that one file. + * + */ + +#ifndef PokemonAutomation_BuildCommit_H +#define PokemonAutomation_BuildCommit_H + +namespace PokemonAutomation{ + + +// "git describe" of the source tree: the commit's hash, with "-dirty" when +// tracked files had changes, or "unknown" when git could not tell. +extern const char* const PROGRAM_COMMIT; + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp index d091dc4b80..5aa77f368e 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp @@ -257,7 +257,7 @@ std::string ProgramSession::start_program(){ // exists before the program thread can report anything. m_logger.log("Starting program..."); m_last_state_change.store(current_time(), std::memory_order_relaxed); - m_run_recorder.begin(identifier()); + m_run_recorder.begin(identifier(), settings_json()); set_state(ProgramState::RUNNING); try{ m_program_thread = dispatch_program_thread([this]{ diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.h b/SerialPrograms/Source/CommonFramework/ProgramSession.h index 36889c5c30..3772ae3f76 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.h +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.h @@ -148,6 +148,10 @@ class ProgramSession : public TrackableProgram{ // Starts the program thread. A test fake makes this fail. virtual AsyncTask dispatch_program_thread(std::function body); + // The program's settings as JSON text, recorded with each run: what the + // settings file stores for it. "" for a session without settings. + virtual std::string settings_json() const{ return ""; } + // virtual void restore_defaults(){ return; } public: diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.cpp b/SerialPrograms/Source/CommonFramework/RunRecord.cpp index 17cffdca16..4d64f4b30b 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord.cpp +++ b/SerialPrograms/Source/CommonFramework/RunRecord.cpp @@ -9,6 +9,8 @@ #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/BuildCommit.h" +#include "CommonFramework/Globals.h" #include "RunRecord.h" namespace PokemonAutomation{ @@ -41,18 +43,22 @@ std::string run_outcome_name(RunOutcome outcome){ RunRecordStore& RunRecordStore::instance(){ - static RunRecordStore store; + static RunRecordStore store(DEFAULT_CAPACITY, false); return store; } -RunRecordStore::RunRecordStore(size_t capacity) +RunRecordStore::RunRecordStore(size_t capacity, bool enabled) // The app's start time, to the microsecond: two instances started in // the same second still get different IDs. : m_app_token(now_to_filestring()) , m_capacity(std::max(capacity, 1)) + , m_enabled(enabled) {} -std::string RunRecordStore::begin(std::string program){ +std::string RunRecordStore::begin(std::string program, std::string settings){ + if (!enabled()){ + return ""; + } std::lock_guard lg(m_lock); while (m_records.size() >= m_capacity){ m_records.pop_front(); @@ -60,6 +66,9 @@ std::string RunRecordStore::begin(std::string program){ RunRecord& record = m_records.emplace_back(); record.run_id = std::format("{}-{:03}", m_app_token, ++m_counter); record.program = std::move(program); + record.version = PROGRAM_VERSION; + record.commit = PROGRAM_COMMIT; + record.settings = std::move(settings); record.start_time = current_time(); return record.run_id; } @@ -105,19 +114,18 @@ void RunRecordStore::log_line(const std::string& run_id, const std::string& text if (record == nullptr){ return; } - record->log_tail.emplace_back(text); - size_t bytes = 0; - for (const std::string& line : record->log_tail){ - bytes += line.size() + 1; - } - size_t drop = 0; - while (bytes > MAX_LOG_BYTES && drop + 1 < record->log_tail.size()){ - bytes -= record->log_tail[drop].size() + 1; - drop++; + const size_t bytes = text.size() + 1; + if (record->log_bytes + bytes > MAX_LOG_BYTES){ + record->dropped_log_lines++; + return; } - if (drop > 0){ - record->log_tail.erase(record->log_tail.begin(), record->log_tail.begin() + drop); - record->log_truncated = true; + record->log_lines++; + record->log_bytes += bytes; + record->log_tail.emplace_back(text); + record->log_tail_bytes += bytes; + while (record->log_tail_bytes > MAX_LOG_TAIL_BYTES && record->log_tail.size() > 1){ + record->log_tail_bytes -= record->log_tail.front().size() + 1; + record->log_tail.pop_front(); } } @@ -182,10 +190,10 @@ SessionRunRecorder::SessionRunRecorder(RunRecordStore& store, Logger& base_logge , m_base_logger(base_logger) {} -std::string SessionRunRecorder::begin(std::string program){ +std::string SessionRunRecorder::begin(std::string program, std::string settings){ std::lock_guard lg(m_lock); - m_run_id = m_store.begin(std::move(program)); - m_active = true; + m_run_id = m_store.begin(std::move(program), std::move(settings)); + m_active = !m_run_id.empty(); return m_run_id; } void SessionRunRecorder::stopping(){ diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.h b/SerialPrograms/Source/CommonFramework/RunRecord.h index f9afd2c86d..294e6ad5d9 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord.h +++ b/SerialPrograms/Source/CommonFramework/RunRecord.h @@ -5,16 +5,18 @@ * What one run of a program did, as the framework saw it: when it started, * how it ended (exactly one outcome per accepted run), the error and the * frame it died with, the events the program reported on the way, and the - * tail of its log. ProgramSession writes these at its execution boundary; - * RunRecordStore keeps the last few for whoever asks, such as the Lab - * channel, which writes them to files (Windows/Lab.md, "Run records"). - * Nothing here depends on the UI, on JSON or on a game. + * lines of its log, and the build and settings it ran with. + * ProgramSession writes these at its execution boundary; RunRecordStore + * keeps the last few for whoever asks, such as a tool that writes them to + * files. The store records nothing until it is enabled. Nothing here + * depends on the UI, on JSON or on a game. * */ #ifndef PokemonAutomation_CommonFramework_RunRecord_H #define PokemonAutomation_CommonFramework_RunRecord_H +#include #include #include #include @@ -67,14 +69,25 @@ struct RunResult{ struct RunRecord{ std::string run_id = {}; std::string program = {}; // The program's identifier, ":". + std::string version = {}; // PROGRAM_VERSION of the app that ran it. + std::string commit = {}; // PROGRAM_COMMIT (BuildCommit.h): the git commit it was built from. + std::string settings = {}; // The program's settings as JSON text, as the settings file stores them. WallClock start_time = WallClock::min(); std::optional end_time = {}; RunStatus status = RunStatus::RUNNING; std::optional result = {}; // Set once, when the status turns TERMINAL. std::vector events = {}; size_t dropped_events = 0; // Events past RunRecordStore::MAX_EVENTS. - std::vector log_tail = {}; // The run's last log lines, oldest first. - bool log_truncated = false; // Whether older lines were dropped from the tail. + + // The run's log. Lines are numbered from 0 in the order they came; + // the tail holds the newest of them, so it starts at line + // `log_lines - log_tail.size()`. A reader that copies the log out as it + // grows keeps its own count of the lines it has. + std::deque log_tail = {}; // At most MAX_LOG_TAIL_BYTES, oldest first. + size_t log_tail_bytes = 0; // The tail's size, a newline per line. + size_t log_lines = 0; // Lines recorded so far. + size_t log_bytes = 0; // Their size, a newline per line, at most MAX_LOG_BYTES. + size_t dropped_log_lines = 0; // Lines past MAX_LOG_BYTES, not recorded. }; @@ -89,30 +102,41 @@ class RunRecorder{ // The last few runs of this app instance. Thread-safe; every call is short // and copies, so a reader never holds a lock while writing files. // +// Disabled, the store records nothing: begin() returns "" and every call +// about that run is a no-op. The app's instance() starts disabled; whoever +// reads the records (e.g. the lab channel) enables it. +// // Run IDs are opaque strings unique to this app instance: a restart cannot // resolve an old ID to a new run. They sort in the order the runs started. // // Memory: at most MAX_EVENTS events per run; the frames of a run's newest // MAX_EVENT_FRAMES events; all frames only for the active run and the -// newest RUNS_WITH_FRAMES terminal runs; a log tail of MAX_LOG_BYTES. -// A consumer that wants a frame reads it soon after the event or the end. +// newest RUNS_WITH_FRAMES terminal runs; a log tail of MAX_LOG_TAIL_BYTES. +// A consumer that wants a frame or every log line reads them soon after +// they come. A run records at most MAX_LOG_BYTES of log in all. class RunRecordStore{ public: static constexpr size_t DEFAULT_CAPACITY = 32; static constexpr size_t MAX_EVENTS = 4096; static constexpr size_t MAX_EVENT_FRAMES = 8; static constexpr size_t RUNS_WITH_FRAMES = 4; - static constexpr size_t MAX_LOG_BYTES = 64 * 1024; + static constexpr size_t MAX_LOG_TAIL_BYTES = 256 * 1024; + static constexpr size_t MAX_LOG_BYTES = 8 * 1024 * 1024; + // The app's store. Disabled until someone enables it. static RunRecordStore& instance(); - explicit RunRecordStore(size_t capacity = DEFAULT_CAPACITY); + explicit RunRecordStore(size_t capacity = DEFAULT_CAPACITY, bool enabled = true); RunRecordStore(const RunRecordStore&) = delete; void operator=(const RunRecordStore&) = delete; - // Opens a record for an accepted start and returns its run ID. The - // oldest record goes when the store is full. - std::string begin(std::string program); + void set_enabled(bool enabled){ m_enabled.store(enabled, std::memory_order_release); } + bool enabled() const{ return m_enabled.load(std::memory_order_acquire); } + + // Opens a record for an accepted start and returns its run ID, or "" + // when the store is disabled. The oldest record goes when the store is + // full. `settings` is the program's settings as JSON text. + std::string begin(std::string program, std::string settings = ""); // Marks a running run as stopping. No effect once it is terminal. void stopping(const std::string& run_id); @@ -139,6 +163,7 @@ class RunRecordStore{ mutable Mutex m_lock; const std::string m_app_token; const size_t m_capacity; + std::atomic m_enabled; uint64_t m_counter = 0; std::deque m_records; }; @@ -152,8 +177,9 @@ class SessionRunRecorder : public RunRecorder, public Logger{ public: SessionRunRecorder(RunRecordStore& store, Logger& base_logger); - // Opens a run and makes it the active one. Returns its ID. - std::string begin(std::string program); + // Opens a run and makes it the active one. Returns its ID, or "" when + // the store is disabled, in which case nothing of it is recorded. + std::string begin(std::string program, std::string settings = ""); void stopping(); // Closes the active run; nothing happens when none is active. void finish(RunResult result); diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp index aedd954fa0..553b1d6489 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp @@ -8,13 +8,16 @@ */ #include +#include #include #include #include #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/BuildCommit.h" #include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Globals.h" #include "CommonFramework/Notifications/ProgramInfo.h" #include "CommonFramework/Panels/PanelSession.h" #include "CommonFramework/Panels/ProgramDescriptor.h" @@ -52,7 +55,7 @@ bool wait_until(const std::function& done, Milliseconds timeout = 5000ms UnitTestResult test_store_ids(){ RunRecordStore store(2); - const std::string a = store.begin("Test:A"); + const std::string a = store.begin("Test:A", R"({"START": 3})"); const std::string b = store.begin("Test:B"); if (a.empty() || a == b || !(a < b)){ return std::format("Run IDs should be distinct and sort in start order: <{}>, <{}>", a, b); @@ -67,6 +70,12 @@ UnitTestResult test_store_ids(){ if (record->start_time == WallClock::min() || record->end_time){ return "A new record has a start time and no end time."; } + if (record->version != PROGRAM_VERSION || record->commit != PROGRAM_COMMIT || record->commit.empty()){ + return std::format("A record names the build: version <{}>, commit <{}>.", record->version, record->commit); + } + if (record->settings != R"({"START": 3})" || !store.get(b)->settings.empty()){ + return "A record keeps the settings it was begun with."; + } const std::string c = store.begin("Test:C"); if (store.get(a) || store.run_ids() != std::vector{b, c}){ return "A full store drops its oldest record."; @@ -131,17 +140,64 @@ UnitTestResult test_store_events_and_log(){ return std::format("Events past the cap are counted, not kept: {} events, {} dropped.", record.events.size(), record.dropped_events); } - const std::string line(1000, 'x'); - for (size_t c = 0; c < 100; c++){ - store.log_line(id, std::format("{} {}", c, line)); + // Lines of 1000 bytes, numbered: 300 are more than the tail holds. + const std::string line(995, 'x'); + for (size_t c = 0; c < 300; c++){ + store.log_line(id, std::format("{:04} {}", c, line)); } record = *store.get(id); size_t bytes = 0; for (const std::string& l : record.log_tail){ bytes += l.size() + 1; } - if (!record.log_truncated || bytes > RunRecordStore::MAX_LOG_BYTES || record.log_tail.back().rfind("99 ", 0) != 0){ - return std::format("The log tail keeps the newest {} bytes: {} lines, {} bytes.", RunRecordStore::MAX_LOG_BYTES, record.log_tail.size(), bytes); + if (record.log_lines != 300 || record.log_bytes != 300 * 1001 || record.dropped_log_lines != 0){ + return std::format("Every line is counted: {} lines, {} bytes.", record.log_lines, record.log_bytes); + } + if (bytes != record.log_tail_bytes || bytes > RunRecordStore::MAX_LOG_TAIL_BYTES || record.log_tail.size() == 300){ + return std::format("The tail keeps at most {} bytes: {} lines, {} bytes.", RunRecordStore::MAX_LOG_TAIL_BYTES, record.log_tail.size(), bytes); + } + const size_t first = record.log_lines - record.log_tail.size(); + if (record.log_tail.front().rfind(std::format("{:04} ", first), 0) != 0 || record.log_tail.back().rfind("0299 ", 0) != 0){ + return "The tail holds the newest lines, from line log_lines - log_tail.size()."; + } + + // Past MAX_LOG_BYTES in all, lines are counted and not kept. + const size_t fit = RunRecordStore::MAX_LOG_BYTES / 1001; + for (size_t c = 300; c < fit + 10; c++){ + store.log_line(id, std::format("{:04} {}", c % 10000, line)); + } + record = *store.get(id); + if (record.log_lines != fit || record.dropped_log_lines != 10 || record.log_bytes > RunRecordStore::MAX_LOG_BYTES){ + return std::format("A run records at most {} bytes of log: {} lines, {} dropped.", RunRecordStore::MAX_LOG_BYTES, record.log_lines, record.dropped_log_lines); + } + return true; +} + +UnitTestResult test_store_disabled(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + RunRecordStore store(4, false); + if (store.enabled() || !store.begin("Test:A").empty() || !store.run_ids().empty()){ + return "A disabled store opens no record."; + } + CountingLogger base; + SessionRunRecorder recorder(store, base); + if (!recorder.begin("Test:A").empty() || !recorder.last_run_id().empty()){ + return "A recorder over a disabled store has no run."; + } + recorder.log("line"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + if (base.lines != 1 || !store.run_ids().empty()){ + return "Lines still reach the base logger; nothing is recorded."; + } + store.set_enabled(true); + const std::string id = recorder.begin("Test:A"); + recorder.log("line"); + if (id.empty() || store.get(id)->log_lines != 1){ + return "Once enabled, the store records."; } return true; } @@ -195,7 +251,7 @@ UnitTestResult test_session_recorder(){ if (base.lines != 3){ return std::format("Every line reaches the base logger: {} of 3.", base.lines); } - if (record.log_tail != std::vector{"during"} || record.events.size() != 1){ + if (record.log_tail != std::deque{"during"} || record.events.size() != 1){ return "Only lines and events during the run are recorded."; } if (record.result->outcome != RunOutcome::COMPLETED || recorder.last_run_id() != id){ @@ -212,6 +268,7 @@ struct FakeScript{ bool fail_dispatch = false; // The program thread cannot be started. std::function start_checks = {}; std::function body = {}; + std::string settings = {}; // settings_json(). }; class FakeDescriptor : public ProgramDescriptor{ @@ -276,6 +333,7 @@ class FakeProgram : public ProgramSession{ m_script.body(*this, env, *m_scope); } } + virtual std::string settings_json() const override{ return m_script.settings; } virtual AsyncTask dispatch_program_thread(std::function body) override{ if (m_script.fail_dispatch){ throw std::runtime_error("no thread"); @@ -328,6 +386,7 @@ UnitTestResult test_immediate_completion(){ .body = [](ProgramSession&, ProgramEnvironment& env, CancellableScope&){ env.run_recorder().event({.kind = "checkpoint", .checkpoint = "00.01", .step = "3/3", .attempt = 1, .text = "ok", .pose = "home (2, 4) N"}); }, + .settings = R"({"START": 1})", }); RunRecord record; UnitTestResult result = run_to_end(program, store, record); @@ -340,6 +399,9 @@ UnitTestResult test_immediate_completion(){ if (record.program != "Test:RunRecordFake" || record.start_time > *record.end_time){ return "The record names the program and its times."; } + if (record.settings != R"({"START": 1})" || record.commit != PROGRAM_COMMIT){ + return "The record has the session's settings and the build's commit."; + } if (record.events.size() != 1 || record.events[0].checkpoint != "00.01" || record.events[0].attempt != size_t(1) || record.events[0].pose != "home (2, 4) N"){ return std::format("The program's event is in the record: {} events.", record.events.size()); } @@ -541,6 +603,7 @@ void add_tests_RunRecord(UnitTestDatabase& database){ database.add("store: one result per run", test_store_one_result); database.add("store: events, frames and log tail", test_store_events_and_log); database.add("store: old runs release frames", test_store_releases_old_frames); + database.add("store: disabled", test_store_disabled); database.add("session recorder", test_session_recorder); database.add("lifecycle: rejected start", test_rejection); database.add("lifecycle: immediate completion", test_immediate_completion); diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp index 8b95f560c7..af14b9b2db 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp @@ -63,6 +63,9 @@ void ConsoleProgramSession::restore_defaults(){ logger().log("Restoring settings to defaults..."); m_instance->restore_defaults(); } +std::string ConsoleProgramSession::settings_json() const{ + return to_json().dump(); +} JsonValue ConsoleProgramSession::to_json() const{ JsonObject obj = std::move(*m_instance->to_json().to_object()); obj["ConsoleSetup"] = m_system_option.to_json(); diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.h b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.h index 97b373df80..50c78e61fe 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.h @@ -45,6 +45,7 @@ class ConsoleProgramSession protected: virtual std::unique_ptr make_env(const ProgramInfo& program_info) override; virtual void internal_run_program(ProgramEnvironment& env) override; + virtual std::string settings_json() const override; private: diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp index 6ed9c67747..b6d6c62b52 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp @@ -72,6 +72,9 @@ void MultiConsoleProgramSession::restore_defaults(){ logger().log("Restoring settings to defaults..."); m_instance->restore_defaults(); } +std::string MultiConsoleProgramSession::settings_json() const{ + return to_json().dump(); +} JsonValue MultiConsoleProgramSession::to_json() const{ JsonObject obj = std::move(*m_instance->to_json().to_object()); obj["ConsoleSetup"] = m_system_option->to_json(); diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.h b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.h index 44fd2f03f4..3230148f11 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.h @@ -46,6 +46,7 @@ class MultiConsoleProgramSession protected: virtual std::unique_ptr make_env(const ProgramInfo& program_info) override; virtual void internal_run_program(ProgramEnvironment& env) override; + virtual std::string settings_json() const override; virtual void on_console_count_lock(bool locked) override{} virtual void shutdown() override; diff --git a/SerialPrograms/cmake/BuildCommit.cmake b/SerialPrograms/cmake/BuildCommit.cmake new file mode 100644 index 0000000000..3c6579f609 --- /dev/null +++ b/SerialPrograms/cmake/BuildCommit.cmake @@ -0,0 +1,34 @@ +# Writes BuildCommit.cpp, the definition of PROGRAM_COMMIT +# (Source/CommonFramework/BuildCommit.h): the git commit the app is built from. +# +# Run as a script on every build: +# cmake -DSOURCE_DIR= -DOUTPUT=<.../BuildCommit.cpp> -P BuildCommit.cmake +# The file is rewritten only when its text changes, so an unchanged commit +# rebuilds nothing. Without git, or outside a repository, the commit is "unknown". + +set(PA_COMMIT "unknown") +find_package(Git QUIET) +if(GIT_FOUND) + # --match with a pattern no tag has: always the hash, never a tag name. + execute_process( + COMMAND "${GIT_EXECUTABLE}" describe --always --dirty --abbrev=12 --match "no-tag-matches-this" + WORKING_DIRECTORY "${SOURCE_DIR}" + OUTPUT_VARIABLE PA_DESCRIBE + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE PA_RESULT + ERROR_QUIET + ) + if(PA_RESULT EQUAL 0 AND NOT PA_DESCRIBE STREQUAL "") + set(PA_COMMIT "${PA_DESCRIBE}") + endif() +endif() + +set(PA_TEXT "// Generated by cmake/BuildCommit.cmake. Do not edit.\n\nnamespace PokemonAutomation{\n extern const char* const PROGRAM_COMMIT;\n const char* const PROGRAM_COMMIT = \"${PA_COMMIT}\";\n}\n") +if(EXISTS "${OUTPUT}") + file(READ "${OUTPUT}" PA_OLD_TEXT) +else() + set(PA_OLD_TEXT "") +endif() +if(NOT PA_OLD_TEXT STREQUAL PA_TEXT) + file(WRITE "${OUTPUT}" "${PA_TEXT}") +endif() diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 48ddcd5849..fba67495f0 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -565,6 +565,7 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/AudioPipeline/UI/AudioDisplayWidget.h Source/CommonFramework/AudioPipeline/UI/AudioSelectorWidget.cpp Source/CommonFramework/AudioPipeline/UI/AudioSelectorWidget.h + Source/CommonFramework/BuildCommit.h Source/CommonFramework/Environment/HardwareValidation.cpp Source/CommonFramework/Environment/HardwareValidation.h Source/CommonFramework/Environment/HardwareValidation_arm64.tpp From 2909e33d4cc212de0888c322d70817cc600dd5dd Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 15:17:13 -0800 Subject: [PATCH 3/4] Run records: meet the code-health limits - The progress hook is a free function, record_run_event(env, event), in place of the single-method RunRecorder interface; ProgramSession and ProgramEnvironment gain no public method (the lab reads run IDs from the store). - The app's store is app_run_records(), switched on by enable_app_run_records(); a store takes an optional flag in place of a defaulted bool and set_enabled(). - start_program() and run_program() hand their accepted-start, preparation, download and end steps to private helpers, so neither grows past its old size or nesting. - The store's events and log tests are two tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Source/CommonFramework/ProgramSession.cpp | 136 +- .../Source/CommonFramework/ProgramSession.h | 26 +- .../Source/CommonFramework/RunRecord.cpp | 18 +- .../Source/CommonFramework/RunRecord.h | 47 +- .../CommonFramework/RunRecord_Tests.cpp | 1255 +++++++++-------- .../Tools/ProgramEnvironment.cpp | 4 +- .../Tools/ProgramEnvironment.h | 12 +- .../Framework/ConsoleSystemSession.cpp | 5 +- 8 files changed, 772 insertions(+), 731 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp index 5aa77f368e..fe9c3ddf22 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp @@ -243,36 +243,8 @@ std::string ProgramSession::start_program(){ m_logger.log("Program is not ready.", COLOR_RED); return "Program is not ready."; - case ProgramState::STOPPED:{ - std::string error = check_validity(); - if (!error.empty()){ - m_logger.log(error, COLOR_RED); - return error; - } - - // Wait for previous program thread to finish. - join_program_thread(); - - // Now start the program. The run is accepted here: its record - // exists before the program thread can report anything. - m_logger.log("Starting program..."); - m_last_state_change.store(current_time(), std::memory_order_relaxed); - m_run_recorder.begin(identifier(), settings_json()); - set_state(ProgramState::RUNNING); - try{ - m_program_thread = dispatch_program_thread([this]{ - run_with_catch( - "ProgramSession::start_program()", - [this]{ run_program_and_record(); } - ); - }); - }catch (Exception& e){ - return abandon_run(std::format("The program thread could not be started: {}", e.message())); - }catch (std::exception& e){ - return abandon_run(std::format("The program thread could not be started: {}", e.what())); - } - return ""; - } + case ProgramState::STOPPED: + return start_accepted_run(); case ProgramState::RUNNING: m_logger.log("Program is already running.", COLOR_RED); return "Program is already running."; @@ -324,6 +296,38 @@ std::string ProgramSession::stop_program(){ +// Under the lock, for a stopped session: checks the settings, then starts +// the program thread. The run is accepted here: its record exists before +// the program thread can report anything. +std::string ProgramSession::start_accepted_run(){ + std::string error = check_validity(); + if (!error.empty()){ + m_logger.log(error, COLOR_RED); + return error; + } + + // Wait for previous program thread to finish. + join_program_thread(); + + m_logger.log("Starting program..."); + m_last_state_change.store(current_time(), std::memory_order_relaxed); + m_run_recorder.begin(identifier(), settings_json()); + set_state(ProgramState::RUNNING); + try{ + m_program_thread = dispatch_program_thread([this]{ + run_with_catch( + "ProgramSession::start_program()", + [this]{ run_program_and_record(); } + ); + }); + }catch (Exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.message())); + }catch (std::exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.what())); + } + return ""; +} + // No thread, so no run: closes the record and leaves the session stopped, // so the next start can try again. Returns `message` for start_program(). std::string ProgramSession::abandon_run(std::string message){ @@ -360,25 +364,51 @@ void ProgramSession::run_program_and_record(){ } } +// Under the lock: false when the run was stopped before the thread got +// going. Throws when the settings are no longer valid. Loads the stats. +bool ProgramSession::prepare_run(){ + std::lock_guard lg(m_lock); + if (current_state() != ProgramState::RUNNING){ + return false; + } + + std::string error = check_validity(); + if (!error.empty()){ + throw UserSetupError(logger(), std::move(error)); + } + + m_current_stats = m_descriptor.make_stats(); + load_historical_stats(); + push_stats(); + return true; +} + +// The run's result when the resources it needs could not be downloaded. +std::optional ProgramSession::download_resources(){ + if (download_prereqs(*m_scope)){ + return std::nullopt; + } + // The details were reported as "error" events. A stop during the + // download is a cancellation, not a failure. + if (current_state() == ProgramState::STOPPING){ + return RunResult{.outcome = RunOutcome::CANCELLED}; + } + return RunResult{.outcome = RunOutcome::FAILED, .error = "A required resource could not be downloaded."}; +} + +void ProgramSession::end_run(){ + std::lock_guard lg(m_lock); + push_stats(); + update_historical_stats_with_current(); + set_state(ProgramState::STOPPED); +} + RunResult ProgramSession::run_program(){ SleepSuppressScope sleep_scope(GlobalSettings::instance().SLEEP_SUPPRESS->PROGRAM_RUNNING); RunResult result; - { - std::lock_guard lg(m_lock); - if (current_state() != ProgramState::RUNNING){ - // Stopped before the thread got going. - return {.outcome = RunOutcome::CANCELLED}; - } - - std::string error = check_validity(); - if (!error.empty()){ - throw UserSetupError(logger(), std::move(error)); - } - - m_current_stats = m_descriptor.make_stats(); - load_historical_stats(); - push_stats(); + if (!prepare_run()){ + return {.outcome = RunOutcome::CANCELLED}; } m_instance->options().reset_state(); @@ -392,13 +422,8 @@ RunResult ProgramSession::run_program(){ { RunningProgramScope running_scope(*this); - if (!download_prereqs(*m_scope)){ - // The details were reported as "error" events. A stop during - // the download is a cancellation, not a failure. - if (current_state() == ProgramState::STOPPING){ - return {.outcome = RunOutcome::CANCELLED}; - } - return {.outcome = RunOutcome::FAILED, .error = "A required resource could not be downloaded."}; + if (std::optional failed = download_resources()){ + return *failed; } std::unique_ptr env = make_env(program_info); try{ @@ -507,12 +532,7 @@ RunResult ProgramSession::run_program(){ } } - { - std::lock_guard lg(m_lock); - push_stats(); - update_historical_stats_with_current(); - set_state(ProgramState::STOPPED); - } + end_run(); return result; } diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.h b/SerialPrograms/Source/CommonFramework/ProgramSession.h index 3772ae3f76..bafb6787a6 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.h +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include "Common/Cpp/Logging/TaggedLogger.h" @@ -72,7 +73,7 @@ class ProgramSession : public TrackableProgram{ // a test passes its own store. ProgramSession( const ProgramDescriptor& descriptor, - RunRecordStore& run_records = RunRecordStore::instance() + RunRecordStore& run_records = app_run_records() ); // Before the child-most class begins destruction, you must first: @@ -111,19 +112,8 @@ class ProgramSession : public TrackableProgram{ Mutex& program_lock() const{ return m_lock; } -public: - // Run records (RunRecord.h) - - // The run the last accepted start opened, active or finished; "" before - // the first start. - std::string last_run_id() const{ return m_run_recorder.last_run_id(); } - - // The program's progress hook. Events go to the active run's record. - RunRecorder& run_recorder(){ return m_run_recorder; } - - // The logger the session's consoles chain off, so that their lines land - // in the active run's log tail. - Logger& run_logger(){ return m_run_recorder; } + // The program's progress hook writes to this session's active run. + friend void record_run_event(ProgramEnvironment& env, RunEvent event); public: @@ -152,6 +142,10 @@ class ProgramSession : public TrackableProgram{ // settings file stores for it. "" for a session without settings. virtual std::string settings_json() const{ return ""; } + // The logger the session's consoles chain off, so that their lines land + // in the active run's log (RunRecord.h). + Logger& run_logger(){ return m_run_recorder; } + // virtual void restore_defaults(){ return; } public: @@ -186,6 +180,10 @@ class ProgramSession : public TrackableProgram{ // exactly one result, whichever way run_program() ends. void run_program_and_record(); RunResult run_program(); + std::string start_accepted_run(); + bool prepare_run(); + std::optional download_resources(); + void end_run(); std::string abandon_run(std::string message); diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.cpp b/SerialPrograms/Source/CommonFramework/RunRecord.cpp index 4d64f4b30b..204fdfffba 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord.cpp +++ b/SerialPrograms/Source/CommonFramework/RunRecord.cpp @@ -42,12 +42,22 @@ std::string run_outcome_name(RunOutcome outcome){ -RunRecordStore& RunRecordStore::instance(){ - static RunRecordStore store(DEFAULT_CAPACITY, false); +namespace{ +std::atomic& app_run_records_enabled(){ + static std::atomic enabled{false}; + return enabled; +} +} + +RunRecordStore& app_run_records(){ + static RunRecordStore store(RunRecordStore::DEFAULT_CAPACITY, &app_run_records_enabled()); return store; } +void enable_app_run_records(){ + app_run_records_enabled().store(true, std::memory_order_release); +} -RunRecordStore::RunRecordStore(size_t capacity, bool enabled) +RunRecordStore::RunRecordStore(size_t capacity, const std::atomic* enabled) // The app's start time, to the microsecond: two instances started in // the same second still get different IDs. : m_app_token(now_to_filestring()) @@ -56,7 +66,7 @@ RunRecordStore::RunRecordStore(size_t capacity, bool enabled) {} std::string RunRecordStore::begin(std::string program, std::string settings){ - if (!enabled()){ + if (m_enabled != nullptr && !m_enabled->load(std::memory_order_acquire)){ return ""; } std::lock_guard lg(m_lock); diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.h b/SerialPrograms/Source/CommonFramework/RunRecord.h index 294e6ad5d9..3af0b9a240 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord.h +++ b/SerialPrograms/Source/CommonFramework/RunRecord.h @@ -91,20 +91,13 @@ struct RunRecord{ }; -// The program's progress hook, reachable as ProgramEnvironment::run_recorder(). -class RunRecorder{ -public: - virtual ~RunRecorder() = default; - virtual void event(RunEvent event) = 0; -}; - - // The last few runs of this app instance. Thread-safe; every call is short // and copies, so a reader never holds a lock while writing files. // -// Disabled, the store records nothing: begin() returns "" and every call -// about that run is a no-op. The app's instance() starts disabled; whoever -// reads the records (e.g. the lab channel) enables it. +// A store given an `enabled` flag records only while the flag holds: +// otherwise begin() returns "" and every call about that run is a no-op. +// The app's store (app_run_records()) records once enable_app_run_records() +// was called, by whoever reads the records, e.g. the lab channel. // // Run IDs are opaque strings unique to this app instance: a restart cannot // resolve an old ID to a new run. They sort in the order the runs started. @@ -123,18 +116,13 @@ class RunRecordStore{ static constexpr size_t MAX_LOG_TAIL_BYTES = 256 * 1024; static constexpr size_t MAX_LOG_BYTES = 8 * 1024 * 1024; - // The app's store. Disabled until someone enables it. - static RunRecordStore& instance(); - - explicit RunRecordStore(size_t capacity = DEFAULT_CAPACITY, bool enabled = true); + // `enabled`, when given, must outlive the store. + explicit RunRecordStore(size_t capacity = DEFAULT_CAPACITY, const std::atomic* enabled = nullptr); RunRecordStore(const RunRecordStore&) = delete; void operator=(const RunRecordStore&) = delete; - void set_enabled(bool enabled){ m_enabled.store(enabled, std::memory_order_release); } - bool enabled() const{ return m_enabled.load(std::memory_order_acquire); } - // Opens a record for an accepted start and returns its run ID, or "" - // when the store is disabled. The oldest record goes when the store is + // while the store is disabled. The oldest record goes when the store is // full. `settings` is the program's settings as JSON text. std::string begin(std::string program, std::string settings = ""); @@ -163,17 +151,25 @@ class RunRecordStore{ mutable Mutex m_lock; const std::string m_app_token; const size_t m_capacity; - std::atomic m_enabled; + const std::atomic* m_enabled; uint64_t m_counter = 0; std::deque m_records; }; -// One session's end of the store: a RunRecorder for the run active now and -// a Logger that copies what passes through it into that run's log tail. -// The session's loggers chain off `log()`, so a run's tail holds the lines +// The app's store, and the switch that makes it record. Until the switch +// is thrown, no run is recorded, so an app nobody reads records from keeps +// nothing. +RunRecordStore& app_run_records(); +void enable_app_run_records(); + + +// One session's end of the store: the run active now, the events the +// program reports about it (ProgramEnvironment.h, record_run_event()), and +// a Logger that copies what passes through it into that run's log. +// The session's loggers chain off `log()`, so a run's log holds the lines // the program and its consoles logged, and nothing from other sessions. -class SessionRunRecorder : public RunRecorder, public Logger{ +class SessionRunRecorder : public Logger{ public: SessionRunRecorder(RunRecordStore& store, Logger& base_logger); @@ -187,7 +183,8 @@ class SessionRunRecorder : public RunRecorder, public Logger{ // The run the last begin() opened, active or finished; "" before the first. std::string last_run_id() const; - virtual void event(RunEvent event) override; + // Adds to the active run; dropped when none is active. + void event(RunEvent event); virtual void log(const std::string& msg, Color color = Color()) override; private: diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp index 553b1d6489..9db9e640ca 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp @@ -1,620 +1,635 @@ -/* Run Record - Tests - * - * From: https://github.com/PokemonAutomation/ - * - * The store on its own, and the run lifecycle through a ProgramSession - * whose program is a scripted fake: no console, no window. - * - */ - -#include -#include -#include -#include -#include -#include -#include "Common/Cpp/Exceptions.h" -#include "Common/Cpp/TestRunners/UnitTest.h" -#include "CommonFramework/BuildCommit.h" -#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" -#include "CommonFramework/Globals.h" -#include "CommonFramework/Notifications/ProgramInfo.h" -#include "CommonFramework/Panels/PanelSession.h" -#include "CommonFramework/Panels/ProgramDescriptor.h" -#include "CommonFramework/ProgramSession.h" -#include "CommonFramework/Tools/ProgramEnvironment.h" -#include "RunRecord.h" -#include "RunRecord_Tests.h" - -namespace PokemonAutomation{ - -namespace{ - -using namespace std::chrono_literals; - - -std::shared_ptr make_frame(size_t width){ - ImageRGB32 image(width, width); - image.fill(0xff000000); - return std::make_shared(std::move(image)); -} - -// Polls `done` every few milliseconds until it holds or `timeout` passes. -bool wait_until(const std::function& done, Milliseconds timeout = 5000ms){ - const WallClock deadline = current_time() + timeout; - while (!done()){ - if (current_time() > deadline){ - return false; - } - std::this_thread::sleep_for(5ms); - } - return true; -} - - - -UnitTestResult test_store_ids(){ - RunRecordStore store(2); - const std::string a = store.begin("Test:A", R"({"START": 3})"); - const std::string b = store.begin("Test:B"); - if (a.empty() || a == b || !(a < b)){ - return std::format("Run IDs should be distinct and sort in start order: <{}>, <{}>", a, b); - } - if (store.run_ids() != std::vector{a, b}){ - return "run_ids() lists the records oldest first."; - } - const std::optional record = store.get(a); - if (!record || record->program != "Test:A" || record->status != RunStatus::RUNNING || record->result){ - return "A new record is running, with its program and no result."; - } - if (record->start_time == WallClock::min() || record->end_time){ - return "A new record has a start time and no end time."; - } - if (record->version != PROGRAM_VERSION || record->commit != PROGRAM_COMMIT || record->commit.empty()){ - return std::format("A record names the build: version <{}>, commit <{}>.", record->version, record->commit); - } - if (record->settings != R"({"START": 3})" || !store.get(b)->settings.empty()){ - return "A record keeps the settings it was begun with."; - } - const std::string c = store.begin("Test:C"); - if (store.get(a) || store.run_ids() != std::vector{b, c}){ - return "A full store drops its oldest record."; - } - if (store.get("nobody")){ - return "An unknown run has no record."; - } - return true; -} - -UnitTestResult test_store_one_result(){ - RunRecordStore store; - const std::string id = store.begin("Test:A"); - store.stopping(id); - if (store.get(id)->status != RunStatus::STOPPING){ - return "stopping() marks a running run."; - } - if (!store.finish(id, {.outcome = RunOutcome::FAILED, .error = "first"})){ - return "The first finish() takes."; - } - store.stopping(id); - if (store.finish(id, {.outcome = RunOutcome::COMPLETED})){ - return "A second finish() is refused."; - } - const RunRecord record = *store.get(id); - if (record.status != RunStatus::TERMINAL || !record.end_time || !record.result){ - return "A finished record is terminal with an end time and a result."; - } - if (record.result->outcome != RunOutcome::FAILED || record.result->error != "first"){ - return std::format("The first result stands: got {}", run_outcome_name(record.result->outcome)); - } - if (store.finish("nobody", {})){ - return "Finishing an unknown run is refused."; - } - return true; -} - -UnitTestResult test_store_events_and_log(){ - RunRecordStore store; - const std::string id = store.begin("Test:A"); - store.event(id, {.kind = "checkpoint", .checkpoint = "00.01", .text = "ok"}); - store.event("nobody", {.kind = "checkpoint"}); - for (size_t c = 0; c < RunRecordStore::MAX_EVENT_FRAMES + 2; c++){ - store.event(id, {.kind = "step", .step = std::to_string(c), .frame = make_frame(4)}); - } - RunRecord record = *store.get(id); - if (record.events.size() != RunRecordStore::MAX_EVENT_FRAMES + 3){ - return std::format("Expected {} events, got {}.", RunRecordStore::MAX_EVENT_FRAMES + 3, record.events.size()); - } - if (record.events[0].time == WallClock::min() || record.events[0].checkpoint != "00.01"){ - return "An event gets the time it was recorded and keeps its fields."; - } - if (record.events[1].frame || record.events[2].frame || !record.events[3].frame || !record.events.back().frame){ - return "Only the newest MAX_EVENT_FRAMES events keep their frames."; - } - - for (size_t c = record.events.size(); c < RunRecordStore::MAX_EVENTS + 5; c++){ - store.event(id, {.kind = "step"}); - } - record = *store.get(id); - if (record.events.size() != RunRecordStore::MAX_EVENTS || record.dropped_events != 5){ - return std::format("Events past the cap are counted, not kept: {} events, {} dropped.", record.events.size(), record.dropped_events); - } - - // Lines of 1000 bytes, numbered: 300 are more than the tail holds. - const std::string line(995, 'x'); - for (size_t c = 0; c < 300; c++){ - store.log_line(id, std::format("{:04} {}", c, line)); - } - record = *store.get(id); - size_t bytes = 0; - for (const std::string& l : record.log_tail){ - bytes += l.size() + 1; - } - if (record.log_lines != 300 || record.log_bytes != 300 * 1001 || record.dropped_log_lines != 0){ - return std::format("Every line is counted: {} lines, {} bytes.", record.log_lines, record.log_bytes); - } - if (bytes != record.log_tail_bytes || bytes > RunRecordStore::MAX_LOG_TAIL_BYTES || record.log_tail.size() == 300){ - return std::format("The tail keeps at most {} bytes: {} lines, {} bytes.", RunRecordStore::MAX_LOG_TAIL_BYTES, record.log_tail.size(), bytes); - } - const size_t first = record.log_lines - record.log_tail.size(); - if (record.log_tail.front().rfind(std::format("{:04} ", first), 0) != 0 || record.log_tail.back().rfind("0299 ", 0) != 0){ - return "The tail holds the newest lines, from line log_lines - log_tail.size()."; - } - - // Past MAX_LOG_BYTES in all, lines are counted and not kept. - const size_t fit = RunRecordStore::MAX_LOG_BYTES / 1001; - for (size_t c = 300; c < fit + 10; c++){ - store.log_line(id, std::format("{:04} {}", c % 10000, line)); - } - record = *store.get(id); - if (record.log_lines != fit || record.dropped_log_lines != 10 || record.log_bytes > RunRecordStore::MAX_LOG_BYTES){ - return std::format("A run records at most {} bytes of log: {} lines, {} dropped.", RunRecordStore::MAX_LOG_BYTES, record.log_lines, record.dropped_log_lines); - } - return true; -} - -UnitTestResult test_store_disabled(){ - struct CountingLogger : public Logger{ - size_t lines = 0; - virtual void log(const std::string&, Color) override{ lines++; } - }; - RunRecordStore store(4, false); - if (store.enabled() || !store.begin("Test:A").empty() || !store.run_ids().empty()){ - return "A disabled store opens no record."; - } - CountingLogger base; - SessionRunRecorder recorder(store, base); - if (!recorder.begin("Test:A").empty() || !recorder.last_run_id().empty()){ - return "A recorder over a disabled store has no run."; - } - recorder.log("line"); - recorder.event({.kind = "checkpoint"}); - recorder.finish({.outcome = RunOutcome::COMPLETED}); - if (base.lines != 1 || !store.run_ids().empty()){ - return "Lines still reach the base logger; nothing is recorded."; - } - store.set_enabled(true); - const std::string id = recorder.begin("Test:A"); - recorder.log("line"); - if (id.empty() || store.get(id)->log_lines != 1){ - return "Once enabled, the store records."; - } - return true; -} - -UnitTestResult test_store_releases_old_frames(){ - RunRecordStore store; - std::vector ids; - for (size_t c = 0; c < RunRecordStore::RUNS_WITH_FRAMES + 1; c++){ - ids.emplace_back(store.begin("Test:A")); - store.event(ids.back(), {.kind = "step", .frame = make_frame(4)}); - store.finish(ids.back(), {.outcome = RunOutcome::FAILED, .error = "e", .frame = make_frame(4)}); - } - const RunRecord oldest = *store.get(ids.front()); - if (oldest.events[0].frame || oldest.result->frame){ - return "A run older than the newest RUNS_WITH_FRAMES terminal runs keeps no frames."; - } - if (oldest.result->error != "e" || oldest.events.size() != 1){ - return "It keeps the rest of its record."; - } - const RunRecord newest = *store.get(ids.back()); - if (!newest.events[0].frame || !newest.result->frame){ - return "The newest terminal runs keep their frames."; - } - return true; -} - -UnitTestResult test_session_recorder(){ - struct CountingLogger : public Logger{ - size_t lines = 0; - virtual void log(const std::string&, Color) override{ lines++; } - }; - RunRecordStore store; - CountingLogger base; - SessionRunRecorder recorder(store, base); - recorder.log("before"); - recorder.event({.kind = "lost"}); - if (!recorder.last_run_id().empty() || !store.run_ids().empty()){ - return "Nothing is recorded before a run begins."; - } - const std::string id = recorder.begin("Test:A"); - if (recorder.last_run_id() != id){ - return "last_run_id() is the run begin() opened."; - } - recorder.log("during"); - recorder.event({.kind = "checkpoint"}); - recorder.finish({.outcome = RunOutcome::COMPLETED}); - recorder.log("after"); - recorder.event({.kind = "late"}); - recorder.finish({.outcome = RunOutcome::FAILED}); - const RunRecord record = *store.get(id); - if (base.lines != 3){ - return std::format("Every line reaches the base logger: {} of 3.", base.lines); - } - if (record.log_tail != std::deque{"during"} || record.events.size() != 1){ - return "Only lines and events during the run are recorded."; - } - if (record.result->outcome != RunOutcome::COMPLETED || recorder.last_run_id() != id){ - return "The result stands and the run stays known after finish()."; - } - return true; -} - - - -// A program with no console: what it does is scripted per test. -struct FakeScript{ - std::string validity = {}; // check_validity(): "" accepts the start. - bool fail_dispatch = false; // The program thread cannot be started. - std::function start_checks = {}; - std::function body = {}; - std::string settings = {}; // settings_json(). -}; - -class FakeDescriptor : public ProgramDescriptor{ -public: - FakeDescriptor() - : ProgramDescriptor( - "Test:RunRecordFake", "Test", "Run Record Fake", - "", "", PanelQualityStanding::DEVELOPER, false, {} - ) - {} - virtual std::unique_ptr make_panel() const override{ return nullptr; } -}; - -class FakeInstance : public ProgramInstance{ -public: - explicit FakeInstance(std::function start_checks) - : m_start_checks(std::move(start_checks)) - {} - virtual void run_start_program_checks(const ProgramDescriptor&, ProgramEnvironment& env) override{ - if (m_start_checks){ - m_start_checks(env); - } - } -private: - std::function m_start_checks; -}; - -class FakeProgram : public ProgramSession{ -public: - FakeProgram(const FakeDescriptor& descriptor, RunRecordStore& store, FakeScript script) - : ProgramSession(descriptor, store) - , m_script(std::move(script)) - { - m_instance = std::make_unique(m_script.start_checks); - } - virtual ~FakeProgram(){ - internal_stop_program(); - join_program_thread(); - } - - // Whether the body has started running. - bool running() const{ return m_running.load(std::memory_order_acquire); } - - // Waits for the program thread to end, so the run's record is closed. - bool wait_until_stopped(){ - if (!wait_until([this]{ return current_state() == ProgramState::STOPPED; })){ - return false; - } - join_program_thread(); - return true; - } - - virtual std::string check_validity() const override{ return m_script.validity; } - -protected: - virtual std::unique_ptr make_env(const ProgramInfo& program_info) override{ - return std::make_unique(program_info, *this, current_stats_tracker(), historical_stats_tracker()); - } - virtual void internal_run_program(ProgramEnvironment& env) override{ - m_running.store(true, std::memory_order_release); - if (m_script.body){ - m_script.body(*this, env, *m_scope); - } - } - virtual std::string settings_json() const override{ return m_script.settings; } - virtual AsyncTask dispatch_program_thread(std::function body) override{ - if (m_script.fail_dispatch){ - throw std::runtime_error("no thread"); - } - return ProgramSession::dispatch_program_thread(std::move(body)); - } - -private: - FakeScript m_script; - std::atomic m_running{false}; -}; - -// Starts the program, waits for it to end, and returns its record. -UnitTestResult run_to_end(FakeProgram& program, RunRecordStore& store, RunRecord& record){ - const std::string error = program.start_program(); - if (!error.empty()){ - return "The start was refused: " + error; - } - if (!program.wait_until_stopped()){ - return "The program did not stop."; - } - const std::optional found = store.get(program.last_run_id()); - if (!found){ - return "The run has no record."; - } - record = *found; - if (record.status != RunStatus::TERMINAL || !record.result){ - return "The record is not terminal after the program stopped."; - } - return true; -} - -UnitTestResult test_rejection(){ - RunRecordStore store; - FakeDescriptor descriptor; - FakeProgram program(descriptor, store, {.validity = "Pick a save."}); - if (program.start_program() != "Pick a save."){ - return "A start refused by check_validity() returns its message."; - } - if (!store.run_ids().empty() || !program.last_run_id().empty()){ - return "A refused start opens no run."; - } - return true; -} - -UnitTestResult test_immediate_completion(){ - RunRecordStore store; - FakeDescriptor descriptor; - FakeProgram program(descriptor, store, { - .body = [](ProgramSession&, ProgramEnvironment& env, CancellableScope&){ - env.run_recorder().event({.kind = "checkpoint", .checkpoint = "00.01", .step = "3/3", .attempt = 1, .text = "ok", .pose = "home (2, 4) N"}); - }, - .settings = R"({"START": 1})", - }); - RunRecord record; - UnitTestResult result = run_to_end(program, store, record); - if (result.result != UnitTestResult::PASSED){ - return result; - } - if (record.result->outcome != RunOutcome::COMPLETED || record.result->error || record.result->frame){ - return "A program that returns completed its run, with no error and no frame."; - } - if (record.program != "Test:RunRecordFake" || record.start_time > *record.end_time){ - return "The record names the program and its times."; - } - if (record.settings != R"({"START": 1})" || record.commit != PROGRAM_COMMIT){ - return "The record has the session's settings and the build's commit."; - } - if (record.events.size() != 1 || record.events[0].checkpoint != "00.01" || record.events[0].attempt != size_t(1) || record.events[0].pose != "home (2, 4) N"){ - return std::format("The program's event is in the record: {} events.", record.events.size()); - } - bool stopped_logged = false; - for (const std::string& line : record.log_tail){ - stopped_logged |= line.find("Program State: STOPPED") != std::string::npos; - } - if (!stopped_logged){ - return "The run's log tail runs through the session's last state change."; - } - return true; -} - -UnitTestResult test_setup_failure(){ - RunRecordStore store; - FakeDescriptor descriptor; - FakeProgram program(descriptor, store, { - .start_checks = [](ProgramEnvironment& env){ - throw UserSetupError(env.logger(), "The controller is not ready."); - }, - }); - RunRecord record; - UnitTestResult result = run_to_end(program, store, record); - if (result.result != UnitTestResult::PASSED){ - return result; - } - if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The controller is not ready."){ - return "A setup error fails the run with its message."; - } - if (record.events.size() != 1 || record.events[0].kind != "error" || record.events[0].text != "The controller is not ready."){ - return "The reported error is an \"error\" event."; - } - return true; -} - -UnitTestResult test_failure_with_frame(){ - RunRecordStore store; - FakeDescriptor descriptor; - FakeProgram program(descriptor, store, { - .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ - ImageRGB32 screen(8, 8); - screen.fill(0xffff0000); - throw OperationFailedExceptionWithScreenshot( - ErrorReportMode::NO_ERROR_REPORT, "The sign's box did not open.", nullptr, std::move(screen) - ); - }, - }); - RunRecord record; - UnitTestResult result = run_to_end(program, store, record); - if (result.result != UnitTestResult::PASSED){ - return result; - } - if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The sign's box did not open."){ - return "The exception's message is the primary error."; - } - if (!record.result->frame || record.result->frame->width() != 8){ - return "The exception's frame is kept with the result."; - } - return true; -} - -UnitTestResult test_cancellation(){ - RunRecordStore store; - FakeDescriptor descriptor; - FakeProgram program(descriptor, store, { - .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope& scope){ - while (true){ - scope.throw_if_cancelled(); - std::this_thread::sleep_for(5ms); - } - }, - }); - if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ - return "The program should start and run."; - } - const std::string id = program.last_run_id(); - if (!program.stop_program().empty() || !program.wait_until_stopped()){ - return "The program should stop."; - } - const RunRecord record = *store.get(id); - if (record.status != RunStatus::TERMINAL || record.result->outcome != RunOutcome::CANCELLED || record.result->error){ - return "A stopped program's run is cancelled, with no error."; - } - return true; -} - -// The body waits for the stop request and then ends its own way. -UnitTestResult run_racing_a_stop(const std::function& ending, RunOutcome expected, const char* what){ - RunRecordStore store; - FakeDescriptor descriptor; - FakeProgram program(descriptor, store, { - .body = [&](ProgramSession& self, ProgramEnvironment&, CancellableScope&){ - wait_until([&]{ return self.current_state() == ProgramState::STOPPING; }); - ending(); - }, - }); - if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ - return "The program should start and run."; - } - const std::string id = program.last_run_id(); - program.stop_program(); - if (!program.wait_until_stopped()){ - return "The program should stop."; - } - const RunRecord record = *store.get(id); - if (record.result->outcome != expected){ - return std::format("{}: expected {}, got {}.", what, run_outcome_name(expected), run_outcome_name(record.result->outcome)); - } - return true; -} -UnitTestResult test_late_stop(){ - UnitTestResult result = run_racing_a_stop( - []{ throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "Failed anyway."); }, - RunOutcome::FAILED, "A failure observed after the stop request stays a failure" - ); - if (result.result != UnitTestResult::PASSED){ - return result; - } - return run_racing_a_stop( - []{}, RunOutcome::COMPLETED, "A normal return racing the stop request stays completed" - ); -} - -UnitTestResult test_dispatch_failure(){ - RunRecordStore store; - FakeDescriptor descriptor; - FakeProgram program(descriptor, store, {.fail_dispatch = true}); - const std::string error = program.start_program(); - if (error.find("could not be started") == std::string::npos){ - return "A start whose thread cannot be dispatched returns an error: <" + error + ">"; - } - const std::optional record = store.get(program.last_run_id()); - if (!record || record->status != RunStatus::TERMINAL || record->result->outcome != RunOutcome::FAILED){ - return "The accepted run is closed as failed."; - } - if (record->result->error != error){ - return "The run's error is the start's error."; - } - if (program.current_state() != ProgramState::STOPPED){ - return "The session is stopped again, so the next start can try."; - } - return true; -} - -UnitTestResult test_second_run_preserves_first(){ - RunRecordStore store; - FakeDescriptor descriptor; - FakeProgram failing(descriptor, store, { - .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ - throw OperationFailedExceptionWithScreenshot( - ErrorReportMode::NO_ERROR_REPORT, "first", nullptr, ImageRGB32(8, 8) - ); - }, - }); - RunRecord first; - UnitTestResult result = run_to_end(failing, store, first); - if (result.result != UnitTestResult::PASSED){ - return result; - } - FakeProgram completing(descriptor, store, {}); - RunRecord second; - result = run_to_end(completing, store, second); - if (result.result != UnitTestResult::PASSED){ - return result; - } - if (first.run_id == second.run_id || !(first.run_id < second.run_id)){ - return "Each run has its own ID, sorting after the earlier one."; - } - const std::optional kept = store.get(first.run_id); - if (!kept || kept->result->outcome != RunOutcome::FAILED || kept->result->error != "first" || !kept->result->frame){ - return "The first run's result and frame are still there after the second run."; - } - if (second.result->outcome != RunOutcome::COMPLETED || store.run_ids() != std::vector{first.run_id, second.run_id}){ - return "The second run has its own, completed, record."; - } - return true; -} - - -class Test_RunRecord : public UnitTest{ -public: - Test_RunRecord(std::string name, std::function body) - : UnitTest("CommonFramework::RunRecord - " + std::move(name)) - , m_body(std::move(body)) - {} - virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - return m_body(); - } - -private: - std::function m_body; -}; - -} - - -void add_tests_RunRecord(UnitTestDatabase& database){ - database.add("store: ids and capacity", test_store_ids); - database.add("store: one result per run", test_store_one_result); - database.add("store: events, frames and log tail", test_store_events_and_log); - database.add("store: old runs release frames", test_store_releases_old_frames); - database.add("store: disabled", test_store_disabled); - database.add("session recorder", test_session_recorder); - database.add("lifecycle: rejected start", test_rejection); - database.add("lifecycle: immediate completion", test_immediate_completion); - database.add("lifecycle: setup failure", test_setup_failure); - database.add("lifecycle: failure with frame", test_failure_with_frame); - database.add("lifecycle: cancellation", test_cancellation); - database.add("lifecycle: late stop", test_late_stop); - database.add("lifecycle: dispatch failure", test_dispatch_failure); - database.add("lifecycle: second run preserves the first", test_second_run_preserves_first); -} - - - -} +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The store on its own, and the run lifecycle through a ProgramSession + * whose program is a scripted fake: no console, no window. + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/BuildCommit.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/Notifications/ProgramInfo.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/ProgramSession.h" +#include "CommonFramework/Tools/ProgramEnvironment.h" +#include "RunRecord.h" +#include "RunRecord_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + +using namespace std::chrono_literals; + + +std::shared_ptr make_frame(size_t width){ + ImageRGB32 image(width, width); + image.fill(0xff000000); + return std::make_shared(std::move(image)); +} + +// Polls `done` every few milliseconds until it holds or `timeout` passes. +bool wait_until(const std::function& done, Milliseconds timeout = 5000ms){ + const WallClock deadline = current_time() + timeout; + while (!done()){ + if (current_time() > deadline){ + return false; + } + std::this_thread::sleep_for(5ms); + } + return true; +} + + + +// The run the store opened last, "" when none. +std::string newest_run(const RunRecordStore& store){ + const std::vector ids = store.run_ids(); + return ids.empty() ? "" : ids.back(); +} + +UnitTestResult test_store_ids(){ + RunRecordStore store(2); + const std::string a = store.begin("Test:A", R"({"START": 3})"); + const std::string b = store.begin("Test:B"); + if (a.empty() || a == b || !(a < b)){ + return std::format("Run IDs should be distinct and sort in start order: <{}>, <{}>", a, b); + } + if (store.run_ids() != std::vector{a, b}){ + return "run_ids() lists the records oldest first."; + } + const std::optional record = store.get(a); + if (!record || record->program != "Test:A" || record->status != RunStatus::RUNNING || record->result){ + return "A new record is running, with its program and no result."; + } + if (record->start_time == WallClock::min() || record->end_time){ + return "A new record has a start time and no end time."; + } + if (record->version != PROGRAM_VERSION || record->commit != PROGRAM_COMMIT || record->commit.empty()){ + return std::format("A record names the build: version <{}>, commit <{}>.", record->version, record->commit); + } + if (record->settings != R"({"START": 3})" || !store.get(b)->settings.empty()){ + return "A record keeps the settings it was begun with."; + } + const std::string c = store.begin("Test:C"); + if (store.get(a) || store.run_ids() != std::vector{b, c}){ + return "A full store drops its oldest record."; + } + if (store.get("nobody")){ + return "An unknown run has no record."; + } + return true; +} + +UnitTestResult test_store_one_result(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.stopping(id); + if (store.get(id)->status != RunStatus::STOPPING){ + return "stopping() marks a running run."; + } + if (!store.finish(id, {.outcome = RunOutcome::FAILED, .error = "first"})){ + return "The first finish() takes."; + } + store.stopping(id); + if (store.finish(id, {.outcome = RunOutcome::COMPLETED})){ + return "A second finish() is refused."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || !record.end_time || !record.result){ + return "A finished record is terminal with an end time and a result."; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "first"){ + return std::format("The first result stands: got {}", run_outcome_name(record.result->outcome)); + } + if (store.finish("nobody", {})){ + return "Finishing an unknown run is refused."; + } + return true; +} + +UnitTestResult test_store_events(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.event(id, {.kind = "checkpoint", .checkpoint = "00.01", .text = "ok"}); + store.event("nobody", {.kind = "checkpoint"}); + for (size_t c = 0; c < RunRecordStore::MAX_EVENT_FRAMES + 2; c++){ + store.event(id, {.kind = "step", .step = std::to_string(c), .frame = make_frame(4)}); + } + RunRecord record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENT_FRAMES + 3){ + return std::format("Expected {} events, got {}.", RunRecordStore::MAX_EVENT_FRAMES + 3, record.events.size()); + } + if (record.events[0].time == WallClock::min() || record.events[0].checkpoint != "00.01"){ + return "An event gets the time it was recorded and keeps its fields."; + } + if (record.events[1].frame || record.events[2].frame || !record.events[3].frame || !record.events.back().frame){ + return "Only the newest MAX_EVENT_FRAMES events keep their frames."; + } + + for (size_t c = record.events.size(); c < RunRecordStore::MAX_EVENTS + 5; c++){ + store.event(id, {.kind = "step"}); + } + record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENTS || record.dropped_events != 5){ + return std::format("Events past the cap are counted, not kept: {} events, {} dropped.", record.events.size(), record.dropped_events); + } + return true; +} + +UnitTestResult test_store_log(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + RunRecord record; + + // Lines of 1000 bytes, numbered: 300 are more than the tail holds. + const std::string line(995, 'x'); + for (size_t c = 0; c < 300; c++){ + store.log_line(id, std::format("{:04} {}", c, line)); + } + record = *store.get(id); + size_t bytes = 0; + for (const std::string& l : record.log_tail){ + bytes += l.size() + 1; + } + if (record.log_lines != 300 || record.log_bytes != 300 * 1001 || record.dropped_log_lines != 0){ + return std::format("Every line is counted: {} lines, {} bytes.", record.log_lines, record.log_bytes); + } + if (bytes != record.log_tail_bytes || bytes > RunRecordStore::MAX_LOG_TAIL_BYTES || record.log_tail.size() == 300){ + return std::format("The tail keeps at most {} bytes: {} lines, {} bytes.", RunRecordStore::MAX_LOG_TAIL_BYTES, record.log_tail.size(), bytes); + } + const size_t first = record.log_lines - record.log_tail.size(); + if (record.log_tail.front().rfind(std::format("{:04} ", first), 0) != 0 || record.log_tail.back().rfind("0299 ", 0) != 0){ + return "The tail holds the newest lines, from line log_lines - log_tail.size()."; + } + + // Past MAX_LOG_BYTES in all, lines are counted and not kept. + const size_t fit = RunRecordStore::MAX_LOG_BYTES / 1001; + for (size_t c = 300; c < fit + 10; c++){ + store.log_line(id, std::format("{:04} {}", c % 10000, line)); + } + record = *store.get(id); + if (record.log_lines != fit || record.dropped_log_lines != 10 || record.log_bytes > RunRecordStore::MAX_LOG_BYTES){ + return std::format("A run records at most {} bytes of log: {} lines, {} dropped.", RunRecordStore::MAX_LOG_BYTES, record.log_lines, record.dropped_log_lines); + } + return true; +} + +UnitTestResult test_store_disabled(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + std::atomic enabled{false}; + RunRecordStore store(4, &enabled); + if (!store.begin("Test:A").empty() || !store.run_ids().empty()){ + return "A disabled store opens no record."; + } + CountingLogger base; + SessionRunRecorder recorder(store, base); + if (!recorder.begin("Test:A").empty() || !recorder.last_run_id().empty()){ + return "A recorder over a disabled store has no run."; + } + recorder.log("line"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + if (base.lines != 1 || !store.run_ids().empty()){ + return "Lines still reach the base logger; nothing is recorded."; + } + enabled.store(true); + const std::string id = recorder.begin("Test:A"); + recorder.log("line"); + if (id.empty() || store.get(id)->log_lines != 1){ + return "Once enabled, the store records."; + } + return true; +} + +UnitTestResult test_store_releases_old_frames(){ + RunRecordStore store; + std::vector ids; + for (size_t c = 0; c < RunRecordStore::RUNS_WITH_FRAMES + 1; c++){ + ids.emplace_back(store.begin("Test:A")); + store.event(ids.back(), {.kind = "step", .frame = make_frame(4)}); + store.finish(ids.back(), {.outcome = RunOutcome::FAILED, .error = "e", .frame = make_frame(4)}); + } + const RunRecord oldest = *store.get(ids.front()); + if (oldest.events[0].frame || oldest.result->frame){ + return "A run older than the newest RUNS_WITH_FRAMES terminal runs keeps no frames."; + } + if (oldest.result->error != "e" || oldest.events.size() != 1){ + return "It keeps the rest of its record."; + } + const RunRecord newest = *store.get(ids.back()); + if (!newest.events[0].frame || !newest.result->frame){ + return "The newest terminal runs keep their frames."; + } + return true; +} + +UnitTestResult test_session_recorder(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + RunRecordStore store; + CountingLogger base; + SessionRunRecorder recorder(store, base); + recorder.log("before"); + recorder.event({.kind = "lost"}); + if (!recorder.last_run_id().empty() || !store.run_ids().empty()){ + return "Nothing is recorded before a run begins."; + } + const std::string id = recorder.begin("Test:A"); + if (recorder.last_run_id() != id){ + return "last_run_id() is the run begin() opened."; + } + recorder.log("during"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + recorder.log("after"); + recorder.event({.kind = "late"}); + recorder.finish({.outcome = RunOutcome::FAILED}); + const RunRecord record = *store.get(id); + if (base.lines != 3){ + return std::format("Every line reaches the base logger: {} of 3.", base.lines); + } + if (record.log_tail != std::deque{"during"} || record.events.size() != 1){ + return "Only lines and events during the run are recorded."; + } + if (record.result->outcome != RunOutcome::COMPLETED || recorder.last_run_id() != id){ + return "The result stands and the run stays known after finish()."; + } + return true; +} + + + +// A program with no console: what it does is scripted per test. +struct FakeScript{ + std::string validity = {}; // check_validity(): "" accepts the start. + bool fail_dispatch = false; // The program thread cannot be started. + std::function start_checks = {}; + std::function body = {}; + std::string settings = {}; // settings_json(). +}; + +class FakeDescriptor : public ProgramDescriptor{ +public: + FakeDescriptor() + : ProgramDescriptor( + "Test:RunRecordFake", "Test", "Run Record Fake", + "", "", PanelQualityStanding::DEVELOPER, false, {} + ) + {} + virtual std::unique_ptr make_panel() const override{ return nullptr; } +}; + +class FakeInstance : public ProgramInstance{ +public: + explicit FakeInstance(std::function start_checks) + : m_start_checks(std::move(start_checks)) + {} + virtual void run_start_program_checks(const ProgramDescriptor&, ProgramEnvironment& env) override{ + if (m_start_checks){ + m_start_checks(env); + } + } +private: + std::function m_start_checks; +}; + +class FakeProgram : public ProgramSession{ +public: + FakeProgram(const FakeDescriptor& descriptor, RunRecordStore& store, FakeScript script) + : ProgramSession(descriptor, store) + , m_script(std::move(script)) + { + m_instance = std::make_unique(m_script.start_checks); + } + virtual ~FakeProgram(){ + internal_stop_program(); + join_program_thread(); + } + + // Whether the body has started running. + bool running() const{ return m_running.load(std::memory_order_acquire); } + + // Waits for the program thread to end, so the run's record is closed. + bool wait_until_stopped(){ + if (!wait_until([this]{ return current_state() == ProgramState::STOPPED; })){ + return false; + } + join_program_thread(); + return true; + } + + virtual std::string check_validity() const override{ return m_script.validity; } + +protected: + virtual std::unique_ptr make_env(const ProgramInfo& program_info) override{ + return std::make_unique(program_info, *this, current_stats_tracker(), historical_stats_tracker()); + } + virtual void internal_run_program(ProgramEnvironment& env) override{ + m_running.store(true, std::memory_order_release); + if (m_script.body){ + m_script.body(*this, env, *m_scope); + } + } + virtual std::string settings_json() const override{ return m_script.settings; } + virtual AsyncTask dispatch_program_thread(std::function body) override{ + if (m_script.fail_dispatch){ + throw std::runtime_error("no thread"); + } + return ProgramSession::dispatch_program_thread(std::move(body)); + } + +private: + FakeScript m_script; + std::atomic m_running{false}; +}; + +// Starts the program, waits for it to end, and returns its record. +UnitTestResult run_to_end(FakeProgram& program, RunRecordStore& store, RunRecord& record){ + const std::string error = program.start_program(); + if (!error.empty()){ + return "The start was refused: " + error; + } + if (!program.wait_until_stopped()){ + return "The program did not stop."; + } + const std::optional found = store.get(newest_run(store)); + if (!found){ + return "The run has no record."; + } + record = *found; + if (record.status != RunStatus::TERMINAL || !record.result){ + return "The record is not terminal after the program stopped."; + } + return true; +} + +UnitTestResult test_rejection(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.validity = "Pick a save."}); + if (program.start_program() != "Pick a save."){ + return "A start refused by check_validity() returns its message."; + } + if (!store.run_ids().empty() || !newest_run(store).empty()){ + return "A refused start opens no run."; + } + return true; +} + +UnitTestResult test_immediate_completion(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment& env, CancellableScope&){ + record_run_event(env, {.kind = "checkpoint", .checkpoint = "00.01", .step = "3/3", .attempt = 1, .text = "ok", .pose = "home (2, 4) N"}); + }, + .settings = R"({"START": 1})", + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::COMPLETED || record.result->error || record.result->frame){ + return "A program that returns completed its run, with no error and no frame."; + } + if (record.program != "Test:RunRecordFake" || record.start_time > *record.end_time){ + return "The record names the program and its times."; + } + if (record.settings != R"({"START": 1})" || record.commit != PROGRAM_COMMIT){ + return "The record has the session's settings and the build's commit."; + } + if (record.events.size() != 1 || record.events[0].checkpoint != "00.01" || record.events[0].attempt != size_t(1) || record.events[0].pose != "home (2, 4) N"){ + return std::format("The program's event is in the record: {} events.", record.events.size()); + } + bool stopped_logged = false; + for (const std::string& line : record.log_tail){ + stopped_logged |= line.find("Program State: STOPPED") != std::string::npos; + } + if (!stopped_logged){ + return "The run's log tail runs through the session's last state change."; + } + return true; +} + +UnitTestResult test_setup_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .start_checks = [](ProgramEnvironment& env){ + throw UserSetupError(env.logger(), "The controller is not ready."); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The controller is not ready."){ + return "A setup error fails the run with its message."; + } + if (record.events.size() != 1 || record.events[0].kind != "error" || record.events[0].text != "The controller is not ready."){ + return "The reported error is an \"error\" event."; + } + return true; +} + +UnitTestResult test_failure_with_frame(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + ImageRGB32 screen(8, 8); + screen.fill(0xffff0000); + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "The sign's box did not open.", nullptr, std::move(screen) + ); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The sign's box did not open."){ + return "The exception's message is the primary error."; + } + if (!record.result->frame || record.result->frame->width() != 8){ + return "The exception's frame is kept with the result."; + } + return true; +} + +UnitTestResult test_cancellation(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope& scope){ + while (true){ + scope.throw_if_cancelled(); + std::this_thread::sleep_for(5ms); + } + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = newest_run(store); + if (!program.stop_program().empty() || !program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || record.result->outcome != RunOutcome::CANCELLED || record.result->error){ + return "A stopped program's run is cancelled, with no error."; + } + return true; +} + +// The body waits for the stop request and then ends its own way. +UnitTestResult run_racing_a_stop(const std::function& ending, RunOutcome expected, const char* what){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [&](ProgramSession& self, ProgramEnvironment&, CancellableScope&){ + wait_until([&]{ return self.current_state() == ProgramState::STOPPING; }); + ending(); + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = newest_run(store); + program.stop_program(); + if (!program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.result->outcome != expected){ + return std::format("{}: expected {}, got {}.", what, run_outcome_name(expected), run_outcome_name(record.result->outcome)); + } + return true; +} +UnitTestResult test_late_stop(){ + UnitTestResult result = run_racing_a_stop( + []{ throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "Failed anyway."); }, + RunOutcome::FAILED, "A failure observed after the stop request stays a failure" + ); + if (result.result != UnitTestResult::PASSED){ + return result; + } + return run_racing_a_stop( + []{}, RunOutcome::COMPLETED, "A normal return racing the stop request stays completed" + ); +} + +UnitTestResult test_dispatch_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.fail_dispatch = true}); + const std::string error = program.start_program(); + if (error.find("could not be started") == std::string::npos){ + return "A start whose thread cannot be dispatched returns an error: <" + error + ">"; + } + const std::optional record = store.get(newest_run(store)); + if (!record || record->status != RunStatus::TERMINAL || record->result->outcome != RunOutcome::FAILED){ + return "The accepted run is closed as failed."; + } + if (record->result->error != error){ + return "The run's error is the start's error."; + } + if (program.current_state() != ProgramState::STOPPED){ + return "The session is stopped again, so the next start can try."; + } + return true; +} + +UnitTestResult test_second_run_preserves_first(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram failing(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "first", nullptr, ImageRGB32(8, 8) + ); + }, + }); + RunRecord first; + UnitTestResult result = run_to_end(failing, store, first); + if (result.result != UnitTestResult::PASSED){ + return result; + } + FakeProgram completing(descriptor, store, {}); + RunRecord second; + result = run_to_end(completing, store, second); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (first.run_id == second.run_id || !(first.run_id < second.run_id)){ + return "Each run has its own ID, sorting after the earlier one."; + } + const std::optional kept = store.get(first.run_id); + if (!kept || kept->result->outcome != RunOutcome::FAILED || kept->result->error != "first" || !kept->result->frame){ + return "The first run's result and frame are still there after the second run."; + } + if (second.result->outcome != RunOutcome::COMPLETED || store.run_ids() != std::vector{first.run_id, second.run_id}){ + return "The second run has its own, completed, record."; + } + return true; +} + + +class Test_RunRecord : public UnitTest{ +public: + Test_RunRecord(std::string name, std::function body) + : UnitTest("CommonFramework::RunRecord - " + std::move(name)) + , m_body(std::move(body)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_body(); + } + +private: + std::function m_body; +}; + +} + + +void add_tests_RunRecord(UnitTestDatabase& database){ + database.add("store: ids and capacity", test_store_ids); + database.add("store: one result per run", test_store_one_result); + database.add("store: events and frames", test_store_events); + database.add("store: numbered log and its cap", test_store_log); + database.add("store: old runs release frames", test_store_releases_old_frames); + database.add("store: disabled", test_store_disabled); + database.add("session recorder", test_session_recorder); + database.add("lifecycle: rejected start", test_rejection); + database.add("lifecycle: immediate completion", test_immediate_completion); + database.add("lifecycle: setup failure", test_setup_failure); + database.add("lifecycle: failure with frame", test_failure_with_frame); + database.add("lifecycle: cancellation", test_cancellation); + database.add("lifecycle: late stop", test_late_stop); + database.add("lifecycle: dispatch failure", test_dispatch_failure); + database.add("lifecycle: second run preserves the first", test_second_run_preserves_first); +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp index 5814e7aaab..44c8958ff2 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp @@ -33,8 +33,8 @@ const ProgramInfo& ProgramEnvironment::program_info() const{ return m_program_info; } -RunRecorder& ProgramEnvironment::run_recorder(){ - return m_session.run_recorder(); +void record_run_event(ProgramEnvironment& env, RunEvent event){ + env.m_session.m_run_recorder.event(std::move(event)); } diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h index 45068ec33c..6dc5709dd6 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h @@ -43,10 +43,7 @@ class ProgramEnvironment{ virtual void log_to_ui(const std::string& msg, Color color = Color()){} -public: - // Progress reporting (RunRecord.h): what the program says about its - // checkpoints, steps and attempts goes to this run's record. - RunRecorder& run_recorder(); + friend void record_run_event(ProgramEnvironment& env, RunEvent event); public: @@ -74,6 +71,13 @@ class ProgramEnvironment{ +// Progress reporting (RunRecord.h): what the program says about its +// checkpoints, steps and attempts goes to this run's record, if the app +// records runs. +void record_run_event(ProgramEnvironment& env, RunEvent event); + + + // Templates template diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp index cb14619341..c5dc3bd0a7 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp @@ -79,10 +79,7 @@ ConsoleSystemSession::ConsoleSystemSession( update_status(); m_history.set_audio_session(m_audio); - m_history.start( - m_audio.input_choice(), - m_video.current_source() != nullptr - ); + m_history.start(m_audio.input_choice(), m_video.current_source() != nullptr); try{ m_overlay.add_stat(m_memory_usage->m_system); From a7bca06d3a2a10b3336ba37206fc61c1f6a764af Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 15:36:29 -0800 Subject: [PATCH 4/4] Run records: name the executable, leave the git commit to the lab Per the decision on the design: main takes no CMake change. A record names the app's version and the path of its executable (a lab build's folder names the build); the git commit (cmake/BuildCommit.cmake) moves to the lab half, #116. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- SerialPrograms/CMakeLists.txt | 18 ---------- .../Source/CommonFramework/BuildCommit.h | 24 ------------- .../Source/CommonFramework/RunRecord.cpp | 6 ++-- .../Source/CommonFramework/RunRecord.h | 4 +-- .../CommonFramework/RunRecord_Tests.cpp | 10 +++--- SerialPrograms/cmake/BuildCommit.cmake | 34 ------------------- SerialPrograms/cmake/SourceFiles.cmake | 1 - 7 files changed, 11 insertions(+), 86 deletions(-) delete mode 100644 SerialPrograms/Source/CommonFramework/BuildCommit.h delete mode 100644 SerialPrograms/cmake/BuildCommit.cmake diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index b0a02a7243..35bb597e32 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -193,24 +193,6 @@ include(cmake/CoreLib.cmake) # Create the GUI library add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES}) -# PROGRAM_COMMIT (Source/CommonFramework/BuildCommit.h): the git commit the app is -# built from. BuildCommit.cpp is written at configure time and checked on every -# build; it changes, and recompiles, only when the commit does. -set(PA_BUILD_COMMIT_CPP "${CMAKE_CURRENT_BINARY_DIR}/generated/BuildCommit.cpp") -set(PA_BUILD_COMMIT_COMMAND - "${CMAKE_COMMAND}" "-DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}" "-DOUTPUT=${PA_BUILD_COMMIT_CPP}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/BuildCommit.cmake" -) -execute_process(COMMAND ${PA_BUILD_COMMIT_COMMAND}) -add_custom_target(BuildCommit - COMMAND ${PA_BUILD_COMMIT_COMMAND} - BYPRODUCTS "${PA_BUILD_COMMIT_CPP}" - COMMENT "Checking the git commit for PROGRAM_COMMIT" - VERBATIM -) -target_sources(SerialProgramsLib PRIVATE "${PA_BUILD_COMMIT_CPP}") -add_dependencies(SerialProgramsLib BuildCommit) - # Create binary target if (APPLE) diff --git a/SerialPrograms/Source/CommonFramework/BuildCommit.h b/SerialPrograms/Source/CommonFramework/BuildCommit.h deleted file mode 100644 index 3e0568cd4f..0000000000 --- a/SerialPrograms/Source/CommonFramework/BuildCommit.h +++ /dev/null @@ -1,24 +0,0 @@ -/* Build Commit - * - * From: https://github.com/PokemonAutomation/ - * - * The git commit the app was built from. The build writes its definition - * (cmake/BuildCommit.cmake) into BuildCommit.cpp in the build folder, and - * rewrites it only when the commit changes, so a new commit recompiles - * that one file. - * - */ - -#ifndef PokemonAutomation_BuildCommit_H -#define PokemonAutomation_BuildCommit_H - -namespace PokemonAutomation{ - - -// "git describe" of the source tree: the commit's hash, with "-dirty" when -// tracked files had changes, or "unknown" when git could not tell. -extern const char* const PROGRAM_COMMIT; - - -} -#endif diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.cpp b/SerialPrograms/Source/CommonFramework/RunRecord.cpp index 204fdfffba..e6feeef259 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord.cpp +++ b/SerialPrograms/Source/CommonFramework/RunRecord.cpp @@ -7,9 +7,9 @@ #include #include #include +#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PrettyPrint.h" -#include "CommonFramework/BuildCommit.h" #include "CommonFramework/Globals.h" #include "RunRecord.h" @@ -77,7 +77,9 @@ std::string RunRecordStore::begin(std::string program, std::string settings){ record.run_id = std::format("{}-{:03}", m_app_token, ++m_counter); record.program = std::move(program); record.version = PROGRAM_VERSION; - record.commit = PROGRAM_COMMIT; + record.executable = QCoreApplication::instance() == nullptr + ? "" + : QCoreApplication::applicationFilePath().toStdString(); record.settings = std::move(settings); record.start_time = current_time(); return record.run_id; diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.h b/SerialPrograms/Source/CommonFramework/RunRecord.h index 3af0b9a240..c6f8c082f2 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord.h +++ b/SerialPrograms/Source/CommonFramework/RunRecord.h @@ -5,7 +5,7 @@ * What one run of a program did, as the framework saw it: when it started, * how it ended (exactly one outcome per accepted run), the error and the * frame it died with, the events the program reported on the way, and the - * lines of its log, and the build and settings it ran with. + * lines of its log, and the app and settings it ran with. * ProgramSession writes these at its execution boundary; RunRecordStore * keeps the last few for whoever asks, such as a tool that writes them to * files. The store records nothing until it is enabled. Nothing here @@ -70,7 +70,7 @@ struct RunRecord{ std::string run_id = {}; std::string program = {}; // The program's identifier, ":". std::string version = {}; // PROGRAM_VERSION of the app that ran it. - std::string commit = {}; // PROGRAM_COMMIT (BuildCommit.h): the git commit it was built from. + std::string executable = {}; // The path of the app's executable, which names a lab build. std::string settings = {}; // The program's settings as JSON text, as the settings file stores them. WallClock start_time = WallClock::min(); std::optional end_time = {}; diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp index 9db9e640ca..4d25c45fb1 100644 --- a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp @@ -13,9 +13,9 @@ #include #include #include +#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/TestRunners/UnitTest.h" -#include "CommonFramework/BuildCommit.h" #include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" #include "CommonFramework/Globals.h" #include "CommonFramework/Notifications/ProgramInfo.h" @@ -76,8 +76,8 @@ UnitTestResult test_store_ids(){ if (record->start_time == WallClock::min() || record->end_time){ return "A new record has a start time and no end time."; } - if (record->version != PROGRAM_VERSION || record->commit != PROGRAM_COMMIT || record->commit.empty()){ - return std::format("A record names the build: version <{}>, commit <{}>.", record->version, record->commit); + if (record->version != PROGRAM_VERSION || (QCoreApplication::instance() != nullptr && record->executable.empty())){ + return std::format("A record names the app: version <{}>, executable <{}>.", record->version, record->executable); } if (record->settings != R"({"START": 3})" || !store.get(b)->settings.empty()){ return "A record keeps the settings it was begun with."; @@ -413,8 +413,8 @@ UnitTestResult test_immediate_completion(){ if (record.program != "Test:RunRecordFake" || record.start_time > *record.end_time){ return "The record names the program and its times."; } - if (record.settings != R"({"START": 1})" || record.commit != PROGRAM_COMMIT){ - return "The record has the session's settings and the build's commit."; + if (record.settings != R"({"START": 1})" || record.version != PROGRAM_VERSION){ + return "The record has the session's settings and the app's version."; } if (record.events.size() != 1 || record.events[0].checkpoint != "00.01" || record.events[0].attempt != size_t(1) || record.events[0].pose != "home (2, 4) N"){ return std::format("The program's event is in the record: {} events.", record.events.size()); diff --git a/SerialPrograms/cmake/BuildCommit.cmake b/SerialPrograms/cmake/BuildCommit.cmake deleted file mode 100644 index 3c6579f609..0000000000 --- a/SerialPrograms/cmake/BuildCommit.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# Writes BuildCommit.cpp, the definition of PROGRAM_COMMIT -# (Source/CommonFramework/BuildCommit.h): the git commit the app is built from. -# -# Run as a script on every build: -# cmake -DSOURCE_DIR= -DOUTPUT=<.../BuildCommit.cpp> -P BuildCommit.cmake -# The file is rewritten only when its text changes, so an unchanged commit -# rebuilds nothing. Without git, or outside a repository, the commit is "unknown". - -set(PA_COMMIT "unknown") -find_package(Git QUIET) -if(GIT_FOUND) - # --match with a pattern no tag has: always the hash, never a tag name. - execute_process( - COMMAND "${GIT_EXECUTABLE}" describe --always --dirty --abbrev=12 --match "no-tag-matches-this" - WORKING_DIRECTORY "${SOURCE_DIR}" - OUTPUT_VARIABLE PA_DESCRIBE - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE PA_RESULT - ERROR_QUIET - ) - if(PA_RESULT EQUAL 0 AND NOT PA_DESCRIBE STREQUAL "") - set(PA_COMMIT "${PA_DESCRIBE}") - endif() -endif() - -set(PA_TEXT "// Generated by cmake/BuildCommit.cmake. Do not edit.\n\nnamespace PokemonAutomation{\n extern const char* const PROGRAM_COMMIT;\n const char* const PROGRAM_COMMIT = \"${PA_COMMIT}\";\n}\n") -if(EXISTS "${OUTPUT}") - file(READ "${OUTPUT}" PA_OLD_TEXT) -else() - set(PA_OLD_TEXT "") -endif() -if(NOT PA_OLD_TEXT STREQUAL PA_TEXT) - file(WRITE "${OUTPUT}" "${PA_TEXT}") -endif() diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index fba67495f0..48ddcd5849 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -565,7 +565,6 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/AudioPipeline/UI/AudioDisplayWidget.h Source/CommonFramework/AudioPipeline/UI/AudioSelectorWidget.cpp Source/CommonFramework/AudioPipeline/UI/AudioSelectorWidget.h - Source/CommonFramework/BuildCommit.h Source/CommonFramework/Environment/HardwareValidation.cpp Source/CommonFramework/Environment/HardwareValidation.h Source/CommonFramework/Environment/HardwareValidation_arm64.tpp