diff --git a/SerialPrograms/Source/CommonFramework/Panels/PanelList.h b/SerialPrograms/Source/CommonFramework/Panels/PanelList.h index 995b028763..b6da3ab2d5 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/PanelList.h +++ b/SerialPrograms/Source/CommonFramework/Panels/PanelList.h @@ -35,6 +35,9 @@ class PanelListDescriptor{ PanelListWidget* make_QWidget(QWidget& parent, PanelHolder& holder) const; + // The entries of the list, as the widget shows them. + std::vector panels() const{ return make_panels(); } + protected: virtual std::vector make_panels() const = 0; diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp index f4176b9199..fe9c3ddf22 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp @@ -4,9 +4,11 @@ * */ +#include #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PanicDump.h" +#include "Common/Cpp/ScopeExit.h" #include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/Logging/Logger.h" @@ -64,11 +66,12 @@ void ProgramSession::remove_listener(Listener& listener){ -ProgramSession::ProgramSession(const ProgramDescriptor& descriptor) +ProgramSession::ProgramSession(const ProgramDescriptor& descriptor, RunRecordStore& run_records) : m_descriptor(descriptor) , m_instance_id(ProgramTracker::instance().add_program(*this)) + , m_run_recorder(run_records, global_logger_raw()) // , m_logger(global_logger_raw(), "Program:" + std::to_string(m_instance_id)) - , m_logger(global_logger_raw(), "Program") + , m_logger(m_run_recorder, "Program") , m_last_state_change(current_time()) , m_state(ProgramState::STOPPED) , m_scope(nullptr) @@ -186,6 +189,9 @@ void ProgramSession::push_stats(){ } void ProgramSession::push_error(const std::string& message){ m_listeners.run_method(&Listener::error, message); + // A diagnostic in the run's record, not its outcome: the stats-save + // error and the download warnings come through here too. + m_run_recorder.event({.kind = "error", .text = message}); } void ProgramSession::push_download_error(const std::string& message){ @@ -237,38 +243,8 @@ std::string ProgramSession::start_program(){ m_logger.log("Program is not ready.", COLOR_RED); return "Program is not ready."; - case ProgramState::STOPPED:{ - std::string error = check_validity(); - if (!error.empty()){ - m_logger.log(error, COLOR_RED); - return error; - } - - // Wait for previous program thread to finish. - join_program_thread(); - - // Now start the program. - m_logger.log("Starting program..."); - m_last_state_change.store(current_time(), std::memory_order_relaxed); - set_state(ProgramState::RUNNING); - m_program_thread = GlobalThreadPools::unlimited_realtime().dispatch_now_blocking( - [this]{ - run_with_catch( - "ProgramSession::start_program()", - [this]{ run_program(); } - ); - } - ); -#if 0 - m_thread = Thread([this]{ - run_with_catch( - "ProgramSession::start_program()", - [this]{ run_program(); } - ); - }); -#endif - return ""; - } + case ProgramState::STOPPED: + return start_accepted_run(); case ProgramState::RUNNING: m_logger.log("Program is already running.", COLOR_RED); return "Program is already running."; @@ -300,6 +276,7 @@ std::string ProgramSession::stop_program(){ case ProgramState::RUNNING: m_logger.log("Stopping program..."); set_state(ProgramState::STOPPING); + m_run_recorder.stopping(); break; case ProgramState::STOPPING: m_logger.log("Program is already stopping.", COLOR_RED); @@ -319,23 +296,119 @@ std::string ProgramSession::stop_program(){ -void ProgramSession::run_program(){ - SleepSuppressScope sleep_scope(GlobalSettings::instance().SLEEP_SUPPRESS->PROGRAM_RUNNING); +// Under the lock, for a stopped session: checks the settings, then starts +// the program thread. The run is accepted here: its record exists before +// the program thread can report anything. +std::string ProgramSession::start_accepted_run(){ + std::string error = check_validity(); + if (!error.empty()){ + m_logger.log(error, COLOR_RED); + return error; + } - { - std::lock_guard lg(m_lock); - if (current_state() != ProgramState::RUNNING){ - return; - } + // Wait for previous program thread to finish. + join_program_thread(); - std::string error = check_validity(); - if (!error.empty()){ - throw UserSetupError(logger(), std::move(error)); - } + m_logger.log("Starting program..."); + m_last_state_change.store(current_time(), std::memory_order_relaxed); + m_run_recorder.begin(identifier(), settings_json()); + set_state(ProgramState::RUNNING); + try{ + m_program_thread = dispatch_program_thread([this]{ + run_with_catch( + "ProgramSession::start_program()", + [this]{ run_program_and_record(); } + ); + }); + }catch (Exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.message())); + }catch (std::exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.what())); + } + return ""; +} - m_current_stats = m_descriptor.make_stats(); - load_historical_stats(); - push_stats(); +// No thread, so no run: closes the record and leaves the session stopped, +// so the next start can try again. Returns `message` for start_program(). +std::string ProgramSession::abandon_run(std::string message){ + m_logger.log(message, COLOR_RED); + m_run_recorder.finish({.outcome = RunOutcome::FAILED, .error = message}); + set_state(ProgramState::STOPPED); + return message; +} + +AsyncTask ProgramSession::dispatch_program_thread(std::function body){ + return GlobalThreadPools::unlimited_realtime().dispatch_now_blocking(std::move(body)); +} + +void ProgramSession::run_program_and_record(){ + // Exactly one result per run: whichever way the thread leaves, the + // record is closed, and a path that set none is a failure, never a + // success. An exception that escapes run_program() (as today it does + // for a validity check that fails after the start) is its result. + std::optional result; + ScopeExit finish_run([&]{ + m_run_recorder.finish(result.value_or(RunResult{ + .outcome = RunOutcome::FAILED, + .error = "The program thread ended without a result.", + })); + }); + try{ + result = run_program(); + }catch (Exception& e){ + result = RunResult{.outcome = RunOutcome::FAILED, .error = e.message().empty() ? e.name() : e.message()}; + throw; + }catch (std::exception& e){ + result = RunResult{.outcome = RunOutcome::FAILED, .error = e.what()}; + throw; + } +} + +// Under the lock: false when the run was stopped before the thread got +// going. Throws when the settings are no longer valid. Loads the stats. +bool ProgramSession::prepare_run(){ + std::lock_guard lg(m_lock); + if (current_state() != ProgramState::RUNNING){ + return false; + } + + std::string error = check_validity(); + if (!error.empty()){ + throw UserSetupError(logger(), std::move(error)); + } + + m_current_stats = m_descriptor.make_stats(); + load_historical_stats(); + push_stats(); + return true; +} + +// The run's result when the resources it needs could not be downloaded. +std::optional ProgramSession::download_resources(){ + if (download_prereqs(*m_scope)){ + return std::nullopt; + } + // The details were reported as "error" events. A stop during the + // download is a cancellation, not a failure. + if (current_state() == ProgramState::STOPPING){ + return RunResult{.outcome = RunOutcome::CANCELLED}; + } + return RunResult{.outcome = RunOutcome::FAILED, .error = "A required resource could not be downloaded."}; +} + +void ProgramSession::end_run(){ + std::lock_guard lg(m_lock); + push_stats(); + update_historical_stats_with_current(); + set_state(ProgramState::STOPPED); +} + +RunResult ProgramSession::run_program(){ + SleepSuppressScope sleep_scope(GlobalSettings::instance().SLEEP_SUPPRESS->PROGRAM_RUNNING); + RunResult result; + + if (!prepare_run()){ + return {.outcome = RunOutcome::CANCELLED}; } m_instance->options().reset_state(); @@ -349,8 +422,8 @@ void ProgramSession::run_program(){ { RunningProgramScope running_scope(*this); - if (!download_prereqs(*m_scope)){ - return; + if (std::optional failed = download_resources()){ + return *failed; } std::unique_ptr env = make_env(program_info); try{ @@ -360,10 +433,13 @@ void ProgramSession::run_program(){ internal_run_program(*env); env->log_to_ui("- Program Finished -"); logger().log("Program finished normally!", COLOR_BLUE); + result.outcome = RunOutcome::COMPLETED; }catch (OperationCancelledException&){ env->log_to_ui("- Program Stopped -"); + result.outcome = RunOutcome::CANCELLED; }catch (ProgramCancelledException&){ env->log_to_ui("- Program Stopped -"); + result.outcome = RunOutcome::CANCELLED; }catch (ProgramFinishedException& e){ logger().log("Program finished early!", COLOR_BLUE); env->log_to_ui("- Program Finished -"); @@ -372,6 +448,8 @@ void ProgramSession::run_program(){ e.message(), e.screenshot_view() ); + result.outcome = RunOutcome::COMPLETED; + result.frame = e.screenshot(); }catch (InvalidConnectionStateException& e){ logger().log("Program stopped due to connection issue.", COLOR_RED); env->log_to_ui("- Invalid Connection -", COLOR_RED); @@ -380,6 +458,7 @@ void ProgramSession::run_program(){ message = e.name(); } report_error(message); + result.error = message; }catch (FatalProgramException& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -390,6 +469,8 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; + result.frame = e.screenshot(); }catch (OperationFailedExceptionWithScreenshot& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -400,6 +481,8 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; + result.frame = e.screenshot(); }catch (OperationFailedException& e){ // no screenshot logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -410,6 +493,7 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; }catch (Exception& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -422,6 +506,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, message ); + result.error = message; }catch (std::exception& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -434,6 +519,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, message ); + result.error = message; }catch (...){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Unknown Error -", COLOR_RED); @@ -442,15 +528,12 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, "Unknown error." ); + result.error = "Unknown error."; } } - { - std::lock_guard lg(m_lock); - push_stats(); - update_historical_stats_with_current(); - set_state(ProgramState::STOPPED); - } + end_run(); + return result; } diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.h b/SerialPrograms/Source/CommonFramework/ProgramSession.h index af12d8c820..bafb6787a6 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.h +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.h @@ -17,6 +17,8 @@ #define PokemonAutomation_CommonFramework_ProgramSession_H #include +#include +#include #include #include #include "Common/Cpp/Logging/TaggedLogger.h" @@ -29,6 +31,7 @@ #include "CommonFramework/Globals.h" //#include "CommonFramework/Logging/Logger.h" #include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/RunRecord.h" #include "CommonFramework/Tools/ProgramEnvironment.h" // #include "CommonFramework/ResourceDownload/ProgramMissingResourceTracker.h" #include "Integrations/ProgramTrackerInterfaces.h" @@ -66,7 +69,12 @@ class ProgramSession : public TrackableProgram{ public: // Setup - ProgramSession(const ProgramDescriptor& descriptor); + // `run_records` is where this session's runs are recorded (RunRecord.h); + // a test passes its own store. + ProgramSession( + const ProgramDescriptor& descriptor, + RunRecordStore& run_records = app_run_records() + ); // Before the child-most class begins destruction, you must first: // 1. Stop the program. @@ -104,6 +112,10 @@ class ProgramSession : public TrackableProgram{ Mutex& program_lock() const{ return m_lock; } + // The program's progress hook writes to this session's active run. + friend void record_run_event(ProgramEnvironment& env, RunEvent event); + + public: // Program Control @@ -114,8 +126,8 @@ class ProgramSession : public TrackableProgram{ std::string stop_program(); // Returns before the program is completely stopped. // Temporary - virtual void async_start() override{ start_program(); } - virtual void async_stop() override{ stop_program(); } + virtual std::string async_start() override{ return start_program(); } + virtual std::string async_stop() override{ return stop_program(); } protected: @@ -123,6 +135,17 @@ class ProgramSession : public TrackableProgram{ virtual void internal_run_program(ProgramEnvironment& env) = 0; virtual void internal_stop_program(); + // Starts the program thread. A test fake makes this fail. + virtual AsyncTask dispatch_program_thread(std::function body); + + // The program's settings as JSON text, recorded with each run: what the + // settings file stores for it. "" for a session without settings. + virtual std::string settings_json() const{ return ""; } + + // The logger the session's consoles chain off, so that their lines land + // in the active run's log (RunRecord.h). + Logger& run_logger(){ return m_run_recorder; } + // virtual void restore_defaults(){ return; } public: @@ -153,13 +176,22 @@ class ProgramSession : public TrackableProgram{ private: - void run_program(); + // The program thread: runs the program and closes its run record with + // exactly one result, whichever way run_program() ends. + void run_program_and_record(); + RunResult run_program(); + std::string start_accepted_run(); + bool prepare_run(); + std::optional download_resources(); + void end_run(); + std::string abandon_run(std::string message); private: const ProgramDescriptor& m_descriptor; uint64_t m_instance_id = 0; + SessionRunRecorder m_run_recorder; // Before m_logger, which chains off it. TaggedLogger m_logger; diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.cpp b/SerialPrograms/Source/CommonFramework/RunRecord.cpp new file mode 100644 index 0000000000..e6feeef259 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord.cpp @@ -0,0 +1,247 @@ +/* Run Record + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Globals.h" +#include "RunRecord.h" + +namespace PokemonAutomation{ + + + +std::string run_status_name(RunStatus status){ + switch (status){ + case RunStatus::RUNNING: + return "running"; + case RunStatus::STOPPING: + return "stopping"; + case RunStatus::TERMINAL: + return "terminal"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown run status."); +} +std::string run_outcome_name(RunOutcome outcome){ + switch (outcome){ + case RunOutcome::COMPLETED: + return "completed"; + case RunOutcome::CANCELLED: + return "cancelled"; + case RunOutcome::FAILED: + return "failed"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown run outcome."); +} + + + +namespace{ +std::atomic& app_run_records_enabled(){ + static std::atomic enabled{false}; + return enabled; +} +} + +RunRecordStore& app_run_records(){ + static RunRecordStore store(RunRecordStore::DEFAULT_CAPACITY, &app_run_records_enabled()); + return store; +} +void enable_app_run_records(){ + app_run_records_enabled().store(true, std::memory_order_release); +} + +RunRecordStore::RunRecordStore(size_t capacity, const std::atomic* enabled) + // The app's start time, to the microsecond: two instances started in + // the same second still get different IDs. + : m_app_token(now_to_filestring()) + , m_capacity(std::max(capacity, 1)) + , m_enabled(enabled) +{} + +std::string RunRecordStore::begin(std::string program, std::string settings){ + if (m_enabled != nullptr && !m_enabled->load(std::memory_order_acquire)){ + return ""; + } + std::lock_guard lg(m_lock); + while (m_records.size() >= m_capacity){ + m_records.pop_front(); + } + RunRecord& record = m_records.emplace_back(); + record.run_id = std::format("{}-{:03}", m_app_token, ++m_counter); + record.program = std::move(program); + record.version = PROGRAM_VERSION; + record.executable = QCoreApplication::instance() == nullptr + ? "" + : QCoreApplication::applicationFilePath().toStdString(); + record.settings = std::move(settings); + record.start_time = current_time(); + return record.run_id; +} + +void RunRecordStore::stopping(const std::string& run_id){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record != nullptr && record->status == RunStatus::RUNNING){ + record->status = RunStatus::STOPPING; + } +} + +void RunRecordStore::event(const std::string& run_id, RunEvent event){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr){ + return; + } + if (record->events.size() >= MAX_EVENTS){ + record->dropped_events++; + return; + } + if (event.time == WallClock::min()){ + event.time = current_time(); + } + record->events.emplace_back(std::move(event)); + + // Keep the frames of the newest events only. + size_t frames = 0; + for (RunEvent& e : std::ranges::reverse_view(record->events)){ + if (e.frame == nullptr){ + continue; + } + if (++frames > MAX_EVENT_FRAMES){ + e.frame.reset(); + } + } +} + +void RunRecordStore::log_line(const std::string& run_id, const std::string& text){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr){ + return; + } + const size_t bytes = text.size() + 1; + if (record->log_bytes + bytes > MAX_LOG_BYTES){ + record->dropped_log_lines++; + return; + } + record->log_lines++; + record->log_bytes += bytes; + record->log_tail.emplace_back(text); + record->log_tail_bytes += bytes; + while (record->log_tail_bytes > MAX_LOG_TAIL_BYTES && record->log_tail.size() > 1){ + record->log_tail_bytes -= record->log_tail.front().size() + 1; + record->log_tail.pop_front(); + } +} + +bool RunRecordStore::finish(const std::string& run_id, RunResult result){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr || record->status == RunStatus::TERMINAL){ + return false; + } + record->status = RunStatus::TERMINAL; + record->end_time = current_time(); + record->result = std::move(result); + + // Frames stay with the newest terminal runs; older ones keep the rest + // of their record. + size_t terminal = 0; + for (RunRecord& r : std::ranges::reverse_view(m_records)){ + if (r.status == RunStatus::TERMINAL && ++terminal > RUNS_WITH_FRAMES){ + release_frames(r); + } + } + return true; +} + +std::vector RunRecordStore::run_ids() const{ + std::lock_guard lg(m_lock); + std::vector ids; + for (const RunRecord& record : m_records){ + ids.emplace_back(record.run_id); + } + return ids; +} + +std::optional RunRecordStore::get(const std::string& run_id) const{ + std::lock_guard lg(m_lock); + for (const RunRecord& record : m_records){ + if (record.run_id == run_id){ + return record; + } + } + return std::nullopt; +} + +RunRecord* RunRecordStore::find(const std::string& run_id){ + auto it = std::ranges::find_if(m_records, [&](const RunRecord& r){ return r.run_id == run_id; }); + return it == m_records.end() ? nullptr : &*it; +} + +void RunRecordStore::release_frames(RunRecord& record){ + for (RunEvent& event : record.events){ + event.frame.reset(); + } + if (record.result){ + record.result->frame.reset(); + } +} + + + +SessionRunRecorder::SessionRunRecorder(RunRecordStore& store, Logger& base_logger) + : m_store(store) + , m_base_logger(base_logger) +{} + +std::string SessionRunRecorder::begin(std::string program, std::string settings){ + std::lock_guard lg(m_lock); + m_run_id = m_store.begin(std::move(program), std::move(settings)); + m_active = !m_run_id.empty(); + return m_run_id; +} +void SessionRunRecorder::stopping(){ + std::lock_guard lg(m_lock); + if (m_active){ + m_store.stopping(m_run_id); + } +} +void SessionRunRecorder::finish(RunResult result){ + std::lock_guard lg(m_lock); + if (!m_active){ + return; + } + m_active = false; + m_store.finish(m_run_id, std::move(result)); +} + +std::string SessionRunRecorder::last_run_id() const{ + std::lock_guard lg(m_lock); + return m_run_id; +} + +void SessionRunRecorder::event(RunEvent event){ + std::lock_guard lg(m_lock); + if (m_active){ + m_store.event(m_run_id, std::move(event)); + } +} +void SessionRunRecorder::log(const std::string& msg, Color color){ + m_base_logger.log(msg, color); + std::lock_guard lg(m_lock); + if (m_active){ + m_store.log_line(m_run_id, msg); + } +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.h b/SerialPrograms/Source/CommonFramework/RunRecord.h new file mode 100644 index 0000000000..c6f8c082f2 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord.h @@ -0,0 +1,201 @@ +/* Run Record + * + * From: https://github.com/PokemonAutomation/ + * + * What one run of a program did, as the framework saw it: when it started, + * how it ended (exactly one outcome per accepted run), the error and the + * frame it died with, the events the program reported on the way, and the + * lines of its log, and the app and settings it ran with. + * ProgramSession writes these at its execution boundary; RunRecordStore + * keeps the last few for whoever asks, such as a tool that writes them to + * files. The store records nothing until it is enabled. Nothing here + * depends on the UI, on JSON or on a game. + * + */ + +#ifndef PokemonAutomation_CommonFramework_RunRecord_H +#define PokemonAutomation_CommonFramework_RunRecord_H + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" + +namespace PokemonAutomation{ + + +enum class RunStatus{ + RUNNING, + STOPPING, // A stop was requested; the outcome is not known yet. + TERMINAL, +}; +enum class RunOutcome{ + COMPLETED, // The program returned or threw ProgramFinishedException. + CANCELLED, // The program ended through the stop path. + FAILED, // A setup, connection or execution error ended it. +}; +std::string run_status_name(RunStatus status); // "running", "stopping", "terminal" +std::string run_outcome_name(RunOutcome outcome); // "completed", "cancelled", "failed" + + +// Something the program says about its own progress: a checkpoint reached, +// a step started, an attempt that failed, a pose. Every field but `kind` +// and `text` is optional; a game fills the ones it has. (The `= {}` let a +// designated initializer leave fields out on every compiler.) +struct RunEvent{ + WallClock time = WallClock::min(); // Set by the recorder when left at min(). + std::string kind = {}; // e.g. "checkpoint", "step", "attempt", "error" + std::optional checkpoint = {}; + std::optional step = {}; + std::optional attempt = {}; + std::string text = {}; + std::shared_ptr frame = {}; // The screen at that moment, if the program had it. + std::optional pose = {}; // Where the player is, as the program puts it. +}; + +// How a run ended. +struct RunResult{ + RunOutcome outcome = RunOutcome::FAILED; + std::optional error = {}; // The primary error of a failed run. + std::shared_ptr frame = {}; // The frame the exception carried, if any. +}; + +struct RunRecord{ + std::string run_id = {}; + std::string program = {}; // The program's identifier, ":". + std::string version = {}; // PROGRAM_VERSION of the app that ran it. + std::string executable = {}; // The path of the app's executable, which names a lab build. + std::string settings = {}; // The program's settings as JSON text, as the settings file stores them. + WallClock start_time = WallClock::min(); + std::optional end_time = {}; + RunStatus status = RunStatus::RUNNING; + std::optional result = {}; // Set once, when the status turns TERMINAL. + std::vector events = {}; + size_t dropped_events = 0; // Events past RunRecordStore::MAX_EVENTS. + + // The run's log. Lines are numbered from 0 in the order they came; + // the tail holds the newest of them, so it starts at line + // `log_lines - log_tail.size()`. A reader that copies the log out as it + // grows keeps its own count of the lines it has. + std::deque log_tail = {}; // At most MAX_LOG_TAIL_BYTES, oldest first. + size_t log_tail_bytes = 0; // The tail's size, a newline per line. + size_t log_lines = 0; // Lines recorded so far. + size_t log_bytes = 0; // Their size, a newline per line, at most MAX_LOG_BYTES. + size_t dropped_log_lines = 0; // Lines past MAX_LOG_BYTES, not recorded. +}; + + +// The last few runs of this app instance. Thread-safe; every call is short +// and copies, so a reader never holds a lock while writing files. +// +// A store given an `enabled` flag records only while the flag holds: +// otherwise begin() returns "" and every call about that run is a no-op. +// The app's store (app_run_records()) records once enable_app_run_records() +// was called, by whoever reads the records, e.g. the lab channel. +// +// Run IDs are opaque strings unique to this app instance: a restart cannot +// resolve an old ID to a new run. They sort in the order the runs started. +// +// Memory: at most MAX_EVENTS events per run; the frames of a run's newest +// MAX_EVENT_FRAMES events; all frames only for the active run and the +// newest RUNS_WITH_FRAMES terminal runs; a log tail of MAX_LOG_TAIL_BYTES. +// A consumer that wants a frame or every log line reads them soon after +// they come. A run records at most MAX_LOG_BYTES of log in all. +class RunRecordStore{ +public: + static constexpr size_t DEFAULT_CAPACITY = 32; + static constexpr size_t MAX_EVENTS = 4096; + static constexpr size_t MAX_EVENT_FRAMES = 8; + static constexpr size_t RUNS_WITH_FRAMES = 4; + static constexpr size_t MAX_LOG_TAIL_BYTES = 256 * 1024; + static constexpr size_t MAX_LOG_BYTES = 8 * 1024 * 1024; + + // `enabled`, when given, must outlive the store. + explicit RunRecordStore(size_t capacity = DEFAULT_CAPACITY, const std::atomic* enabled = nullptr); + RunRecordStore(const RunRecordStore&) = delete; + void operator=(const RunRecordStore&) = delete; + + // Opens a record for an accepted start and returns its run ID, or "" + // while the store is disabled. The oldest record goes when the store is + // full. `settings` is the program's settings as JSON text. + std::string begin(std::string program, std::string settings = ""); + + // Marks a running run as stopping. No effect once it is terminal. + void stopping(const std::string& run_id); + + // Appends to the run. Both are dropped for an unknown run. + void event(const std::string& run_id, RunEvent event); + void log_line(const std::string& run_id, const std::string& text); + + // Closes the run with its one result. Returns false, and changes + // nothing, when the run is unknown or already terminal. + bool finish(const std::string& run_id, RunResult result); + + // The IDs of the records held, oldest first. + std::vector run_ids() const; + + // A copy of the record, or nothing for an unknown run. + std::optional get(const std::string& run_id) const; + + +private: + RunRecord* find(const std::string& run_id); + void release_frames(RunRecord& record); + + mutable Mutex m_lock; + const std::string m_app_token; + const size_t m_capacity; + const std::atomic* m_enabled; + uint64_t m_counter = 0; + std::deque m_records; +}; + + +// The app's store, and the switch that makes it record. Until the switch +// is thrown, no run is recorded, so an app nobody reads records from keeps +// nothing. +RunRecordStore& app_run_records(); +void enable_app_run_records(); + + +// One session's end of the store: the run active now, the events the +// program reports about it (ProgramEnvironment.h, record_run_event()), and +// a Logger that copies what passes through it into that run's log. +// The session's loggers chain off `log()`, so a run's log holds the lines +// the program and its consoles logged, and nothing from other sessions. +class SessionRunRecorder : public Logger{ +public: + SessionRunRecorder(RunRecordStore& store, Logger& base_logger); + + // Opens a run and makes it the active one. Returns its ID, or "" when + // the store is disabled, in which case nothing of it is recorded. + std::string begin(std::string program, std::string settings = ""); + void stopping(); + // Closes the active run; nothing happens when none is active. + void finish(RunResult result); + + // The run the last begin() opened, active or finished; "" before the first. + std::string last_run_id() const; + + // Adds to the active run; dropped when none is active. + void event(RunEvent event); + virtual void log(const std::string& msg, Color color = Color()) override; + +private: + RunRecordStore& m_store; + Logger& m_base_logger; + mutable Mutex m_lock; + std::string m_run_id; + bool m_active = false; +}; + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp new file mode 100644 index 0000000000..4d25c45fb1 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp @@ -0,0 +1,635 @@ +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The store on its own, and the run lifecycle through a ProgramSession + * whose program is a scripted fake: no console, no window. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/Notifications/ProgramInfo.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/ProgramSession.h" +#include "CommonFramework/Tools/ProgramEnvironment.h" +#include "RunRecord.h" +#include "RunRecord_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + +using namespace std::chrono_literals; + + +std::shared_ptr make_frame(size_t width){ + ImageRGB32 image(width, width); + image.fill(0xff000000); + return std::make_shared(std::move(image)); +} + +// Polls `done` every few milliseconds until it holds or `timeout` passes. +bool wait_until(const std::function& done, Milliseconds timeout = 5000ms){ + const WallClock deadline = current_time() + timeout; + while (!done()){ + if (current_time() > deadline){ + return false; + } + std::this_thread::sleep_for(5ms); + } + return true; +} + + + +// The run the store opened last, "" when none. +std::string newest_run(const RunRecordStore& store){ + const std::vector ids = store.run_ids(); + return ids.empty() ? "" : ids.back(); +} + +UnitTestResult test_store_ids(){ + RunRecordStore store(2); + const std::string a = store.begin("Test:A", R"({"START": 3})"); + const std::string b = store.begin("Test:B"); + if (a.empty() || a == b || !(a < b)){ + return std::format("Run IDs should be distinct and sort in start order: <{}>, <{}>", a, b); + } + if (store.run_ids() != std::vector{a, b}){ + return "run_ids() lists the records oldest first."; + } + const std::optional record = store.get(a); + if (!record || record->program != "Test:A" || record->status != RunStatus::RUNNING || record->result){ + return "A new record is running, with its program and no result."; + } + if (record->start_time == WallClock::min() || record->end_time){ + return "A new record has a start time and no end time."; + } + if (record->version != PROGRAM_VERSION || (QCoreApplication::instance() != nullptr && record->executable.empty())){ + return std::format("A record names the app: version <{}>, executable <{}>.", record->version, record->executable); + } + if (record->settings != R"({"START": 3})" || !store.get(b)->settings.empty()){ + return "A record keeps the settings it was begun with."; + } + const std::string c = store.begin("Test:C"); + if (store.get(a) || store.run_ids() != std::vector{b, c}){ + return "A full store drops its oldest record."; + } + if (store.get("nobody")){ + return "An unknown run has no record."; + } + return true; +} + +UnitTestResult test_store_one_result(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.stopping(id); + if (store.get(id)->status != RunStatus::STOPPING){ + return "stopping() marks a running run."; + } + if (!store.finish(id, {.outcome = RunOutcome::FAILED, .error = "first"})){ + return "The first finish() takes."; + } + store.stopping(id); + if (store.finish(id, {.outcome = RunOutcome::COMPLETED})){ + return "A second finish() is refused."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || !record.end_time || !record.result){ + return "A finished record is terminal with an end time and a result."; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "first"){ + return std::format("The first result stands: got {}", run_outcome_name(record.result->outcome)); + } + if (store.finish("nobody", {})){ + return "Finishing an unknown run is refused."; + } + return true; +} + +UnitTestResult test_store_events(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.event(id, {.kind = "checkpoint", .checkpoint = "00.01", .text = "ok"}); + store.event("nobody", {.kind = "checkpoint"}); + for (size_t c = 0; c < RunRecordStore::MAX_EVENT_FRAMES + 2; c++){ + store.event(id, {.kind = "step", .step = std::to_string(c), .frame = make_frame(4)}); + } + RunRecord record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENT_FRAMES + 3){ + return std::format("Expected {} events, got {}.", RunRecordStore::MAX_EVENT_FRAMES + 3, record.events.size()); + } + if (record.events[0].time == WallClock::min() || record.events[0].checkpoint != "00.01"){ + return "An event gets the time it was recorded and keeps its fields."; + } + if (record.events[1].frame || record.events[2].frame || !record.events[3].frame || !record.events.back().frame){ + return "Only the newest MAX_EVENT_FRAMES events keep their frames."; + } + + for (size_t c = record.events.size(); c < RunRecordStore::MAX_EVENTS + 5; c++){ + store.event(id, {.kind = "step"}); + } + record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENTS || record.dropped_events != 5){ + return std::format("Events past the cap are counted, not kept: {} events, {} dropped.", record.events.size(), record.dropped_events); + } + return true; +} + +UnitTestResult test_store_log(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + RunRecord record; + + // Lines of 1000 bytes, numbered: 300 are more than the tail holds. + const std::string line(995, 'x'); + for (size_t c = 0; c < 300; c++){ + store.log_line(id, std::format("{:04} {}", c, line)); + } + record = *store.get(id); + size_t bytes = 0; + for (const std::string& l : record.log_tail){ + bytes += l.size() + 1; + } + if (record.log_lines != 300 || record.log_bytes != 300 * 1001 || record.dropped_log_lines != 0){ + return std::format("Every line is counted: {} lines, {} bytes.", record.log_lines, record.log_bytes); + } + if (bytes != record.log_tail_bytes || bytes > RunRecordStore::MAX_LOG_TAIL_BYTES || record.log_tail.size() == 300){ + return std::format("The tail keeps at most {} bytes: {} lines, {} bytes.", RunRecordStore::MAX_LOG_TAIL_BYTES, record.log_tail.size(), bytes); + } + const size_t first = record.log_lines - record.log_tail.size(); + if (record.log_tail.front().rfind(std::format("{:04} ", first), 0) != 0 || record.log_tail.back().rfind("0299 ", 0) != 0){ + return "The tail holds the newest lines, from line log_lines - log_tail.size()."; + } + + // Past MAX_LOG_BYTES in all, lines are counted and not kept. + const size_t fit = RunRecordStore::MAX_LOG_BYTES / 1001; + for (size_t c = 300; c < fit + 10; c++){ + store.log_line(id, std::format("{:04} {}", c % 10000, line)); + } + record = *store.get(id); + if (record.log_lines != fit || record.dropped_log_lines != 10 || record.log_bytes > RunRecordStore::MAX_LOG_BYTES){ + return std::format("A run records at most {} bytes of log: {} lines, {} dropped.", RunRecordStore::MAX_LOG_BYTES, record.log_lines, record.dropped_log_lines); + } + return true; +} + +UnitTestResult test_store_disabled(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + std::atomic enabled{false}; + RunRecordStore store(4, &enabled); + if (!store.begin("Test:A").empty() || !store.run_ids().empty()){ + return "A disabled store opens no record."; + } + CountingLogger base; + SessionRunRecorder recorder(store, base); + if (!recorder.begin("Test:A").empty() || !recorder.last_run_id().empty()){ + return "A recorder over a disabled store has no run."; + } + recorder.log("line"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + if (base.lines != 1 || !store.run_ids().empty()){ + return "Lines still reach the base logger; nothing is recorded."; + } + enabled.store(true); + const std::string id = recorder.begin("Test:A"); + recorder.log("line"); + if (id.empty() || store.get(id)->log_lines != 1){ + return "Once enabled, the store records."; + } + return true; +} + +UnitTestResult test_store_releases_old_frames(){ + RunRecordStore store; + std::vector ids; + for (size_t c = 0; c < RunRecordStore::RUNS_WITH_FRAMES + 1; c++){ + ids.emplace_back(store.begin("Test:A")); + store.event(ids.back(), {.kind = "step", .frame = make_frame(4)}); + store.finish(ids.back(), {.outcome = RunOutcome::FAILED, .error = "e", .frame = make_frame(4)}); + } + const RunRecord oldest = *store.get(ids.front()); + if (oldest.events[0].frame || oldest.result->frame){ + return "A run older than the newest RUNS_WITH_FRAMES terminal runs keeps no frames."; + } + if (oldest.result->error != "e" || oldest.events.size() != 1){ + return "It keeps the rest of its record."; + } + const RunRecord newest = *store.get(ids.back()); + if (!newest.events[0].frame || !newest.result->frame){ + return "The newest terminal runs keep their frames."; + } + return true; +} + +UnitTestResult test_session_recorder(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + RunRecordStore store; + CountingLogger base; + SessionRunRecorder recorder(store, base); + recorder.log("before"); + recorder.event({.kind = "lost"}); + if (!recorder.last_run_id().empty() || !store.run_ids().empty()){ + return "Nothing is recorded before a run begins."; + } + const std::string id = recorder.begin("Test:A"); + if (recorder.last_run_id() != id){ + return "last_run_id() is the run begin() opened."; + } + recorder.log("during"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + recorder.log("after"); + recorder.event({.kind = "late"}); + recorder.finish({.outcome = RunOutcome::FAILED}); + const RunRecord record = *store.get(id); + if (base.lines != 3){ + return std::format("Every line reaches the base logger: {} of 3.", base.lines); + } + if (record.log_tail != std::deque{"during"} || record.events.size() != 1){ + return "Only lines and events during the run are recorded."; + } + if (record.result->outcome != RunOutcome::COMPLETED || recorder.last_run_id() != id){ + return "The result stands and the run stays known after finish()."; + } + return true; +} + + + +// A program with no console: what it does is scripted per test. +struct FakeScript{ + std::string validity = {}; // check_validity(): "" accepts the start. + bool fail_dispatch = false; // The program thread cannot be started. + std::function start_checks = {}; + std::function body = {}; + std::string settings = {}; // settings_json(). +}; + +class FakeDescriptor : public ProgramDescriptor{ +public: + FakeDescriptor() + : ProgramDescriptor( + "Test:RunRecordFake", "Test", "Run Record Fake", + "", "", PanelQualityStanding::DEVELOPER, false, {} + ) + {} + virtual std::unique_ptr make_panel() const override{ return nullptr; } +}; + +class FakeInstance : public ProgramInstance{ +public: + explicit FakeInstance(std::function start_checks) + : m_start_checks(std::move(start_checks)) + {} + virtual void run_start_program_checks(const ProgramDescriptor&, ProgramEnvironment& env) override{ + if (m_start_checks){ + m_start_checks(env); + } + } +private: + std::function m_start_checks; +}; + +class FakeProgram : public ProgramSession{ +public: + FakeProgram(const FakeDescriptor& descriptor, RunRecordStore& store, FakeScript script) + : ProgramSession(descriptor, store) + , m_script(std::move(script)) + { + m_instance = std::make_unique(m_script.start_checks); + } + virtual ~FakeProgram(){ + internal_stop_program(); + join_program_thread(); + } + + // Whether the body has started running. + bool running() const{ return m_running.load(std::memory_order_acquire); } + + // Waits for the program thread to end, so the run's record is closed. + bool wait_until_stopped(){ + if (!wait_until([this]{ return current_state() == ProgramState::STOPPED; })){ + return false; + } + join_program_thread(); + return true; + } + + virtual std::string check_validity() const override{ return m_script.validity; } + +protected: + virtual std::unique_ptr make_env(const ProgramInfo& program_info) override{ + return std::make_unique(program_info, *this, current_stats_tracker(), historical_stats_tracker()); + } + virtual void internal_run_program(ProgramEnvironment& env) override{ + m_running.store(true, std::memory_order_release); + if (m_script.body){ + m_script.body(*this, env, *m_scope); + } + } + virtual std::string settings_json() const override{ return m_script.settings; } + virtual AsyncTask dispatch_program_thread(std::function body) override{ + if (m_script.fail_dispatch){ + throw std::runtime_error("no thread"); + } + return ProgramSession::dispatch_program_thread(std::move(body)); + } + +private: + FakeScript m_script; + std::atomic m_running{false}; +}; + +// Starts the program, waits for it to end, and returns its record. +UnitTestResult run_to_end(FakeProgram& program, RunRecordStore& store, RunRecord& record){ + const std::string error = program.start_program(); + if (!error.empty()){ + return "The start was refused: " + error; + } + if (!program.wait_until_stopped()){ + return "The program did not stop."; + } + const std::optional found = store.get(newest_run(store)); + if (!found){ + return "The run has no record."; + } + record = *found; + if (record.status != RunStatus::TERMINAL || !record.result){ + return "The record is not terminal after the program stopped."; + } + return true; +} + +UnitTestResult test_rejection(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.validity = "Pick a save."}); + if (program.start_program() != "Pick a save."){ + return "A start refused by check_validity() returns its message."; + } + if (!store.run_ids().empty() || !newest_run(store).empty()){ + return "A refused start opens no run."; + } + return true; +} + +UnitTestResult test_immediate_completion(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment& env, CancellableScope&){ + record_run_event(env, {.kind = "checkpoint", .checkpoint = "00.01", .step = "3/3", .attempt = 1, .text = "ok", .pose = "home (2, 4) N"}); + }, + .settings = R"({"START": 1})", + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::COMPLETED || record.result->error || record.result->frame){ + return "A program that returns completed its run, with no error and no frame."; + } + if (record.program != "Test:RunRecordFake" || record.start_time > *record.end_time){ + return "The record names the program and its times."; + } + if (record.settings != R"({"START": 1})" || record.version != PROGRAM_VERSION){ + return "The record has the session's settings and the app's version."; + } + if (record.events.size() != 1 || record.events[0].checkpoint != "00.01" || record.events[0].attempt != size_t(1) || record.events[0].pose != "home (2, 4) N"){ + return std::format("The program's event is in the record: {} events.", record.events.size()); + } + bool stopped_logged = false; + for (const std::string& line : record.log_tail){ + stopped_logged |= line.find("Program State: STOPPED") != std::string::npos; + } + if (!stopped_logged){ + return "The run's log tail runs through the session's last state change."; + } + return true; +} + +UnitTestResult test_setup_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .start_checks = [](ProgramEnvironment& env){ + throw UserSetupError(env.logger(), "The controller is not ready."); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The controller is not ready."){ + return "A setup error fails the run with its message."; + } + if (record.events.size() != 1 || record.events[0].kind != "error" || record.events[0].text != "The controller is not ready."){ + return "The reported error is an \"error\" event."; + } + return true; +} + +UnitTestResult test_failure_with_frame(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + ImageRGB32 screen(8, 8); + screen.fill(0xffff0000); + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "The sign's box did not open.", nullptr, std::move(screen) + ); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The sign's box did not open."){ + return "The exception's message is the primary error."; + } + if (!record.result->frame || record.result->frame->width() != 8){ + return "The exception's frame is kept with the result."; + } + return true; +} + +UnitTestResult test_cancellation(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope& scope){ + while (true){ + scope.throw_if_cancelled(); + std::this_thread::sleep_for(5ms); + } + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = newest_run(store); + if (!program.stop_program().empty() || !program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || record.result->outcome != RunOutcome::CANCELLED || record.result->error){ + return "A stopped program's run is cancelled, with no error."; + } + return true; +} + +// The body waits for the stop request and then ends its own way. +UnitTestResult run_racing_a_stop(const std::function& ending, RunOutcome expected, const char* what){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [&](ProgramSession& self, ProgramEnvironment&, CancellableScope&){ + wait_until([&]{ return self.current_state() == ProgramState::STOPPING; }); + ending(); + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = newest_run(store); + program.stop_program(); + if (!program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.result->outcome != expected){ + return std::format("{}: expected {}, got {}.", what, run_outcome_name(expected), run_outcome_name(record.result->outcome)); + } + return true; +} +UnitTestResult test_late_stop(){ + UnitTestResult result = run_racing_a_stop( + []{ throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "Failed anyway."); }, + RunOutcome::FAILED, "A failure observed after the stop request stays a failure" + ); + if (result.result != UnitTestResult::PASSED){ + return result; + } + return run_racing_a_stop( + []{}, RunOutcome::COMPLETED, "A normal return racing the stop request stays completed" + ); +} + +UnitTestResult test_dispatch_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.fail_dispatch = true}); + const std::string error = program.start_program(); + if (error.find("could not be started") == std::string::npos){ + return "A start whose thread cannot be dispatched returns an error: <" + error + ">"; + } + const std::optional record = store.get(newest_run(store)); + if (!record || record->status != RunStatus::TERMINAL || record->result->outcome != RunOutcome::FAILED){ + return "The accepted run is closed as failed."; + } + if (record->result->error != error){ + return "The run's error is the start's error."; + } + if (program.current_state() != ProgramState::STOPPED){ + return "The session is stopped again, so the next start can try."; + } + return true; +} + +UnitTestResult test_second_run_preserves_first(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram failing(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "first", nullptr, ImageRGB32(8, 8) + ); + }, + }); + RunRecord first; + UnitTestResult result = run_to_end(failing, store, first); + if (result.result != UnitTestResult::PASSED){ + return result; + } + FakeProgram completing(descriptor, store, {}); + RunRecord second; + result = run_to_end(completing, store, second); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (first.run_id == second.run_id || !(first.run_id < second.run_id)){ + return "Each run has its own ID, sorting after the earlier one."; + } + const std::optional kept = store.get(first.run_id); + if (!kept || kept->result->outcome != RunOutcome::FAILED || kept->result->error != "first" || !kept->result->frame){ + return "The first run's result and frame are still there after the second run."; + } + if (second.result->outcome != RunOutcome::COMPLETED || store.run_ids() != std::vector{first.run_id, second.run_id}){ + return "The second run has its own, completed, record."; + } + return true; +} + + +class Test_RunRecord : public UnitTest{ +public: + Test_RunRecord(std::string name, std::function body) + : UnitTest("CommonFramework::RunRecord - " + std::move(name)) + , m_body(std::move(body)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_body(); + } + +private: + std::function m_body; +}; + +} + + +void add_tests_RunRecord(UnitTestDatabase& database){ + database.add("store: ids and capacity", test_store_ids); + database.add("store: one result per run", test_store_one_result); + database.add("store: events and frames", test_store_events); + database.add("store: numbered log and its cap", test_store_log); + database.add("store: old runs release frames", test_store_releases_old_frames); + database.add("store: disabled", test_store_disabled); + database.add("session recorder", test_session_recorder); + database.add("lifecycle: rejected start", test_rejection); + database.add("lifecycle: immediate completion", test_immediate_completion); + database.add("lifecycle: setup failure", test_setup_failure); + database.add("lifecycle: failure with frame", test_failure_with_frame); + database.add("lifecycle: cancellation", test_cancellation); + database.add("lifecycle: late stop", test_late_stop); + database.add("lifecycle: dispatch failure", test_dispatch_failure); + database.add("lifecycle: second run preserves the first", test_second_run_preserves_first); +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h new file mode 100644 index 0000000000..9a3b88afcd --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h @@ -0,0 +1,20 @@ +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonFramework_RunRecord_Tests_H +#define PokemonAutomation_CommonFramework_RunRecord_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +void add_tests_RunRecord(UnitTestDatabase& database); + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp index 07e95a1a16..44c8958ff2 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp @@ -33,6 +33,10 @@ const ProgramInfo& ProgramEnvironment::program_info() const{ return m_program_info; } +void record_run_event(ProgramEnvironment& env, RunEvent event){ + env.m_session.m_run_recorder.event(std::move(event)); +} + void ProgramEnvironment::update_stats(){ m_session.report_stats_changed(); diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h index 72d6d812df..6dc5709dd6 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h @@ -9,6 +9,7 @@ #include "Common/Compiler.h" #include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/RunRecord.h" namespace PokemonAutomation{ @@ -42,6 +43,9 @@ class ProgramEnvironment{ virtual void log_to_ui(const std::string& msg, Color color = Color()){} + friend void record_run_event(ProgramEnvironment& env, RunEvent event); + + public: // Stats Management @@ -67,6 +71,13 @@ class ProgramEnvironment{ +// Progress reporting (RunRecord.h): what the program says about its +// checkpoints, steps and attempts goes to this run's record, if the app +// records runs. +void record_run_event(ProgramEnvironment& env, RunEvent event); + + + // Templates template diff --git a/SerialPrograms/Source/CommonFramework/Windows/Lab.md b/SerialPrograms/Source/CommonFramework/Windows/Lab.md new file mode 100644 index 0000000000..86172c1fde --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Windows/Lab.md @@ -0,0 +1,273 @@ +# Lab Channel + +Lets a script drive a running SerialPrograms app through files, with no +window focus and no click: select a program, set its options, start and stop +it, read its state, and press buttons or check detectors on the console +between runs. Meant for a lab that drives one app per console from the shell. + +The channel is one folder per app instance, independent of the selected +program. Requests are text files; replies are text files. Nothing else is +needed on the script side. + +## Turning it on + +Create the folder `Lab/` inside the profile's user-files folder (the folder +that holds `SerialPrograms.log`, `Screenshots/` and the settings; with +`--profile NAME` that is `Profiles/NAME/`) **before launching the app**. An +app that finds it at startup polls it every 500 ms from a worker and creates +`inbox/`, `outbox/` and `done/` inside it; an app that does not find it +starts nothing, so normal users see no change. Creating the folder later +takes effect at the next launch. + +## Files + +| Path | Direction | Meaning | +|---|---|---| +| `Lab/inbox/.txt` | script writes | A request: one command per line, run in order. Requests run one at a time, in name order. | +| `Lab/done/.txt` | app moves | The request, moved here when it starts. | +| `Lab/outbox/.out` | app writes | The request's journal (below). It appears only once the request has finished; `.tmp` shows the progress so far. | +| `Lab/outbox/_