diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp index f4176b9199..fe9c3ddf22 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp @@ -4,9 +4,11 @@ * */ +#include #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PanicDump.h" +#include "Common/Cpp/ScopeExit.h" #include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/Logging/Logger.h" @@ -64,11 +66,12 @@ void ProgramSession::remove_listener(Listener& listener){ -ProgramSession::ProgramSession(const ProgramDescriptor& descriptor) +ProgramSession::ProgramSession(const ProgramDescriptor& descriptor, RunRecordStore& run_records) : m_descriptor(descriptor) , m_instance_id(ProgramTracker::instance().add_program(*this)) + , m_run_recorder(run_records, global_logger_raw()) // , m_logger(global_logger_raw(), "Program:" + std::to_string(m_instance_id)) - , m_logger(global_logger_raw(), "Program") + , m_logger(m_run_recorder, "Program") , m_last_state_change(current_time()) , m_state(ProgramState::STOPPED) , m_scope(nullptr) @@ -186,6 +189,9 @@ void ProgramSession::push_stats(){ } void ProgramSession::push_error(const std::string& message){ m_listeners.run_method(&Listener::error, message); + // A diagnostic in the run's record, not its outcome: the stats-save + // error and the download warnings come through here too. + m_run_recorder.event({.kind = "error", .text = message}); } void ProgramSession::push_download_error(const std::string& message){ @@ -237,38 +243,8 @@ std::string ProgramSession::start_program(){ m_logger.log("Program is not ready.", COLOR_RED); return "Program is not ready."; - case ProgramState::STOPPED:{ - std::string error = check_validity(); - if (!error.empty()){ - m_logger.log(error, COLOR_RED); - return error; - } - - // Wait for previous program thread to finish. - join_program_thread(); - - // Now start the program. - m_logger.log("Starting program..."); - m_last_state_change.store(current_time(), std::memory_order_relaxed); - set_state(ProgramState::RUNNING); - m_program_thread = GlobalThreadPools::unlimited_realtime().dispatch_now_blocking( - [this]{ - run_with_catch( - "ProgramSession::start_program()", - [this]{ run_program(); } - ); - } - ); -#if 0 - m_thread = Thread([this]{ - run_with_catch( - "ProgramSession::start_program()", - [this]{ run_program(); } - ); - }); -#endif - return ""; - } + case ProgramState::STOPPED: + return start_accepted_run(); case ProgramState::RUNNING: m_logger.log("Program is already running.", COLOR_RED); return "Program is already running."; @@ -300,6 +276,7 @@ std::string ProgramSession::stop_program(){ case ProgramState::RUNNING: m_logger.log("Stopping program..."); set_state(ProgramState::STOPPING); + m_run_recorder.stopping(); break; case ProgramState::STOPPING: m_logger.log("Program is already stopping.", COLOR_RED); @@ -319,23 +296,119 @@ std::string ProgramSession::stop_program(){ -void ProgramSession::run_program(){ - SleepSuppressScope sleep_scope(GlobalSettings::instance().SLEEP_SUPPRESS->PROGRAM_RUNNING); +// Under the lock, for a stopped session: checks the settings, then starts +// the program thread. The run is accepted here: its record exists before +// the program thread can report anything. +std::string ProgramSession::start_accepted_run(){ + std::string error = check_validity(); + if (!error.empty()){ + m_logger.log(error, COLOR_RED); + return error; + } - { - std::lock_guard lg(m_lock); - if (current_state() != ProgramState::RUNNING){ - return; - } + // Wait for previous program thread to finish. + join_program_thread(); - std::string error = check_validity(); - if (!error.empty()){ - throw UserSetupError(logger(), std::move(error)); - } + m_logger.log("Starting program..."); + m_last_state_change.store(current_time(), std::memory_order_relaxed); + m_run_recorder.begin(identifier(), settings_json()); + set_state(ProgramState::RUNNING); + try{ + m_program_thread = dispatch_program_thread([this]{ + run_with_catch( + "ProgramSession::start_program()", + [this]{ run_program_and_record(); } + ); + }); + }catch (Exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.message())); + }catch (std::exception& e){ + return abandon_run(std::format("The program thread could not be started: {}", e.what())); + } + return ""; +} - m_current_stats = m_descriptor.make_stats(); - load_historical_stats(); - push_stats(); +// No thread, so no run: closes the record and leaves the session stopped, +// so the next start can try again. Returns `message` for start_program(). +std::string ProgramSession::abandon_run(std::string message){ + m_logger.log(message, COLOR_RED); + m_run_recorder.finish({.outcome = RunOutcome::FAILED, .error = message}); + set_state(ProgramState::STOPPED); + return message; +} + +AsyncTask ProgramSession::dispatch_program_thread(std::function body){ + return GlobalThreadPools::unlimited_realtime().dispatch_now_blocking(std::move(body)); +} + +void ProgramSession::run_program_and_record(){ + // Exactly one result per run: whichever way the thread leaves, the + // record is closed, and a path that set none is a failure, never a + // success. An exception that escapes run_program() (as today it does + // for a validity check that fails after the start) is its result. + std::optional result; + ScopeExit finish_run([&]{ + m_run_recorder.finish(result.value_or(RunResult{ + .outcome = RunOutcome::FAILED, + .error = "The program thread ended without a result.", + })); + }); + try{ + result = run_program(); + }catch (Exception& e){ + result = RunResult{.outcome = RunOutcome::FAILED, .error = e.message().empty() ? e.name() : e.message()}; + throw; + }catch (std::exception& e){ + result = RunResult{.outcome = RunOutcome::FAILED, .error = e.what()}; + throw; + } +} + +// Under the lock: false when the run was stopped before the thread got +// going. Throws when the settings are no longer valid. Loads the stats. +bool ProgramSession::prepare_run(){ + std::lock_guard lg(m_lock); + if (current_state() != ProgramState::RUNNING){ + return false; + } + + std::string error = check_validity(); + if (!error.empty()){ + throw UserSetupError(logger(), std::move(error)); + } + + m_current_stats = m_descriptor.make_stats(); + load_historical_stats(); + push_stats(); + return true; +} + +// The run's result when the resources it needs could not be downloaded. +std::optional ProgramSession::download_resources(){ + if (download_prereqs(*m_scope)){ + return std::nullopt; + } + // The details were reported as "error" events. A stop during the + // download is a cancellation, not a failure. + if (current_state() == ProgramState::STOPPING){ + return RunResult{.outcome = RunOutcome::CANCELLED}; + } + return RunResult{.outcome = RunOutcome::FAILED, .error = "A required resource could not be downloaded."}; +} + +void ProgramSession::end_run(){ + std::lock_guard lg(m_lock); + push_stats(); + update_historical_stats_with_current(); + set_state(ProgramState::STOPPED); +} + +RunResult ProgramSession::run_program(){ + SleepSuppressScope sleep_scope(GlobalSettings::instance().SLEEP_SUPPRESS->PROGRAM_RUNNING); + RunResult result; + + if (!prepare_run()){ + return {.outcome = RunOutcome::CANCELLED}; } m_instance->options().reset_state(); @@ -349,8 +422,8 @@ void ProgramSession::run_program(){ { RunningProgramScope running_scope(*this); - if (!download_prereqs(*m_scope)){ - return; + if (std::optional failed = download_resources()){ + return *failed; } std::unique_ptr env = make_env(program_info); try{ @@ -360,10 +433,13 @@ void ProgramSession::run_program(){ internal_run_program(*env); env->log_to_ui("- Program Finished -"); logger().log("Program finished normally!", COLOR_BLUE); + result.outcome = RunOutcome::COMPLETED; }catch (OperationCancelledException&){ env->log_to_ui("- Program Stopped -"); + result.outcome = RunOutcome::CANCELLED; }catch (ProgramCancelledException&){ env->log_to_ui("- Program Stopped -"); + result.outcome = RunOutcome::CANCELLED; }catch (ProgramFinishedException& e){ logger().log("Program finished early!", COLOR_BLUE); env->log_to_ui("- Program Finished -"); @@ -372,6 +448,8 @@ void ProgramSession::run_program(){ e.message(), e.screenshot_view() ); + result.outcome = RunOutcome::COMPLETED; + result.frame = e.screenshot(); }catch (InvalidConnectionStateException& e){ logger().log("Program stopped due to connection issue.", COLOR_RED); env->log_to_ui("- Invalid Connection -", COLOR_RED); @@ -380,6 +458,7 @@ void ProgramSession::run_program(){ message = e.name(); } report_error(message); + result.error = message; }catch (FatalProgramException& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -390,6 +469,8 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; + result.frame = e.screenshot(); }catch (OperationFailedExceptionWithScreenshot& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -400,6 +481,8 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; + result.frame = e.screenshot(); }catch (OperationFailedException& e){ // no screenshot logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -410,6 +493,7 @@ void ProgramSession::run_program(){ } report_error(message); e.send_fatal_error_notif_and_telemetry_report(*env, m_instance->NOTIFICATION_ERROR_FATAL); + result.error = message; }catch (Exception& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -422,6 +506,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, message ); + result.error = message; }catch (std::exception& e){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Program Error -", COLOR_RED); @@ -434,6 +519,7 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, message ); + result.error = message; }catch (...){ logger().log("Program stopped with an exception!", COLOR_RED); env->log_to_ui("- Unknown Error -", COLOR_RED); @@ -442,15 +528,12 @@ void ProgramSession::run_program(){ *env, m_instance->NOTIFICATION_ERROR_FATAL, "Unknown error." ); + result.error = "Unknown error."; } } - { - std::lock_guard lg(m_lock); - push_stats(); - update_historical_stats_with_current(); - set_state(ProgramState::STOPPED); - } + end_run(); + return result; } diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.h b/SerialPrograms/Source/CommonFramework/ProgramSession.h index af12d8c820..bafb6787a6 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.h +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.h @@ -17,6 +17,8 @@ #define PokemonAutomation_CommonFramework_ProgramSession_H #include +#include +#include #include #include #include "Common/Cpp/Logging/TaggedLogger.h" @@ -29,6 +31,7 @@ #include "CommonFramework/Globals.h" //#include "CommonFramework/Logging/Logger.h" #include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/RunRecord.h" #include "CommonFramework/Tools/ProgramEnvironment.h" // #include "CommonFramework/ResourceDownload/ProgramMissingResourceTracker.h" #include "Integrations/ProgramTrackerInterfaces.h" @@ -66,7 +69,12 @@ class ProgramSession : public TrackableProgram{ public: // Setup - ProgramSession(const ProgramDescriptor& descriptor); + // `run_records` is where this session's runs are recorded (RunRecord.h); + // a test passes its own store. + ProgramSession( + const ProgramDescriptor& descriptor, + RunRecordStore& run_records = app_run_records() + ); // Before the child-most class begins destruction, you must first: // 1. Stop the program. @@ -104,6 +112,10 @@ class ProgramSession : public TrackableProgram{ Mutex& program_lock() const{ return m_lock; } + // The program's progress hook writes to this session's active run. + friend void record_run_event(ProgramEnvironment& env, RunEvent event); + + public: // Program Control @@ -114,8 +126,8 @@ class ProgramSession : public TrackableProgram{ std::string stop_program(); // Returns before the program is completely stopped. // Temporary - virtual void async_start() override{ start_program(); } - virtual void async_stop() override{ stop_program(); } + virtual std::string async_start() override{ return start_program(); } + virtual std::string async_stop() override{ return stop_program(); } protected: @@ -123,6 +135,17 @@ class ProgramSession : public TrackableProgram{ virtual void internal_run_program(ProgramEnvironment& env) = 0; virtual void internal_stop_program(); + // Starts the program thread. A test fake makes this fail. + virtual AsyncTask dispatch_program_thread(std::function body); + + // The program's settings as JSON text, recorded with each run: what the + // settings file stores for it. "" for a session without settings. + virtual std::string settings_json() const{ return ""; } + + // The logger the session's consoles chain off, so that their lines land + // in the active run's log (RunRecord.h). + Logger& run_logger(){ return m_run_recorder; } + // virtual void restore_defaults(){ return; } public: @@ -153,13 +176,22 @@ class ProgramSession : public TrackableProgram{ private: - void run_program(); + // The program thread: runs the program and closes its run record with + // exactly one result, whichever way run_program() ends. + void run_program_and_record(); + RunResult run_program(); + std::string start_accepted_run(); + bool prepare_run(); + std::optional download_resources(); + void end_run(); + std::string abandon_run(std::string message); private: const ProgramDescriptor& m_descriptor; uint64_t m_instance_id = 0; + SessionRunRecorder m_run_recorder; // Before m_logger, which chains off it. TaggedLogger m_logger; diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.cpp b/SerialPrograms/Source/CommonFramework/RunRecord.cpp new file mode 100644 index 0000000000..e6feeef259 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord.cpp @@ -0,0 +1,247 @@ +/* Run Record + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Globals.h" +#include "RunRecord.h" + +namespace PokemonAutomation{ + + + +std::string run_status_name(RunStatus status){ + switch (status){ + case RunStatus::RUNNING: + return "running"; + case RunStatus::STOPPING: + return "stopping"; + case RunStatus::TERMINAL: + return "terminal"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown run status."); +} +std::string run_outcome_name(RunOutcome outcome){ + switch (outcome){ + case RunOutcome::COMPLETED: + return "completed"; + case RunOutcome::CANCELLED: + return "cancelled"; + case RunOutcome::FAILED: + return "failed"; + } + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown run outcome."); +} + + + +namespace{ +std::atomic& app_run_records_enabled(){ + static std::atomic enabled{false}; + return enabled; +} +} + +RunRecordStore& app_run_records(){ + static RunRecordStore store(RunRecordStore::DEFAULT_CAPACITY, &app_run_records_enabled()); + return store; +} +void enable_app_run_records(){ + app_run_records_enabled().store(true, std::memory_order_release); +} + +RunRecordStore::RunRecordStore(size_t capacity, const std::atomic* enabled) + // The app's start time, to the microsecond: two instances started in + // the same second still get different IDs. + : m_app_token(now_to_filestring()) + , m_capacity(std::max(capacity, 1)) + , m_enabled(enabled) +{} + +std::string RunRecordStore::begin(std::string program, std::string settings){ + if (m_enabled != nullptr && !m_enabled->load(std::memory_order_acquire)){ + return ""; + } + std::lock_guard lg(m_lock); + while (m_records.size() >= m_capacity){ + m_records.pop_front(); + } + RunRecord& record = m_records.emplace_back(); + record.run_id = std::format("{}-{:03}", m_app_token, ++m_counter); + record.program = std::move(program); + record.version = PROGRAM_VERSION; + record.executable = QCoreApplication::instance() == nullptr + ? "" + : QCoreApplication::applicationFilePath().toStdString(); + record.settings = std::move(settings); + record.start_time = current_time(); + return record.run_id; +} + +void RunRecordStore::stopping(const std::string& run_id){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record != nullptr && record->status == RunStatus::RUNNING){ + record->status = RunStatus::STOPPING; + } +} + +void RunRecordStore::event(const std::string& run_id, RunEvent event){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr){ + return; + } + if (record->events.size() >= MAX_EVENTS){ + record->dropped_events++; + return; + } + if (event.time == WallClock::min()){ + event.time = current_time(); + } + record->events.emplace_back(std::move(event)); + + // Keep the frames of the newest events only. + size_t frames = 0; + for (RunEvent& e : std::ranges::reverse_view(record->events)){ + if (e.frame == nullptr){ + continue; + } + if (++frames > MAX_EVENT_FRAMES){ + e.frame.reset(); + } + } +} + +void RunRecordStore::log_line(const std::string& run_id, const std::string& text){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr){ + return; + } + const size_t bytes = text.size() + 1; + if (record->log_bytes + bytes > MAX_LOG_BYTES){ + record->dropped_log_lines++; + return; + } + record->log_lines++; + record->log_bytes += bytes; + record->log_tail.emplace_back(text); + record->log_tail_bytes += bytes; + while (record->log_tail_bytes > MAX_LOG_TAIL_BYTES && record->log_tail.size() > 1){ + record->log_tail_bytes -= record->log_tail.front().size() + 1; + record->log_tail.pop_front(); + } +} + +bool RunRecordStore::finish(const std::string& run_id, RunResult result){ + std::lock_guard lg(m_lock); + RunRecord* record = find(run_id); + if (record == nullptr || record->status == RunStatus::TERMINAL){ + return false; + } + record->status = RunStatus::TERMINAL; + record->end_time = current_time(); + record->result = std::move(result); + + // Frames stay with the newest terminal runs; older ones keep the rest + // of their record. + size_t terminal = 0; + for (RunRecord& r : std::ranges::reverse_view(m_records)){ + if (r.status == RunStatus::TERMINAL && ++terminal > RUNS_WITH_FRAMES){ + release_frames(r); + } + } + return true; +} + +std::vector RunRecordStore::run_ids() const{ + std::lock_guard lg(m_lock); + std::vector ids; + for (const RunRecord& record : m_records){ + ids.emplace_back(record.run_id); + } + return ids; +} + +std::optional RunRecordStore::get(const std::string& run_id) const{ + std::lock_guard lg(m_lock); + for (const RunRecord& record : m_records){ + if (record.run_id == run_id){ + return record; + } + } + return std::nullopt; +} + +RunRecord* RunRecordStore::find(const std::string& run_id){ + auto it = std::ranges::find_if(m_records, [&](const RunRecord& r){ return r.run_id == run_id; }); + return it == m_records.end() ? nullptr : &*it; +} + +void RunRecordStore::release_frames(RunRecord& record){ + for (RunEvent& event : record.events){ + event.frame.reset(); + } + if (record.result){ + record.result->frame.reset(); + } +} + + + +SessionRunRecorder::SessionRunRecorder(RunRecordStore& store, Logger& base_logger) + : m_store(store) + , m_base_logger(base_logger) +{} + +std::string SessionRunRecorder::begin(std::string program, std::string settings){ + std::lock_guard lg(m_lock); + m_run_id = m_store.begin(std::move(program), std::move(settings)); + m_active = !m_run_id.empty(); + return m_run_id; +} +void SessionRunRecorder::stopping(){ + std::lock_guard lg(m_lock); + if (m_active){ + m_store.stopping(m_run_id); + } +} +void SessionRunRecorder::finish(RunResult result){ + std::lock_guard lg(m_lock); + if (!m_active){ + return; + } + m_active = false; + m_store.finish(m_run_id, std::move(result)); +} + +std::string SessionRunRecorder::last_run_id() const{ + std::lock_guard lg(m_lock); + return m_run_id; +} + +void SessionRunRecorder::event(RunEvent event){ + std::lock_guard lg(m_lock); + if (m_active){ + m_store.event(m_run_id, std::move(event)); + } +} +void SessionRunRecorder::log(const std::string& msg, Color color){ + m_base_logger.log(msg, color); + std::lock_guard lg(m_lock); + if (m_active){ + m_store.log_line(m_run_id, msg); + } +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/RunRecord.h b/SerialPrograms/Source/CommonFramework/RunRecord.h new file mode 100644 index 0000000000..c6f8c082f2 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord.h @@ -0,0 +1,201 @@ +/* Run Record + * + * From: https://github.com/PokemonAutomation/ + * + * What one run of a program did, as the framework saw it: when it started, + * how it ended (exactly one outcome per accepted run), the error and the + * frame it died with, the events the program reported on the way, and the + * lines of its log, and the app and settings it ran with. + * ProgramSession writes these at its execution boundary; RunRecordStore + * keeps the last few for whoever asks, such as a tool that writes them to + * files. The store records nothing until it is enabled. Nothing here + * depends on the UI, on JSON or on a game. + * + */ + +#ifndef PokemonAutomation_CommonFramework_RunRecord_H +#define PokemonAutomation_CommonFramework_RunRecord_H + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" + +namespace PokemonAutomation{ + + +enum class RunStatus{ + RUNNING, + STOPPING, // A stop was requested; the outcome is not known yet. + TERMINAL, +}; +enum class RunOutcome{ + COMPLETED, // The program returned or threw ProgramFinishedException. + CANCELLED, // The program ended through the stop path. + FAILED, // A setup, connection or execution error ended it. +}; +std::string run_status_name(RunStatus status); // "running", "stopping", "terminal" +std::string run_outcome_name(RunOutcome outcome); // "completed", "cancelled", "failed" + + +// Something the program says about its own progress: a checkpoint reached, +// a step started, an attempt that failed, a pose. Every field but `kind` +// and `text` is optional; a game fills the ones it has. (The `= {}` let a +// designated initializer leave fields out on every compiler.) +struct RunEvent{ + WallClock time = WallClock::min(); // Set by the recorder when left at min(). + std::string kind = {}; // e.g. "checkpoint", "step", "attempt", "error" + std::optional checkpoint = {}; + std::optional step = {}; + std::optional attempt = {}; + std::string text = {}; + std::shared_ptr frame = {}; // The screen at that moment, if the program had it. + std::optional pose = {}; // Where the player is, as the program puts it. +}; + +// How a run ended. +struct RunResult{ + RunOutcome outcome = RunOutcome::FAILED; + std::optional error = {}; // The primary error of a failed run. + std::shared_ptr frame = {}; // The frame the exception carried, if any. +}; + +struct RunRecord{ + std::string run_id = {}; + std::string program = {}; // The program's identifier, ":". + std::string version = {}; // PROGRAM_VERSION of the app that ran it. + std::string executable = {}; // The path of the app's executable, which names a lab build. + std::string settings = {}; // The program's settings as JSON text, as the settings file stores them. + WallClock start_time = WallClock::min(); + std::optional end_time = {}; + RunStatus status = RunStatus::RUNNING; + std::optional result = {}; // Set once, when the status turns TERMINAL. + std::vector events = {}; + size_t dropped_events = 0; // Events past RunRecordStore::MAX_EVENTS. + + // The run's log. Lines are numbered from 0 in the order they came; + // the tail holds the newest of them, so it starts at line + // `log_lines - log_tail.size()`. A reader that copies the log out as it + // grows keeps its own count of the lines it has. + std::deque log_tail = {}; // At most MAX_LOG_TAIL_BYTES, oldest first. + size_t log_tail_bytes = 0; // The tail's size, a newline per line. + size_t log_lines = 0; // Lines recorded so far. + size_t log_bytes = 0; // Their size, a newline per line, at most MAX_LOG_BYTES. + size_t dropped_log_lines = 0; // Lines past MAX_LOG_BYTES, not recorded. +}; + + +// The last few runs of this app instance. Thread-safe; every call is short +// and copies, so a reader never holds a lock while writing files. +// +// A store given an `enabled` flag records only while the flag holds: +// otherwise begin() returns "" and every call about that run is a no-op. +// The app's store (app_run_records()) records once enable_app_run_records() +// was called, by whoever reads the records, e.g. the lab channel. +// +// Run IDs are opaque strings unique to this app instance: a restart cannot +// resolve an old ID to a new run. They sort in the order the runs started. +// +// Memory: at most MAX_EVENTS events per run; the frames of a run's newest +// MAX_EVENT_FRAMES events; all frames only for the active run and the +// newest RUNS_WITH_FRAMES terminal runs; a log tail of MAX_LOG_TAIL_BYTES. +// A consumer that wants a frame or every log line reads them soon after +// they come. A run records at most MAX_LOG_BYTES of log in all. +class RunRecordStore{ +public: + static constexpr size_t DEFAULT_CAPACITY = 32; + static constexpr size_t MAX_EVENTS = 4096; + static constexpr size_t MAX_EVENT_FRAMES = 8; + static constexpr size_t RUNS_WITH_FRAMES = 4; + static constexpr size_t MAX_LOG_TAIL_BYTES = 256 * 1024; + static constexpr size_t MAX_LOG_BYTES = 8 * 1024 * 1024; + + // `enabled`, when given, must outlive the store. + explicit RunRecordStore(size_t capacity = DEFAULT_CAPACITY, const std::atomic* enabled = nullptr); + RunRecordStore(const RunRecordStore&) = delete; + void operator=(const RunRecordStore&) = delete; + + // Opens a record for an accepted start and returns its run ID, or "" + // while the store is disabled. The oldest record goes when the store is + // full. `settings` is the program's settings as JSON text. + std::string begin(std::string program, std::string settings = ""); + + // Marks a running run as stopping. No effect once it is terminal. + void stopping(const std::string& run_id); + + // Appends to the run. Both are dropped for an unknown run. + void event(const std::string& run_id, RunEvent event); + void log_line(const std::string& run_id, const std::string& text); + + // Closes the run with its one result. Returns false, and changes + // nothing, when the run is unknown or already terminal. + bool finish(const std::string& run_id, RunResult result); + + // The IDs of the records held, oldest first. + std::vector run_ids() const; + + // A copy of the record, or nothing for an unknown run. + std::optional get(const std::string& run_id) const; + + +private: + RunRecord* find(const std::string& run_id); + void release_frames(RunRecord& record); + + mutable Mutex m_lock; + const std::string m_app_token; + const size_t m_capacity; + const std::atomic* m_enabled; + uint64_t m_counter = 0; + std::deque m_records; +}; + + +// The app's store, and the switch that makes it record. Until the switch +// is thrown, no run is recorded, so an app nobody reads records from keeps +// nothing. +RunRecordStore& app_run_records(); +void enable_app_run_records(); + + +// One session's end of the store: the run active now, the events the +// program reports about it (ProgramEnvironment.h, record_run_event()), and +// a Logger that copies what passes through it into that run's log. +// The session's loggers chain off `log()`, so a run's log holds the lines +// the program and its consoles logged, and nothing from other sessions. +class SessionRunRecorder : public Logger{ +public: + SessionRunRecorder(RunRecordStore& store, Logger& base_logger); + + // Opens a run and makes it the active one. Returns its ID, or "" when + // the store is disabled, in which case nothing of it is recorded. + std::string begin(std::string program, std::string settings = ""); + void stopping(); + // Closes the active run; nothing happens when none is active. + void finish(RunResult result); + + // The run the last begin() opened, active or finished; "" before the first. + std::string last_run_id() const; + + // Adds to the active run; dropped when none is active. + void event(RunEvent event); + virtual void log(const std::string& msg, Color color = Color()) override; + +private: + RunRecordStore& m_store; + Logger& m_base_logger; + mutable Mutex m_lock; + std::string m_run_id; + bool m_active = false; +}; + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp new file mode 100644 index 0000000000..4d25c45fb1 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.cpp @@ -0,0 +1,635 @@ +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The store on its own, and the run lifecycle through a ProgramSession + * whose program is a scripted fake: no console, no window. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/Notifications/ProgramInfo.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/ProgramSession.h" +#include "CommonFramework/Tools/ProgramEnvironment.h" +#include "RunRecord.h" +#include "RunRecord_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + +using namespace std::chrono_literals; + + +std::shared_ptr make_frame(size_t width){ + ImageRGB32 image(width, width); + image.fill(0xff000000); + return std::make_shared(std::move(image)); +} + +// Polls `done` every few milliseconds until it holds or `timeout` passes. +bool wait_until(const std::function& done, Milliseconds timeout = 5000ms){ + const WallClock deadline = current_time() + timeout; + while (!done()){ + if (current_time() > deadline){ + return false; + } + std::this_thread::sleep_for(5ms); + } + return true; +} + + + +// The run the store opened last, "" when none. +std::string newest_run(const RunRecordStore& store){ + const std::vector ids = store.run_ids(); + return ids.empty() ? "" : ids.back(); +} + +UnitTestResult test_store_ids(){ + RunRecordStore store(2); + const std::string a = store.begin("Test:A", R"({"START": 3})"); + const std::string b = store.begin("Test:B"); + if (a.empty() || a == b || !(a < b)){ + return std::format("Run IDs should be distinct and sort in start order: <{}>, <{}>", a, b); + } + if (store.run_ids() != std::vector{a, b}){ + return "run_ids() lists the records oldest first."; + } + const std::optional record = store.get(a); + if (!record || record->program != "Test:A" || record->status != RunStatus::RUNNING || record->result){ + return "A new record is running, with its program and no result."; + } + if (record->start_time == WallClock::min() || record->end_time){ + return "A new record has a start time and no end time."; + } + if (record->version != PROGRAM_VERSION || (QCoreApplication::instance() != nullptr && record->executable.empty())){ + return std::format("A record names the app: version <{}>, executable <{}>.", record->version, record->executable); + } + if (record->settings != R"({"START": 3})" || !store.get(b)->settings.empty()){ + return "A record keeps the settings it was begun with."; + } + const std::string c = store.begin("Test:C"); + if (store.get(a) || store.run_ids() != std::vector{b, c}){ + return "A full store drops its oldest record."; + } + if (store.get("nobody")){ + return "An unknown run has no record."; + } + return true; +} + +UnitTestResult test_store_one_result(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.stopping(id); + if (store.get(id)->status != RunStatus::STOPPING){ + return "stopping() marks a running run."; + } + if (!store.finish(id, {.outcome = RunOutcome::FAILED, .error = "first"})){ + return "The first finish() takes."; + } + store.stopping(id); + if (store.finish(id, {.outcome = RunOutcome::COMPLETED})){ + return "A second finish() is refused."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || !record.end_time || !record.result){ + return "A finished record is terminal with an end time and a result."; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "first"){ + return std::format("The first result stands: got {}", run_outcome_name(record.result->outcome)); + } + if (store.finish("nobody", {})){ + return "Finishing an unknown run is refused."; + } + return true; +} + +UnitTestResult test_store_events(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + store.event(id, {.kind = "checkpoint", .checkpoint = "00.01", .text = "ok"}); + store.event("nobody", {.kind = "checkpoint"}); + for (size_t c = 0; c < RunRecordStore::MAX_EVENT_FRAMES + 2; c++){ + store.event(id, {.kind = "step", .step = std::to_string(c), .frame = make_frame(4)}); + } + RunRecord record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENT_FRAMES + 3){ + return std::format("Expected {} events, got {}.", RunRecordStore::MAX_EVENT_FRAMES + 3, record.events.size()); + } + if (record.events[0].time == WallClock::min() || record.events[0].checkpoint != "00.01"){ + return "An event gets the time it was recorded and keeps its fields."; + } + if (record.events[1].frame || record.events[2].frame || !record.events[3].frame || !record.events.back().frame){ + return "Only the newest MAX_EVENT_FRAMES events keep their frames."; + } + + for (size_t c = record.events.size(); c < RunRecordStore::MAX_EVENTS + 5; c++){ + store.event(id, {.kind = "step"}); + } + record = *store.get(id); + if (record.events.size() != RunRecordStore::MAX_EVENTS || record.dropped_events != 5){ + return std::format("Events past the cap are counted, not kept: {} events, {} dropped.", record.events.size(), record.dropped_events); + } + return true; +} + +UnitTestResult test_store_log(){ + RunRecordStore store; + const std::string id = store.begin("Test:A"); + RunRecord record; + + // Lines of 1000 bytes, numbered: 300 are more than the tail holds. + const std::string line(995, 'x'); + for (size_t c = 0; c < 300; c++){ + store.log_line(id, std::format("{:04} {}", c, line)); + } + record = *store.get(id); + size_t bytes = 0; + for (const std::string& l : record.log_tail){ + bytes += l.size() + 1; + } + if (record.log_lines != 300 || record.log_bytes != 300 * 1001 || record.dropped_log_lines != 0){ + return std::format("Every line is counted: {} lines, {} bytes.", record.log_lines, record.log_bytes); + } + if (bytes != record.log_tail_bytes || bytes > RunRecordStore::MAX_LOG_TAIL_BYTES || record.log_tail.size() == 300){ + return std::format("The tail keeps at most {} bytes: {} lines, {} bytes.", RunRecordStore::MAX_LOG_TAIL_BYTES, record.log_tail.size(), bytes); + } + const size_t first = record.log_lines - record.log_tail.size(); + if (record.log_tail.front().rfind(std::format("{:04} ", first), 0) != 0 || record.log_tail.back().rfind("0299 ", 0) != 0){ + return "The tail holds the newest lines, from line log_lines - log_tail.size()."; + } + + // Past MAX_LOG_BYTES in all, lines are counted and not kept. + const size_t fit = RunRecordStore::MAX_LOG_BYTES / 1001; + for (size_t c = 300; c < fit + 10; c++){ + store.log_line(id, std::format("{:04} {}", c % 10000, line)); + } + record = *store.get(id); + if (record.log_lines != fit || record.dropped_log_lines != 10 || record.log_bytes > RunRecordStore::MAX_LOG_BYTES){ + return std::format("A run records at most {} bytes of log: {} lines, {} dropped.", RunRecordStore::MAX_LOG_BYTES, record.log_lines, record.dropped_log_lines); + } + return true; +} + +UnitTestResult test_store_disabled(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + std::atomic enabled{false}; + RunRecordStore store(4, &enabled); + if (!store.begin("Test:A").empty() || !store.run_ids().empty()){ + return "A disabled store opens no record."; + } + CountingLogger base; + SessionRunRecorder recorder(store, base); + if (!recorder.begin("Test:A").empty() || !recorder.last_run_id().empty()){ + return "A recorder over a disabled store has no run."; + } + recorder.log("line"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + if (base.lines != 1 || !store.run_ids().empty()){ + return "Lines still reach the base logger; nothing is recorded."; + } + enabled.store(true); + const std::string id = recorder.begin("Test:A"); + recorder.log("line"); + if (id.empty() || store.get(id)->log_lines != 1){ + return "Once enabled, the store records."; + } + return true; +} + +UnitTestResult test_store_releases_old_frames(){ + RunRecordStore store; + std::vector ids; + for (size_t c = 0; c < RunRecordStore::RUNS_WITH_FRAMES + 1; c++){ + ids.emplace_back(store.begin("Test:A")); + store.event(ids.back(), {.kind = "step", .frame = make_frame(4)}); + store.finish(ids.back(), {.outcome = RunOutcome::FAILED, .error = "e", .frame = make_frame(4)}); + } + const RunRecord oldest = *store.get(ids.front()); + if (oldest.events[0].frame || oldest.result->frame){ + return "A run older than the newest RUNS_WITH_FRAMES terminal runs keeps no frames."; + } + if (oldest.result->error != "e" || oldest.events.size() != 1){ + return "It keeps the rest of its record."; + } + const RunRecord newest = *store.get(ids.back()); + if (!newest.events[0].frame || !newest.result->frame){ + return "The newest terminal runs keep their frames."; + } + return true; +} + +UnitTestResult test_session_recorder(){ + struct CountingLogger : public Logger{ + size_t lines = 0; + virtual void log(const std::string&, Color) override{ lines++; } + }; + RunRecordStore store; + CountingLogger base; + SessionRunRecorder recorder(store, base); + recorder.log("before"); + recorder.event({.kind = "lost"}); + if (!recorder.last_run_id().empty() || !store.run_ids().empty()){ + return "Nothing is recorded before a run begins."; + } + const std::string id = recorder.begin("Test:A"); + if (recorder.last_run_id() != id){ + return "last_run_id() is the run begin() opened."; + } + recorder.log("during"); + recorder.event({.kind = "checkpoint"}); + recorder.finish({.outcome = RunOutcome::COMPLETED}); + recorder.log("after"); + recorder.event({.kind = "late"}); + recorder.finish({.outcome = RunOutcome::FAILED}); + const RunRecord record = *store.get(id); + if (base.lines != 3){ + return std::format("Every line reaches the base logger: {} of 3.", base.lines); + } + if (record.log_tail != std::deque{"during"} || record.events.size() != 1){ + return "Only lines and events during the run are recorded."; + } + if (record.result->outcome != RunOutcome::COMPLETED || recorder.last_run_id() != id){ + return "The result stands and the run stays known after finish()."; + } + return true; +} + + + +// A program with no console: what it does is scripted per test. +struct FakeScript{ + std::string validity = {}; // check_validity(): "" accepts the start. + bool fail_dispatch = false; // The program thread cannot be started. + std::function start_checks = {}; + std::function body = {}; + std::string settings = {}; // settings_json(). +}; + +class FakeDescriptor : public ProgramDescriptor{ +public: + FakeDescriptor() + : ProgramDescriptor( + "Test:RunRecordFake", "Test", "Run Record Fake", + "", "", PanelQualityStanding::DEVELOPER, false, {} + ) + {} + virtual std::unique_ptr make_panel() const override{ return nullptr; } +}; + +class FakeInstance : public ProgramInstance{ +public: + explicit FakeInstance(std::function start_checks) + : m_start_checks(std::move(start_checks)) + {} + virtual void run_start_program_checks(const ProgramDescriptor&, ProgramEnvironment& env) override{ + if (m_start_checks){ + m_start_checks(env); + } + } +private: + std::function m_start_checks; +}; + +class FakeProgram : public ProgramSession{ +public: + FakeProgram(const FakeDescriptor& descriptor, RunRecordStore& store, FakeScript script) + : ProgramSession(descriptor, store) + , m_script(std::move(script)) + { + m_instance = std::make_unique(m_script.start_checks); + } + virtual ~FakeProgram(){ + internal_stop_program(); + join_program_thread(); + } + + // Whether the body has started running. + bool running() const{ return m_running.load(std::memory_order_acquire); } + + // Waits for the program thread to end, so the run's record is closed. + bool wait_until_stopped(){ + if (!wait_until([this]{ return current_state() == ProgramState::STOPPED; })){ + return false; + } + join_program_thread(); + return true; + } + + virtual std::string check_validity() const override{ return m_script.validity; } + +protected: + virtual std::unique_ptr make_env(const ProgramInfo& program_info) override{ + return std::make_unique(program_info, *this, current_stats_tracker(), historical_stats_tracker()); + } + virtual void internal_run_program(ProgramEnvironment& env) override{ + m_running.store(true, std::memory_order_release); + if (m_script.body){ + m_script.body(*this, env, *m_scope); + } + } + virtual std::string settings_json() const override{ return m_script.settings; } + virtual AsyncTask dispatch_program_thread(std::function body) override{ + if (m_script.fail_dispatch){ + throw std::runtime_error("no thread"); + } + return ProgramSession::dispatch_program_thread(std::move(body)); + } + +private: + FakeScript m_script; + std::atomic m_running{false}; +}; + +// Starts the program, waits for it to end, and returns its record. +UnitTestResult run_to_end(FakeProgram& program, RunRecordStore& store, RunRecord& record){ + const std::string error = program.start_program(); + if (!error.empty()){ + return "The start was refused: " + error; + } + if (!program.wait_until_stopped()){ + return "The program did not stop."; + } + const std::optional found = store.get(newest_run(store)); + if (!found){ + return "The run has no record."; + } + record = *found; + if (record.status != RunStatus::TERMINAL || !record.result){ + return "The record is not terminal after the program stopped."; + } + return true; +} + +UnitTestResult test_rejection(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.validity = "Pick a save."}); + if (program.start_program() != "Pick a save."){ + return "A start refused by check_validity() returns its message."; + } + if (!store.run_ids().empty() || !newest_run(store).empty()){ + return "A refused start opens no run."; + } + return true; +} + +UnitTestResult test_immediate_completion(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment& env, CancellableScope&){ + record_run_event(env, {.kind = "checkpoint", .checkpoint = "00.01", .step = "3/3", .attempt = 1, .text = "ok", .pose = "home (2, 4) N"}); + }, + .settings = R"({"START": 1})", + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::COMPLETED || record.result->error || record.result->frame){ + return "A program that returns completed its run, with no error and no frame."; + } + if (record.program != "Test:RunRecordFake" || record.start_time > *record.end_time){ + return "The record names the program and its times."; + } + if (record.settings != R"({"START": 1})" || record.version != PROGRAM_VERSION){ + return "The record has the session's settings and the app's version."; + } + if (record.events.size() != 1 || record.events[0].checkpoint != "00.01" || record.events[0].attempt != size_t(1) || record.events[0].pose != "home (2, 4) N"){ + return std::format("The program's event is in the record: {} events.", record.events.size()); + } + bool stopped_logged = false; + for (const std::string& line : record.log_tail){ + stopped_logged |= line.find("Program State: STOPPED") != std::string::npos; + } + if (!stopped_logged){ + return "The run's log tail runs through the session's last state change."; + } + return true; +} + +UnitTestResult test_setup_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .start_checks = [](ProgramEnvironment& env){ + throw UserSetupError(env.logger(), "The controller is not ready."); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The controller is not ready."){ + return "A setup error fails the run with its message."; + } + if (record.events.size() != 1 || record.events[0].kind != "error" || record.events[0].text != "The controller is not ready."){ + return "The reported error is an \"error\" event."; + } + return true; +} + +UnitTestResult test_failure_with_frame(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + ImageRGB32 screen(8, 8); + screen.fill(0xffff0000); + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "The sign's box did not open.", nullptr, std::move(screen) + ); + }, + }); + RunRecord record; + UnitTestResult result = run_to_end(program, store, record); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (record.result->outcome != RunOutcome::FAILED || record.result->error != "The sign's box did not open."){ + return "The exception's message is the primary error."; + } + if (!record.result->frame || record.result->frame->width() != 8){ + return "The exception's frame is kept with the result."; + } + return true; +} + +UnitTestResult test_cancellation(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope& scope){ + while (true){ + scope.throw_if_cancelled(); + std::this_thread::sleep_for(5ms); + } + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = newest_run(store); + if (!program.stop_program().empty() || !program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.status != RunStatus::TERMINAL || record.result->outcome != RunOutcome::CANCELLED || record.result->error){ + return "A stopped program's run is cancelled, with no error."; + } + return true; +} + +// The body waits for the stop request and then ends its own way. +UnitTestResult run_racing_a_stop(const std::function& ending, RunOutcome expected, const char* what){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, { + .body = [&](ProgramSession& self, ProgramEnvironment&, CancellableScope&){ + wait_until([&]{ return self.current_state() == ProgramState::STOPPING; }); + ending(); + }, + }); + if (!program.start_program().empty() || !wait_until([&]{ return program.running(); })){ + return "The program should start and run."; + } + const std::string id = newest_run(store); + program.stop_program(); + if (!program.wait_until_stopped()){ + return "The program should stop."; + } + const RunRecord record = *store.get(id); + if (record.result->outcome != expected){ + return std::format("{}: expected {}, got {}.", what, run_outcome_name(expected), run_outcome_name(record.result->outcome)); + } + return true; +} +UnitTestResult test_late_stop(){ + UnitTestResult result = run_racing_a_stop( + []{ throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "Failed anyway."); }, + RunOutcome::FAILED, "A failure observed after the stop request stays a failure" + ); + if (result.result != UnitTestResult::PASSED){ + return result; + } + return run_racing_a_stop( + []{}, RunOutcome::COMPLETED, "A normal return racing the stop request stays completed" + ); +} + +UnitTestResult test_dispatch_failure(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram program(descriptor, store, {.fail_dispatch = true}); + const std::string error = program.start_program(); + if (error.find("could not be started") == std::string::npos){ + return "A start whose thread cannot be dispatched returns an error: <" + error + ">"; + } + const std::optional record = store.get(newest_run(store)); + if (!record || record->status != RunStatus::TERMINAL || record->result->outcome != RunOutcome::FAILED){ + return "The accepted run is closed as failed."; + } + if (record->result->error != error){ + return "The run's error is the start's error."; + } + if (program.current_state() != ProgramState::STOPPED){ + return "The session is stopped again, so the next start can try."; + } + return true; +} + +UnitTestResult test_second_run_preserves_first(){ + RunRecordStore store; + FakeDescriptor descriptor; + FakeProgram failing(descriptor, store, { + .body = [](ProgramSession&, ProgramEnvironment&, CancellableScope&){ + throw OperationFailedExceptionWithScreenshot( + ErrorReportMode::NO_ERROR_REPORT, "first", nullptr, ImageRGB32(8, 8) + ); + }, + }); + RunRecord first; + UnitTestResult result = run_to_end(failing, store, first); + if (result.result != UnitTestResult::PASSED){ + return result; + } + FakeProgram completing(descriptor, store, {}); + RunRecord second; + result = run_to_end(completing, store, second); + if (result.result != UnitTestResult::PASSED){ + return result; + } + if (first.run_id == second.run_id || !(first.run_id < second.run_id)){ + return "Each run has its own ID, sorting after the earlier one."; + } + const std::optional kept = store.get(first.run_id); + if (!kept || kept->result->outcome != RunOutcome::FAILED || kept->result->error != "first" || !kept->result->frame){ + return "The first run's result and frame are still there after the second run."; + } + if (second.result->outcome != RunOutcome::COMPLETED || store.run_ids() != std::vector{first.run_id, second.run_id}){ + return "The second run has its own, completed, record."; + } + return true; +} + + +class Test_RunRecord : public UnitTest{ +public: + Test_RunRecord(std::string name, std::function body) + : UnitTest("CommonFramework::RunRecord - " + std::move(name)) + , m_body(std::move(body)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_body(); + } + +private: + std::function m_body; +}; + +} + + +void add_tests_RunRecord(UnitTestDatabase& database){ + database.add("store: ids and capacity", test_store_ids); + database.add("store: one result per run", test_store_one_result); + database.add("store: events and frames", test_store_events); + database.add("store: numbered log and its cap", test_store_log); + database.add("store: old runs release frames", test_store_releases_old_frames); + database.add("store: disabled", test_store_disabled); + database.add("session recorder", test_session_recorder); + database.add("lifecycle: rejected start", test_rejection); + database.add("lifecycle: immediate completion", test_immediate_completion); + database.add("lifecycle: setup failure", test_setup_failure); + database.add("lifecycle: failure with frame", test_failure_with_frame); + database.add("lifecycle: cancellation", test_cancellation); + database.add("lifecycle: late stop", test_late_stop); + database.add("lifecycle: dispatch failure", test_dispatch_failure); + database.add("lifecycle: second run preserves the first", test_second_run_preserves_first); +} + + + +} diff --git a/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h new file mode 100644 index 0000000000..9a3b88afcd --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/RunRecord_Tests.h @@ -0,0 +1,20 @@ +/* Run Record - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonFramework_RunRecord_Tests_H +#define PokemonAutomation_CommonFramework_RunRecord_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +void add_tests_RunRecord(UnitTestDatabase& database); + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp index 07e95a1a16..44c8958ff2 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.cpp @@ -33,6 +33,10 @@ const ProgramInfo& ProgramEnvironment::program_info() const{ return m_program_info; } +void record_run_event(ProgramEnvironment& env, RunEvent event){ + env.m_session.m_run_recorder.event(std::move(event)); +} + void ProgramEnvironment::update_stats(){ m_session.report_stats_changed(); diff --git a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h index 72d6d812df..6dc5709dd6 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h +++ b/SerialPrograms/Source/CommonFramework/Tools/ProgramEnvironment.h @@ -9,6 +9,7 @@ #include "Common/Compiler.h" #include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/RunRecord.h" namespace PokemonAutomation{ @@ -42,6 +43,9 @@ class ProgramEnvironment{ virtual void log_to_ui(const std::string& msg, Color color = Color()){} + friend void record_run_event(ProgramEnvironment& env, RunEvent event); + + public: // Stats Management @@ -67,6 +71,13 @@ class ProgramEnvironment{ +// Progress reporting (RunRecord.h): what the program says about its +// checkpoints, steps and attempts goes to this run's record, if the app +// records runs. +void record_run_event(ProgramEnvironment& env, RunEvent event); + + + // Templates template diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index b5905f9836..c75135d5cb 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -23,6 +23,7 @@ #include "CommonFramework/AudioPipeline/Spectrum/FFTStreamer.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ProgramStats/StatsTracking.h" +#include "CommonFramework/RunRecord_Tests.h" #include "CommonFramework/Tools/GlobalThreadPools.h" #include "CommonFramework/Windows/LabControl.h" #include "CommonTools/VisualDetectors/BlackBorderDetector.h" @@ -73,6 +74,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ add_tests_SinkPipeline(ret); add_tests_AudioFileLoader(ret); add_tests_LabControl(ret); + add_tests_RunRecord(ret); OCR::add_tests(ret); Kernels::add_tests(ret); Pokemon::add_tests_AutoStorySelect(ret); diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp index 2cf4ceb24d..d56f6c79c4 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsolePanelSession.cpp @@ -7,6 +7,7 @@ #include "Common/Cpp/EarlyShutdown.h" #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Logging/GlobalLogger.h" #include "ConsolePanelSession.h" namespace PokemonAutomation{ @@ -28,7 +29,7 @@ ConsolePanelSession::ConsolePanelSession(const ConsolePanelDescriptor& descripto : UiState(descriptor) , m_descriptor(descriptor) , m_system_option(descriptor.num_controllers()) - , m_system(m_system_option, true, 0) + , m_system(global_logger_raw(), m_system_option, true, 0) , m_instance(descriptor.make_instance(m_system)) {} diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp index f8c81bbd7a..af14b9b2db 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.cpp @@ -39,6 +39,7 @@ ConsoleProgramSession::ConsoleProgramSession( , m_descriptor(descriptor) , m_system_option(1, std::move(extra_option)) , m_system( + run_logger(), m_system_option, descriptor.allow_commands_while_running(), 0, @@ -62,6 +63,9 @@ void ConsoleProgramSession::restore_defaults(){ logger().log("Restoring settings to defaults..."); m_instance->restore_defaults(); } +std::string ConsoleProgramSession::settings_json() const{ + return to_json().dump(); +} JsonValue ConsoleProgramSession::to_json() const{ JsonObject obj = std::move(*m_instance->to_json().to_object()); obj["ConsoleSetup"] = m_system_option.to_json(); diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.h b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.h index 97b373df80..50c78e61fe 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleProgramSession.h @@ -45,6 +45,7 @@ class ConsoleProgramSession protected: virtual std::unique_ptr make_env(const ProgramInfo& program_info) override; virtual void internal_run_program(ProgramEnvironment& env) override; + virtual std::string settings_json() const override; private: diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp index 057853192f..ae0e978c1d 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp @@ -50,6 +50,7 @@ ConsoleSystemSession::~ConsoleSystemSession(){ ); } ConsoleSystemSession::ConsoleSystemSession( + Logger& base_logger, ConsoleSystemOption& option, bool allow_commands_while_locked, size_t console_number, @@ -57,7 +58,7 @@ ConsoleSystemSession::ConsoleSystemSession( ) : m_allow_commands_while_locked(allow_commands_while_locked) , m_console_number(console_number) - , m_logger(global_logger_raw(), "Console " + std::to_string(console_number)) + , m_logger(base_logger, "Console " + std::to_string(console_number)) , m_option(option) , m_video(m_logger, option.m_video) , m_audio(m_logger, option.m_audio) @@ -78,10 +79,7 @@ ConsoleSystemSession::ConsoleSystemSession( update_status(); m_history.set_audio_session(m_audio); - m_history.start( - m_audio.input_choice(), - m_video.current_source() != nullptr - ); + m_history.start(m_audio.input_choice(), m_video.current_source() != nullptr); try{ m_overlay.add_stat(m_memory_usage->m_system); diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h index 62ec876c71..8c08e359a8 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h @@ -66,7 +66,11 @@ class ConsoleSystemSession final virtual bool try_shutdown() noexcept override; ~ConsoleSystemSession(); + // `base_logger` is what this console's logger writes through: a + // program session passes its run logger (ProgramSession::run_logger()), + // so the console's lines land in the run's record. ConsoleSystemSession( + Logger& base_logger, ConsoleSystemOption& option, bool allow_commands_while_locked, size_t console_number, diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsolePanelSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsolePanelSession.cpp index 0b42002c1c..2de1af248c 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsolePanelSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsolePanelSession.cpp @@ -7,6 +7,7 @@ #include "Common/Cpp/EarlyShutdown.h" #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/Logging/Logger.h" #include "MultiConsolePanelSession.h" @@ -34,7 +35,7 @@ MultiConsolePanelSession::MultiConsolePanelSession(const MultiConsolePanelDescri descriptor.default_consoles(), descriptor.option_factory() ) - , m_system(m_system_option, true) + , m_system(global_logger_raw(), m_system_option, true) , m_instance(descriptor.make_instance(m_system)) {} diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp index 3f6cbb6e18..b6d6c62b52 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.cpp @@ -47,7 +47,7 @@ MultiConsoleProgramSession::MultiConsoleProgramSession( descriptor.default_consoles() ) ) - , m_system(*m_system_option, descriptor.allow_commands_while_running(), instance_id()) + , m_system(run_logger(), *m_system_option, descriptor.allow_commands_while_running(), instance_id()) { auto instance = descriptor.make_instance(m_system); instance->update_active_consoles(m_system_option->active_consoles()); @@ -72,6 +72,9 @@ void MultiConsoleProgramSession::restore_defaults(){ logger().log("Restoring settings to defaults..."); m_instance->restore_defaults(); } +std::string MultiConsoleProgramSession::settings_json() const{ + return to_json().dump(); +} JsonValue MultiConsoleProgramSession::to_json() const{ JsonObject obj = std::move(*m_instance->to_json().to_object()); obj["ConsoleSetup"] = m_system_option->to_json(); diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.h b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.h index 44fd2f03f4..3230148f11 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleProgramSession.h @@ -46,6 +46,7 @@ class MultiConsoleProgramSession protected: virtual std::unique_ptr make_env(const ProgramInfo& program_info) override; virtual void internal_run_program(ProgramEnvironment& env) override; + virtual std::string settings_json() const override; virtual void on_console_count_lock(bool locked) override{} virtual void shutdown() override; diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp index dde05b293c..613d62dabf 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.cpp @@ -40,11 +40,13 @@ MultiConsoleSystemSession::~MultiConsoleSystemSession(){ } MultiConsoleSystemSession::MultiConsoleSystemSession( + Logger& base_logger, MultiConsoleSystemOption& option, bool allow_commands_while_locked, std::optional program_tracking_id ) : m_logger(global_logger_tagged()) + , m_base_logger(base_logger) , m_option(option) , m_allow_commands_while_locked(allow_commands_while_locked) , m_program_tracking_id(program_tracking_id) @@ -54,6 +56,7 @@ MultiConsoleSystemSession::MultiConsoleSystemSession( for (size_t c = 0; c < count; c++){ m_consoles.emplace_back( std::make_unique( + m_base_logger, option[c], allow_commands_while_locked, c, @@ -121,6 +124,7 @@ void MultiConsoleSystemSession::set_active_consoles(size_t count){ for (size_t c = 0; c < count; c++){ m_consoles.emplace_back( std::make_unique( + m_base_logger, m_option[c], m_allow_commands_while_locked, c, diff --git a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h index e31786be68..117ccf260d 100644 --- a/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h +++ b/SerialPrograms/Source/GameConsole/Framework/MultiConsoleSystemSession.h @@ -47,7 +47,10 @@ class MultiConsoleSystemSession // Construct the multi-console session. // By default it constructs the generic console. But you can inject your // own factory for your own custom subclass. + // `base_logger` is what the consoles' loggers write through; see + // ConsoleSystemSession. MultiConsoleSystemSession( + Logger& base_logger, MultiConsoleSystemOption& option, bool allow_commands_while_locked, std::optional program_tracking_id = {} @@ -85,6 +88,7 @@ class MultiConsoleSystemSession private: Logger& m_logger; + Logger& m_base_logger; // What the consoles' loggers write through. MultiConsoleSystemOption& m_option; bool m_allow_commands_while_locked; diff --git a/SerialPrograms/Source/Integrations/ProgramTracker.cpp b/SerialPrograms/Source/Integrations/ProgramTracker.cpp index ca3c242416..b7734cb55c 100644 --- a/SerialPrograms/Source/Integrations/ProgramTracker.cpp +++ b/SerialPrograms/Source/Integrations/ProgramTracker.cpp @@ -173,8 +173,7 @@ std::string ProgramTracker::start_program(uint64_t program_id){ if (program == nullptr){ return error; } - program->async_start(); - return ""; + return program->async_start(); } std::string ProgramTracker::stop_program(uint64_t program_id){ std::string header = make_header("stop_program", program_id); @@ -184,8 +183,7 @@ std::string ProgramTracker::stop_program(uint64_t program_id){ if (program == nullptr){ return error; } - program->async_stop(); - return ""; + return program->async_stop(); } std::string ProgramTracker::run_controller_command( uint64_t console_id, uint64_t controller_index, diff --git a/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h b/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h index 9d4f4c3125..e771bb80cb 100644 --- a/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h +++ b/SerialPrograms/Source/Integrations/ProgramTrackerInterfaces.h @@ -36,8 +36,8 @@ class TrackableProgram{ virtual ProgramState current_state() const = 0; virtual std::string current_stats() const = 0; - virtual void async_start() = 0; - virtual void async_stop() = 0; + virtual std::string async_start() = 0; + virtual std::string async_stop() = 0; }; diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp index 2d107709b9..07b85fbec2 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp @@ -6,6 +6,7 @@ #include "Common/Compiler.h" #include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" @@ -274,7 +275,7 @@ class Test_UpdatePopupDetector : public ImageUnitTest{ const std::vector& images ) const override{ GameConsole::ConsoleSystemOption option(1); - GameConsole::ConsoleSystemSession session(option, false, 0, {}); + GameConsole::ConsoleSystemSession session(global_logger_raw(), option, false, 0, {}); ConsoleHandle console(session); UpdatePopupDetector detector(console); diff --git a/SerialPrograms/Source/VideoGames/PokemonLA/Inference/Sounds/PokemonLA_ShinySoundDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLA/Inference/Sounds/PokemonLA_ShinySoundDetector.cpp index b58b5f526e..40103a79c1 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLA/Inference/Sounds/PokemonLA_ShinySoundDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLA/Inference/Sounds/PokemonLA_ShinySoundDetector.cpp @@ -4,6 +4,7 @@ * */ +#include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonTools/Audio/SpectrogramMatcher.h" #include "CommonTools/Audio/AudioTemplateCache.h" #include "Tests/TestUtils.h" @@ -55,7 +56,7 @@ class Test_ShinySoundDetector : public UnitTest{ virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ GameConsole::ConsoleSystemOption option(1); - GameConsole::ConsoleSystemSession session(option, false, 0, {}); + GameConsole::ConsoleSystemSession session(global_logger_raw(), option, false, 0, {}); ConsoleHandle console(session); ShinySoundDetector detector(console, [&](float error_coefficient) -> bool{ return true; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 1603b71bc5..edf2aa8484 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -687,6 +687,10 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/PersistentSettings.h Source/CommonFramework/ProgramSession.cpp Source/CommonFramework/ProgramSession.h + Source/CommonFramework/RunRecord.cpp + Source/CommonFramework/RunRecord.h + Source/CommonFramework/RunRecord_Tests.cpp + Source/CommonFramework/RunRecord_Tests.h Source/CommonFramework/ProgramStats/StatsDatabase.cpp Source/CommonFramework/ProgramStats/StatsDatabase.h Source/CommonFramework/ProgramStats/StatsTracking.cpp