diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp new file mode 100644 index 0000000000..4be721658c --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp @@ -0,0 +1,177 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "ScreenMachine.h" + +namespace PokemonAutomation{ + +using namespace std::chrono_literals; + + +namespace{ + +const ScreenState* classify(const std::vector& states, const ImageViewRGB32& screen){ + for (const ScreenState& state : states){ + if (state.matches && state.matches(screen)){ + return &state; + } + } + return nullptr; +} + +} // namespace + + +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b){ + if (!a || !b || a->width() != b->width() || a->height() != b->height()){ + return false; + } + return ImageMatch::pixel_RMSD(a, b) < 2.0; +} + +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation){ + for (const ScreenState& state : states){ + if (state.matches_observation && state.matches_observation(observation)){ + return &state; + } + } + return nullptr; +} + + +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, + const std::vector& states, StepTrace& trace, VideoSnapshot& frame +){ + const ScreenState* last = nullptr; + WallClock video_lost = WallClock::max(); + for (size_t c = 0; c < 20; c++){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + // The app resets a frozen capture by itself, which takes a second or two. + const WallClock now = current_time(); + video_lost = std::min(video_lost, now); + if (now - video_lost > std::chrono::seconds(10)){ + trace.fail("No video from the capture card for 10s."); + } + c--; + scope.wait_for(500ms); + continue; + } + video_lost = WallClock::max(); + const ScreenState* now = classify(states, screen); + frame = std::move(screen); + if (now != nullptr && now == last){ + return *now; + } + last = now; + scope.wait_for(150ms); + } + if (last == nullptr){ + trace.fail("The screen matches none of the known states."); + } + return *last; // Still changing after 3s: go with the latest read. +} + + +void ScreenMachineChecks::check(const ScreenState& state, VideoSnapshot frame){ + const std::optional found = problem(state, std::move(frame), current_time()); + if (found){ + m_trace.fail(*found); + } +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, const ScreenObservation& observation){ + return problem(state, observation.frame ? *observation.frame : VideoSnapshot(), observation.when); +} + +std::optional ScreenMachineChecks::unchanged_problem(const ScreenState& state, WallClock when){ + if (state.max_still > Milliseconds::zero()){ + if (when - m_changed_at > state.max_still){ + return std::format("The screen stayed still on \"{}\" for more than {}s.", state.name, state.max_still.count() / 1000); + } + return std::nullopt; + } + if (++m_unchanged >= MAX_UNCHANGED){ + return std::format("The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name); + } + return std::nullopt; +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, VideoSnapshot frame, WallClock when){ + if (same_screen(m_last_frame, frame)){ + std::optional stalled = unchanged_problem(state, when); + if (stalled){ + return stalled; + } + }else{ + m_unchanged = 0; + m_changed_at = when; + } + m_last_frame = std::move(frame); + + if (m_previous != nullptr && m_previous != &state && !m_previous->next.empty() && + std::find(m_previous->next.begin(), m_previous->next.end(), state.name) == m_previous->next.end() + ){ + return std::format("Unexpected screen after \"{}\": \"{}\".", m_previous->name, state.name); + } + m_in_a_row = m_previous == &state ? m_in_a_row + 1 : 1; + if (m_in_a_row > state.max_in_a_row){ + return std::format("Screen \"{}\" came up {} times in a row: the step for it is not getting anywhere.", state.name, m_in_a_row); + } + if (++m_visits[state.name] > state.max_visits){ + return std::format("Screen \"{}\" came up more than {} times: the step for it is not working.", state.name, state.max_visits); + } + m_previous = &state; + return std::nullopt; +} + + +ScreenMachineResult run_screen_machine_on_observations( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +){ + // Looks before the run stops even when nothing else ends it (looks with + // no time, states with no visit limit). + constexpr size_t MAX_LOOKS = 10000; + ScreenMachineChecks checks(trace); + std::optional start; + for (size_t looks = 0; looks < MAX_LOOKS; looks++){ + scope.throw_if_cancelled(); + const ScreenObservation observation = observe(); + if (!start){ + start = observation.when; + } + if (observation.when - *start > timeout){ + return {.terminal = "", .failure = std::format("Did not finish within {}s.", timeout.count() / 1000)}; + } + const ScreenState* state = match_screen_state(states, observation); + if (state == nullptr){ + return {.terminal = "", .failure = "The screen matches none of the known states."}; + } + trace.screen(state->name); + std::optional found = checks.problem(*state, observation); + if (found){ + return {.terminal = "", .failure = std::move(*found)}; + } + if (state->act){ + state->act(); + } + if (state->terminal){ + return {.terminal = state->name, .failure = ""}; + } + } + return {.terminal = "", .failure = std::format("Did not finish within {} looks.", MAX_LOOKS)}; +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h new file mode 100644 index 0000000000..4f11d7066e --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h @@ -0,0 +1,169 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + * A closed loop over known screens: read the screen, take the one step that screen + * calls for, read again. Every step follows a read, a screen counts only once two + * frames agree, and each state lists the screens it may lead to, so a dropped or + * wrong input shows up as an unexpected screen or a state that keeps coming back. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_H +#define PokemonAutomation_CommonTools_ScreenMachine_H + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/CancellableScope.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "ScreenObservation.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +// Single recognizable screen state, including detection predicate and handler action. +struct ScreenState{ + std::string name; + // Whether this is the screen. States are tried in order; the first match wins, so + // the last state should match anything. + std::function matches; + // Whether this is the screen, from what a look at the game observed + // instead of from its pixels, so the state also runs in a simulation. + // A state may have either or both; read_screen_state() uses `matches` + // and skips a state without it. + std::function matches_observation = nullptr; + // The one step to take on this screen. + std::function act; + // The screens the step may lead to besides staying here. Empty: any. + std::vector next; + // Fail once the screen has been seen this many times. + size_t max_visits; + // End the run after this state's step. + bool terminal = false; + // Fail once the screen has been seen this many times in a row. + size_t max_in_a_row = SIZE_MAX; + // How long the screen may stay unchanged on this state. A waiting + // state (a fade, a still cutscene, a hold before a screen counts) sets + // it: its looks never count as a stall, and it fails only once the + // screen has not changed for longer than this. Zero: the state's step + // is input that should change the screen, and MAX_UNCHANGED such + // steps that do not are a stall. + Milliseconds max_still = Milliseconds::zero(); +}; + + +// The state on screen, once two frames 150ms apart agree. Waits out a video +// gap of up to 10s. Fails through `trace` if no state matches. +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, + const std::vector& states, StepTrace& trace, VideoSnapshot& frame +); + +// The first of `states` whose matches_observation() accepts `observation`; +// null when none does. States without one are skipped. +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation); + +// The checks between reads: the screen must be one the previous state can lead +// to, no state may come up too often, and steps must keep changing the screen +// (a waiting state's only for its ScreenState::max_still). +class ScreenMachineChecks{ +public: + ScreenMachineChecks(StepTrace& trace) + : m_trace(trace) + {} + // Fails through the trace on a problem. + void check(const ScreenState& state, VideoSnapshot frame); + // The problem with `state` coming up on `observation`, if any. The + // screen counts as unchanged only when it and the look before both have + // a frame and the frames match; how long, by the looks' `when`. + std::optional problem(const ScreenState& state, const ScreenObservation& observation); + +private: + std::optional problem(const ScreenState& state, VideoSnapshot frame, WallClock when); + // The stall problem, if any, of `state` on a frame unchanged since `m_changed_at`. + std::optional unchanged_problem(const ScreenState& state, WallClock when); + + // A step that changes nothing on screen is a dead end (e.g. a menu the states do + // not know): fail after a few instead of running out the visit limit. + static constexpr size_t MAX_UNCHANGED = 8; + + StepTrace& m_trace; + std::map m_visits; + const ScreenState* m_previous = nullptr; + VideoSnapshot m_last_frame; + // When the screen last changed (or was first read). + WallClock m_changed_at = WallClock::min(); + size_t m_unchanged = 0; + size_t m_in_a_row = 0; +}; + + +// Runs the states until a terminal one has acted. Returns its name. Fails through +// `trace` on an unexpected screen, a state seen too often, a stall or the timeout. +template +std::string run_screen_machine( + VideoStream& stream, ControllerContext& context, + const std::vector& states, Milliseconds timeout, StepTrace& trace +); + + +// How a run of states over observations ended. +struct ScreenMachineResult{ + // The terminal state that acted; empty when the run failed. + std::string terminal; + // Why it failed: no state matched, an unexpected screen, a state seen too + // often, a stall, or the timeout. + std::string failure; + + bool succeeded() const{ + return failure.empty(); + } +}; + +// The same loop as run_screen_machine() over looks from `observe` instead of +// frames, so it runs on a console and in a simulation alike: each look picks +// the first state whose matches_observation() accepts it (one look, no second +// read: an observation is already settled), the trace records it, the checks +// apply, then the state acts. Time is the looks' own (`when`): the run fails +// once `timeout` has passed since the first look, and after 10000 looks in +// any case. A failure is returned, not thrown. +ScreenMachineResult run_screen_machine_on_observations( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +); + +template +std::string run_screen_machine( + VideoStream& stream, ControllerContext& context, + const std::vector& states, Milliseconds timeout, StepTrace& trace +){ + ScreenMachineChecks checks(trace); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + context.wait_for_all_requests(); + VideoSnapshot frame; + const ScreenState& state = read_screen_state(stream, context, states, trace, frame); + trace.screen(state.name); + checks.check(state, std::move(frame)); + state.act(); + if (state.terminal){ + context.wait_for_all_requests(); + return state.name; + } + } + trace.fail(std::format("Did not finish within {}s.", timeout.count() / 1000)); +} + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h new file mode 100644 index 0000000000..ba1e6e84b3 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h @@ -0,0 +1,47 @@ +/* Screen Observation + * + * From: https://github.com/PokemonAutomation/ + * + * What one look at the game showed, as facts rather than pixels: the + * events on screen (a fade, a dialog box, a battle) and named readings. A + * console fills it from a frame with the game's detectors; a simulation + * fills it from its model and has no frame. A ScreenState that matches on + * an observation (ScreenState::matches_observation) therefore runs in + * both. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenObservation_H +#define PokemonAutomation_CommonTools_ScreenObservation_H + +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/ScreenEvents/ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// One look at the game. +struct ScreenObservation{ + WallClock when = WallClock::min(); + ScreenEvents events = {}; + // Named numbers a reader found: "bag.RARE_CANDY", "facing", + // "heading" (degrees clockwise from north, [0, 360)). + std::map> readings = {}; + // The frame looked at; none in a simulation. A state that needs it + // can only run on a console. + std::optional frame = std::nullopt; +}; + +// Whether `a` and `b` both have a frame of the same size within the +// still-screen RMSD threshold (2.0). +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp new file mode 100644 index 0000000000..b9a12614c8 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp @@ -0,0 +1,104 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/PrettyPrint.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +StepTrace::StepTrace(VideoStream& stream, std::string label) + : m_stream(&stream) + , m_label(std::move(label)) + , m_folder(std::format("{}StepTrace/{}-{}/", SCREENSHOTS_PATH(), now_to_filestring(), m_label)) + , m_start(current_time()) +{ + Filesystem::create_directories(Filesystem::Path(m_folder)); + log(std::format("[{}] Trace screenshots: {}", m_label, m_folder)); +} + +StepTrace::StepTrace(std::string label, std::function log) + : m_log(std::move(log)) + , m_label(std::move(label)) + , m_start(current_time()) +{} + +void StepTrace::log(const std::string& line) const{ + if (m_stream != nullptr){ + m_stream->log(line); + }else if (m_log){ + m_log(line); + } +} + +std::string StepTrace::save_screenshot(const std::string& name) const{ + if (m_stream == nullptr){ + return "(no video)"; + } + const std::string path = std::format("{}{:03}-{}.png", m_folder, m_count, name); + VideoSnapshot screen = m_stream->video().snapshot(); + if (!screen){ + return "(no video frame)"; + } + screen.frame->save(path); + return path; +} + +void StepTrace::screen(const std::string& name){ + if (name == m_last){ + return; + } + m_last = name; + m_recent.push_back(name); + if (m_recent.size() > 12){ + m_recent.pop_front(); + } + m_count++; + const double seconds = std::chrono::duration(current_time() - m_start).count(); + const std::string path = save_screenshot(name); + log(std::format("[{}] screen {}: {} at {}s, {}", m_label, m_count, name, tostr_fixed(seconds, 1), path)); +} + +void StepTrace::action(const std::string& what){ + log(std::format("[{}] {}", m_label, what)); +} + +std::string StepTrace::recent_screens() const{ + std::string text; + for (const std::string& name : m_recent){ + if (!text.empty()){ + text += " -> "; + } + text += name; + } + return text.empty() ? "(none)" : text; +} + +void StepTrace::fail(const std::string& message) const{ + if (m_stream == nullptr){ + throw OperationFailedException( + ErrorReportMode::NO_ERROR_REPORT, + std::format("{}: {} Recent screens: {}.", m_label, message, recent_screens()) + ); + } + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("{}: {} Recent screens: {}. Trace: {}", m_label, message, recent_screens(), m_folder), + *m_stream + ); +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h new file mode 100644 index 0000000000..7b125a37cf --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h @@ -0,0 +1,67 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + * A step log for closed-loop routines: every change of recognized screen is + * logged and saved as a numbered screenshot, every input is logged, and a + * failure reports the screens that led up to it. A trace of observations + * (ScreenObservation.h), which may come from a simulation, logs the same + * lines with no screenshots. + * + */ + +#ifndef PokemonAutomation_CommonTools_StepTrace_H +#define PokemonAutomation_CommonTools_StepTrace_H + +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/Tools/VideoStream.h" + +namespace PokemonAutomation{ + + +// Recorder that captures screen transitions and recent actions for failure diagnostics. +class StepTrace{ +public: + // Screenshots go to /StepTrace/