diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp new file mode 100644 index 0000000000..7086d9c9c1 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp @@ -0,0 +1,177 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "ScreenMachine.h" + +namespace PokemonAutomation{ + +using namespace std::chrono_literals; + + +namespace{ + +const ScreenState* classify(const std::vector& states, const ImageViewRGB32& screen){ + for (const ScreenState& state : states){ + if (state.matches && state.matches(screen)){ + return &state; + } + } + return nullptr; +} + +} // namespace + + +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b){ + if (!a || !b || a->width() != b->width() || a->height() != b->height()){ + return false; + } + return ImageMatch::pixel_RMSD(a, b) < 2.0; +} + +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation){ + for (const ScreenState& state : states){ + if (state.matches_observation && state.matches_observation(observation)){ + return &state; + } + } + return nullptr; +} + + +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, + const std::vector& states, StepTrace& trace, VideoSnapshot& frame +){ + const ScreenState* last = nullptr; + WallClock video_lost = WallClock::max(); + for (size_t c = 0; c < 20; c++){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + // The app resets a frozen capture by itself, which takes a second or two. + const WallClock now = current_time(); + video_lost = std::min(video_lost, now); + if (now - video_lost > std::chrono::seconds(10)){ + trace.fail("No video from the capture card for 10s."); + } + c--; + scope.wait_for(500ms); + continue; + } + video_lost = WallClock::max(); + const ScreenState* now = classify(states, screen); + frame = std::move(screen); + if (now != nullptr && now == last){ + return *now; + } + last = now; + scope.wait_for(150ms); + } + if (last == nullptr){ + trace.fail("The screen matches none of the known states."); + } + return *last; // Still changing after 3s: go with the latest read. +} + + +void ScreenMachineChecks::check(const ScreenState& state, VideoSnapshot frame){ + const std::optional found = problem(state, std::move(frame), current_time()); + if (found){ + m_trace.fail(*found); + } +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, const ScreenObservation& observation){ + return problem(state, observation.frame ? *observation.frame : VideoSnapshot(), observation.when); +} + +std::optional ScreenMachineChecks::unchanged_problem(const ScreenState& state, WallClock when){ + if (state.max_still > Milliseconds::zero()){ + if (when - m_changed_at > state.max_still){ + return std::format("The screen stayed still on \"{}\" for more than {}s.", state.name, state.max_still.count() / 1000); + } + return std::nullopt; + } + if (++m_unchanged >= MAX_UNCHANGED){ + return std::format("The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name); + } + return std::nullopt; +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, VideoSnapshot frame, WallClock when){ + if (same_screen(m_last_frame, frame)){ + std::optional stalled = unchanged_problem(state, when); + if (stalled){ + return stalled; + } + }else{ + m_unchanged = 0; + m_changed_at = when; + } + m_last_frame = std::move(frame); + + if (m_previous != nullptr && m_previous != &state && !m_previous->next.empty() && + std::find(m_previous->next.begin(), m_previous->next.end(), state.name) == m_previous->next.end() + ){ + return std::format("Unexpected screen after \"{}\": \"{}\".", m_previous->name, state.name); + } + m_in_a_row = m_previous == &state ? m_in_a_row + 1 : 1; + if (m_in_a_row > state.max_in_a_row){ + return std::format("Screen \"{}\" came up {} times in a row: the step for it is not getting anywhere.", state.name, m_in_a_row); + } + if (++m_visits[state.name] > state.max_visits){ + return std::format("Screen \"{}\" came up more than {} times: the step for it is not working.", state.name, state.max_visits); + } + m_previous = &state; + return std::nullopt; +} + + +ScreenMachineResult run_screen_machine_on_observations( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +){ + // Looks before the run stops even when nothing else ends it (looks with + // no time, states with no visit limit). + constexpr size_t MAX_LOOKS = 10000; + ScreenMachineChecks checks(trace); + std::optional start; + for (size_t looks = 0; looks < MAX_LOOKS; looks++){ + scope.throw_if_cancelled(); + const ScreenObservation observation = observe(); + if (!start){ + start = observation.when; + } + if (observation.when - *start > timeout){ + return {.terminal = "", .failure = std::format("Did not finish within {}s.", timeout.count() / 1000)}; + } + const ScreenState* state = match_screen_state(states, observation); + if (state == nullptr){ + return {.terminal = "", .failure = "The screen matches none of the known states."}; + } + trace.screen(state->name); + std::optional found = checks.problem(*state, observation); + if (found){ + return {.terminal = "", .failure = std::move(*found)}; + } + if (state->act){ + state->act(); + } + if (state->terminal){ + return {.terminal = state->name, .failure = ""}; + } + } + return {.terminal = "", .failure = std::format("Did not finish within {} looks.", MAX_LOOKS)}; +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h new file mode 100644 index 0000000000..20a9b7c304 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h @@ -0,0 +1,176 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + * A closed loop over known screens: read the screen, take the one step that screen + * calls for, read again. Every step follows a read, a screen counts only once two + * frames agree, and each state lists the screens it may lead to, so a dropped or + * wrong input shows up as an unexpected screen or a state that keeps coming back. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_H +#define PokemonAutomation_CommonTools_ScreenMachine_H + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/CancellableScope.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "ScreenObservation.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +// Single recognizable screen state, including detection predicate and handler action. +struct ScreenState{ + std::string name; + // Whether this is the screen. States are tried in order; the first match wins, so + // the last state should match anything. + std::function matches; + // Whether this is the screen, from what a look at the game observed + // instead of from its pixels, so the state also runs in a simulation. + // A state may have either or both; read_screen_state() uses `matches` + // and skips a state without it. + std::function matches_observation = nullptr; + // The one step to take on this screen; none for a state that only + // waits. + std::function act; + // The screens the step may lead to besides staying here. Empty: any. + std::vector next; + // Fail once the screen has been seen this many times. + size_t max_visits = SIZE_MAX; + // End the run after this state's step. + bool terminal = false; + // Fail once the screen has been seen this many times in a row. + size_t max_in_a_row = SIZE_MAX; + // How long the screen may stay unchanged on this state. A waiting + // state (a fade, a still cutscene, a hold before a screen counts) sets + // it: its looks never count as a stall, and it fails only once the + // screen has not changed for longer than this. Zero: the state's step + // is input that should change the screen, and MAX_UNCHANGED such + // steps that do not are a stall. + Milliseconds max_still = Milliseconds::zero(); +}; + + +// Whether `a` and `b` both have a frame of the same size within the +// still-screen RMSD threshold (2.0). +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b); + +// The state on screen, once two frames 150ms apart agree. Waits out a video +// gap of up to 10s. Fails through `trace` if no state matches. +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, + const std::vector& states, StepTrace& trace, VideoSnapshot& frame +); + +// The first of `states` whose matches_observation() accepts `observation`; +// null when none does. States without one are skipped. +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation); + +// The checks between reads: the screen must be one the previous state can lead +// to, no state may come up too often, and steps must keep changing the screen +// (a waiting state's only for its ScreenState::max_still). +class ScreenMachineChecks{ +public: + ScreenMachineChecks(StepTrace& trace) + : m_trace(trace) + {} + // Fails through the trace on a problem. + void check(const ScreenState& state, VideoSnapshot frame); + // The problem with `state` coming up on `observation`, if any. The + // screen counts as unchanged only when it and the look before both have + // a frame and the frames match; how long, by the looks' `when`. + std::optional problem(const ScreenState& state, const ScreenObservation& observation); + +private: + std::optional problem(const ScreenState& state, VideoSnapshot frame, WallClock when); + // The stall problem, if any, of `state` on a frame unchanged since `m_changed_at`. + std::optional unchanged_problem(const ScreenState& state, WallClock when); + + // A step that changes nothing on screen is a dead end (e.g. a menu the states do + // not know): fail after a few instead of running out the visit limit. + static constexpr size_t MAX_UNCHANGED = 8; + + StepTrace& m_trace; + std::map m_visits; + const ScreenState* m_previous = nullptr; + VideoSnapshot m_last_frame; + // When the screen last changed (or was first read). + WallClock m_changed_at = WallClock::min(); + size_t m_unchanged = 0; + size_t m_in_a_row = 0; +}; + + +// Runs the states until a terminal one has acted. Returns its name. Fails through +// `trace` on an unexpected screen, a state seen too often, a stall or the timeout. +template +std::string run_screen_machine( + VideoStream& stream, ControllerContext& context, + const std::vector& states, Milliseconds timeout, StepTrace& trace +); + + +// How a run of states over observations ended. +struct ScreenMachineResult{ + // The terminal state that acted; empty when the run failed. + std::string terminal; + // Why it failed: no state matched, an unexpected screen, a state seen too + // often, a stall, or the timeout. + std::string failure; + + bool succeeded() const{ + return failure.empty(); + } +}; + +// The same loop as run_screen_machine() over looks from `observe` instead of +// frames, so it runs on a console and in a simulation alike: each look picks +// the first state whose matches_observation() accepts it (one look, no second +// read: an observation is already settled), the trace records it, the checks +// apply, then the state acts. Time is the looks' own (`when`): the run fails +// once `timeout` has passed since the first look, and after 10000 looks in +// any case. A failure is returned, not thrown. +ScreenMachineResult run_screen_machine_on_observations( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +); + +template +std::string run_screen_machine( + VideoStream& stream, ControllerContext& context, + const std::vector& states, Milliseconds timeout, StepTrace& trace +){ + ScreenMachineChecks checks(trace); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + context.wait_for_all_requests(); + VideoSnapshot frame; + const ScreenState& state = read_screen_state(stream, context, states, trace, frame); + trace.screen(state.name); + checks.check(state, std::move(frame)); + if (state.act){ + state.act(); + } + if (state.terminal){ + context.wait_for_all_requests(); + return state.name; + } + } + trace.fail(std::format("Did not finish within {}s.", timeout.count() / 1000)); +} + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp new file mode 100644 index 0000000000..b0a6ff1f4b --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp @@ -0,0 +1,312 @@ +/* Screen Machine Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The screen machine run on scripted observations: which state a look + * matches, a run that ends on its terminal state, one that fails (no state + * matches, a state comes up too often, the time runs out), and the still + * checks (a waiting state's limit, a step that changes nothing). + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "ScreenMachine.h" +#include "ScreenMachine_Tests.h" + +namespace PokemonAutomation{ + + +namespace{ + +using namespace std::chrono_literals; + + +std::string joined(const std::vector& items){ + std::string text; + for (const std::string& item : items){ + text += text.empty() ? item : ", " + item; + } + return text; +} + + +// What a look showed, matched against states that match on observations. +struct MatchCase{ + std::string_view name; + struct Input{ + std::vector events; + }; + + // The state matched; none when no state did. + struct Expectation{ + std::optional state = std::nullopt; + }; + + Input input; + Expectation expectation; +}; + +const MatchCase MATCH_CASES[] = { + {.name = "the first state that matches", .input = {.events = {ScreenEvent::BATTLE_STARTED, ScreenEvent::DIALOG_BOX}}, .expectation = {.state = "battle"}}, + {.name = "text", .input = {.events = {ScreenEvent::DIALOG_BOX}}, .expectation = {.state = "text"}}, + {.name = "overworld", .input = {.events = {ScreenEvent::PLAYER_CAN_MOVE}}, .expectation = {.state = "overworld"}}, + {.name = "an image-only state is skipped", .input = {.events = {ScreenEvent::FADE_WHITE}}, .expectation = {}}, +}; + +ScreenState observation_state(std::string name, ScreenEvent event){ + ScreenState state; + state.name = std::move(name); + state.matches_observation = [event](const ScreenObservation& observation){ + return observation.events.has(event); + }; + state.max_visits = 10; + return state; +} + +class Test_MatchScreenState : public UnitTest{ +public: + explicit Test_MatchScreenState(MatchCase test) + : UnitTest(std::format("CommonTools::ScreenMachine - match: {}", test.name)) + , m_case(std::move(test)) + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + std::vector states{ + observation_state("battle", ScreenEvent::BATTLE_STARTED), + observation_state("text", ScreenEvent::DIALOG_BOX), + observation_state("overworld", ScreenEvent::PLAYER_CAN_MOVE), + }; + ScreenState image_only; + image_only.name = "image_only"; + image_only.matches = [](const ImageViewRGB32&){ + return true; + }; + states.push_back(std::move(image_only)); + ScreenObservation observation; + for (ScreenEvent event : m_case.input.events){ + observation.events.set(event, true); + } + const ScreenState* state = match_screen_state(states, observation); + const std::string got = state ? state->name : "none"; + if (got != m_case.expectation.state.value_or("none")){ + return std::format("matched {}", got); + } + return true; + } + +private: + MatchCase m_case; +}; + + +// States for the choice, prompt and text events; the choice ends the run. +std::vector choice_states(std::vector& acted){ + const auto state = [&acted](std::string name, ScreenEvent event){ + ScreenState result; + result.name = name; + result.matches_observation = [event](const ScreenObservation& observation){ + return observation.events.has(event); + }; + result.act = [&acted, name]{ + acted.push_back(name); + }; + result.max_visits = 3; + return result; + }; + std::vector states{ + state("choice", ScreenEvent::CHOICE_BOX), + state("prompt", ScreenEvent::INTERACT_PROMPT), + state("text", ScreenEvent::DIALOG_BOX), + }; + states[0].terminal = true; + return states; +} + +// The machine over `events`, one look each, the last repeating, a second +// apart. +ScreenMachineResult run_on_events( + const std::vector& events, CancellableScope& scope, Milliseconds timeout, + std::vector& acted, std::vector& lines +){ + const std::vector states = choice_states(acted); + StepTrace trace("test", [&lines](const std::string& line){ + lines.push_back(line); + }); + size_t next = 0; + const WallClock start = current_time(); + const std::function observe = [&]{ + ScreenObservation observation; + observation.when = start + std::chrono::seconds(next); + observation.events.set(events[std::min(next, events.size() - 1)], true); + next++; + return observation; + }; + return run_screen_machine_on_observations(observe, scope, states, timeout, trace); +} + +UnitTestResult observation_machine_ends(CancellableScope& scope){ + std::vector acted; + std::vector lines; + const ScreenMachineResult result = run_on_events( + {ScreenEvent::INTERACT_PROMPT, ScreenEvent::DIALOG_BOX, ScreenEvent::CHOICE_BOX}, scope, 60s, acted, lines + ); + if (!result.succeeded() || result.terminal != "choice"){ + return std::format("expected the run to end on choice, got \"{}\"", result.failure); + } + if (joined(acted) != "prompt, text, choice"){ + return std::format("expected the states to act in order, got {}", joined(acted)); + } + if (lines.size() != 3 || !lines[0].starts_with("[test] screen 1: prompt at ")){ + return std::format("expected a trace line per screen, got {}", joined(lines)); + } + return true; +} + +UnitTestResult observation_machine_fails(CancellableScope& scope){ + std::vector acted; + std::vector lines; + ScreenMachineResult result = run_on_events({ScreenEvent::BATTLE_STARTED}, scope, 60s, acted, lines); + if (result.failure != "The screen matches none of the known states."){ + return std::format("expected no state to match, got \"{}\"", result.failure); + } + result = run_on_events({ScreenEvent::DIALOG_BOX}, scope, 60s, acted, lines); + if (!result.failure.starts_with("Screen \"text\" came up more than 3 times")){ + return std::format("expected text to come up too often, got \"{}\"", result.failure); + } + result = run_on_events({ScreenEvent::DIALOG_BOX, ScreenEvent::INTERACT_PROMPT, ScreenEvent::DIALOG_BOX}, scope, 1s, acted, lines); + if (result.failure != "Did not finish within 1s."){ + return std::format("expected the looks' time to run out, got \"{}\"", result.failure); + } + return true; +} + +// A look `seconds` after `start` on a uniform frame of `shade`, showing +// `event`. +ScreenObservation framed_look(WallClock start, int seconds, uint32_t shade, ScreenEvent event){ + ScreenObservation observation; + observation.when = start + std::chrono::seconds(seconds); + observation.events.set(event, true); + ImageRGB32 image(16, 16); + image.fill(shade); + observation.frame = VideoSnapshot(std::move(image), observation.when); + return observation; +} + +// A fade that waits (still up to 30 s), a box whose step is a press, and +// the overworld that ends the run, over `looks`, the last repeating. +ScreenMachineResult run_still_states(const std::vector& looks, CancellableScope& scope){ + ScreenState fade; + fade.name = "fade"; + fade.matches_observation = [](const ScreenObservation& observation){ + return observation.events.has(ScreenEvent::FADE_BLACK); + }; + fade.max_still = 30s; + size_t presses = 0; + ScreenState text; + text.name = "text"; + text.matches_observation = [](const ScreenObservation& observation){ + return observation.events.has(ScreenEvent::DIALOG_BOX); + }; + text.act = [&presses]{ + presses++; + }; + ScreenState done; + done.name = "done"; + done.matches_observation = [](const ScreenObservation& observation){ + return observation.events.has(ScreenEvent::PLAYER_CAN_MOVE); + }; + done.terminal = true; + const std::vector states{fade, text, done}; + StepTrace trace("test", [](const std::string&){}); + size_t next = 0; + const std::function observe = [&]{ + return looks[std::min(next++, looks.size() - 1)]; + }; + return run_screen_machine_on_observations(observe, scope, states, 600s, trace); +} + +UnitTestResult still_states(CancellableScope& scope){ + const WallClock start = current_time(); + std::vector waits; + for (int second = 0; second < 12; second++){ + waits.push_back(framed_look(start, second, 0xff000000, ScreenEvent::FADE_BLACK)); + } + waits.push_back(framed_look(start, 12, 0xff808080, ScreenEvent::PLAYER_CAN_MOVE)); + ScreenMachineResult result = run_still_states(waits, scope); + if (result.terminal != "done"){ + return std::format("expected a 12 s still fade to end on done, got \"{}\"", result.failure); + } + std::vector too_long; + for (int second = 0; second < 40; second++){ + too_long.push_back(framed_look(start, second, 0xff000000, ScreenEvent::FADE_BLACK)); + } + result = run_still_states(too_long, scope); + if (result.failure != "The screen stayed still on \"fade\" for more than 30s."){ + return std::format("expected the fade to stay still too long, got \"{}\"", result.failure); + } + std::vector presses; + for (int second = 0; second < 12; second++){ + presses.push_back(framed_look(start, second, 0xffffffff, ScreenEvent::DIALOG_BOX)); + } + result = run_still_states(presses, scope); + if (result.failure != "The screen stopped changing: 8 steps on \"text\" did nothing."){ + return std::format("expected the box to stall, got \"{}\"", result.failure); + } + return true; +} + + +// A named test; it returns its own failure. +struct NamedTest{ + std::string_view name; + UnitTestResult (*test)(CancellableScope& scope); +}; + +class Test_ScreenMachine : public UnitTest{ +public: + explicit Test_ScreenMachine(NamedTest test) + : UnitTest(std::format("CommonTools::ScreenMachine - {}", test.name)) + , m_test(test) + {} + + virtual UnitTestResult run(Logger&, CancellableScope& scope) const override{ + return m_test.test(scope); + } + +private: + const NamedTest m_test; +}; + +const NamedTest SCREEN_MACHINE_TESTS[] = { + {.name = "a run ends on its terminal state", .test = observation_machine_ends}, + {.name = "a run fails: no match, too many visits, timeout", .test = observation_machine_fails}, + {.name = "a waiting state's limit and a stall", .test = still_states}, +}; + + +} // namespace + + +void add_tests_ScreenMachine(UnitTestDatabase& database){ + for (const MatchCase& test : MATCH_CASES){ + database.add(test); + } + for (const NamedTest& test : SCREEN_MACHINE_TESTS){ + database.add(test); + } +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h new file mode 100644 index 0000000000..99d9aefa53 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h @@ -0,0 +1,21 @@ +/* Screen Machine Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_Tests_H +#define PokemonAutomation_CommonTools_ScreenMachine_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +// Registers the tests of the screen machine run on observations. +void add_tests_ScreenMachine(UnitTestDatabase& database); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp new file mode 100644 index 0000000000..e4b6a5636e --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp @@ -0,0 +1,104 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/PrettyPrint.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +StepTrace::StepTrace(VideoStream& stream, std::string label) + : m_stream(&stream) + , m_label(std::move(label)) + , m_folder(std::format("{}StepTrace/{}-{}/", SCREENSHOTS_PATH(), now_to_filestring(), m_label)) + , m_start(current_time()) +{ + Filesystem::create_directories(Filesystem::Path(m_folder)); + log(std::format("[{}] Trace screenshots: {}", m_label, m_folder)); +} + +StepTrace::StepTrace(std::string label, std::function log) + : m_log(std::move(log)) + , m_label(std::move(label)) + , m_start(current_time()) +{} + +void StepTrace::log(const std::string& line) const{ + if (m_stream != nullptr){ + m_stream->log(line); + }else if (m_log){ + m_log(line); + } +} + +std::string StepTrace::save_screenshot(const std::string& name) const{ + if (m_stream == nullptr){ + return "(no video)"; + } + const std::string path = std::format("{}{:03}-{}.png", m_folder, m_count, name); + VideoSnapshot screen = m_stream->video().snapshot(); + if (!screen){ + return "(no video frame)"; + } + screen.frame->save(path); + return path; +} + +void StepTrace::screen(const std::string& name){ + if (name == m_last){ + return; + } + m_last = name; + m_recent.push_back(name); + if (m_recent.size() > 12){ + m_recent.pop_front(); + } + m_count++; + const double seconds = std::chrono::duration(current_time() - m_start).count(); + const std::string path = save_screenshot(name); + log(std::format("[{}] screen {}: {} at {}s, {}", m_label, m_count, name, tostr_fixed(seconds, 1), path)); +} + +void StepTrace::action(const std::string& what){ + log(std::format("[{}] {}", m_label, what)); +} + +std::string StepTrace::recent_screens() const{ + std::string text; + for (const std::string& name : m_recent){ + if (!text.empty()){ + text += " -> "; + } + text += name; + } + return text.empty() ? "(none)" : text; +} + +void StepTrace::fail(const std::string& message) const{ + if (m_stream == nullptr){ + throw OperationFailedException( + ErrorReportMode::NO_ERROR_REPORT, + std::format("{}: {} Recent screens: {}.", m_label, message, recent_screens()) + ); + } + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("{}: {} Recent screens: {}. Trace: {}", m_label, message, recent_screens(), m_folder), + *m_stream + ); +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h new file mode 100644 index 0000000000..673df989bb --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h @@ -0,0 +1,67 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + * A step log for closed-loop routines: every change of recognized screen is + * logged and saved as a numbered screenshot, every input is logged, and a + * failure reports the screens that led up to it. A trace of observations + * (ScreenObservation.h), which may come from a simulation, logs the same + * lines with no screenshots. + * + */ + +#ifndef PokemonAutomation_CommonTools_StepTrace_H +#define PokemonAutomation_CommonTools_StepTrace_H + +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/Tools/VideoStream.h" + +namespace PokemonAutomation{ + + +// Recorder that captures screen transitions and recent actions for failure diagnostics. +class StepTrace{ +public: + // Screenshots go to /StepTrace/