diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp new file mode 100644 index 0000000000..e748864aa2 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp @@ -0,0 +1,136 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "CommonTools/ScreenEvents/ScreenEvents.h" +#include "ScreenMachine.h" + +namespace PokemonAutomation{ + + +namespace{ + + +// The first of `states` whose matches() accepts `observation`; null when +// none does. +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation){ + for (const ScreenState& state : states){ + if (state.matches && state.matches(observation)){ + return &state; + } + } + return nullptr; +} + + +// The checks between looks: no state may come up too often, and steps must +// keep changing the screen (a waiting state's only for its +// ScreenState::max_still). +class ScreenMachineChecks{ +public: + // A step that changes nothing on screen is a dead end (e.g. a menu the + // states do not know): fail after this many instead of running out the + // visit limit. + static constexpr size_t MAX_UNCHANGED = 8; + + // The problem with `state` coming up on `observation`, if any. The + // screen counts as unchanged only when it and the look before both have + // a frame and the frames match (same_screen()); how long, by the looks' + // `when`. + std::optional problem(const ScreenState& state, const ScreenObservation& observation); + +private: + // The stall problem, if any, of `state` on a frame unchanged since + // `m_changed_at`. + std::optional unchanged_problem(const ScreenState& state, WallClock when); + + std::map> m_visits; + VideoSnapshot m_last_frame; + // When the screen last changed (or was first looked at). + WallClock m_changed_at = WallClock::min(); + size_t m_unchanged = 0; +}; + + +std::optional ScreenMachineChecks::unchanged_problem(const ScreenState& state, WallClock when){ + if (state.max_still > Milliseconds::zero()){ + if (when - m_changed_at > state.max_still){ + return std::format("The screen stayed still on \"{}\" for more than {}s.", state.name, state.max_still.count() / 1000); + } + return std::nullopt; + } + if (++m_unchanged >= MAX_UNCHANGED){ + return std::format("The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name); + } + return std::nullopt; +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, const ScreenObservation& observation){ + VideoSnapshot frame = observation.frame ? *observation.frame : VideoSnapshot(); + if (same_screen(m_last_frame, frame)){ + std::optional stalled = unchanged_problem(state, observation.when); + if (stalled){ + return stalled; + } + }else{ + m_unchanged = 0; + m_changed_at = observation.when; + } + m_last_frame = std::move(frame); + if (++m_visits[state.name] > state.max_visits){ + return std::format("Screen \"{}\" came up more than {} times: the step for it is not working.", state.name, state.max_visits); + } + return std::nullopt; +} + + +} // namespace + + + +ScreenMachineResult run_screen_machine( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +){ + // Looks before the run stops even when nothing else ends it (looks with + // no time, states with no visit limit). + constexpr size_t MAX_LOOKS = 10000; + ScreenMachineChecks checks; + std::optional start; + for (size_t looks = 0; looks < MAX_LOOKS; looks++){ + scope.throw_if_cancelled(); + const ScreenObservation observation = observe(); + if (!start){ + start = observation.when; + } + if (observation.when - *start > timeout){ + return {.terminal = "", .failure = std::format("Did not finish within {}s.", timeout.count() / 1000)}; + } + const ScreenState* state = match_screen_state(states, observation); + if (state == nullptr){ + return {.terminal = "", .failure = "The screen matches none of the known states."}; + } + trace.screen(state->name, observation.when); + std::optional found = checks.problem(*state, observation); + if (found){ + return {.terminal = "", .failure = std::move(*found)}; + } + if (state->act){ + state->act(); + } + if (state->terminal){ + return {.terminal = state->name, .failure = ""}; + } + } + return {.terminal = "", .failure = std::format("Did not finish within {} looks.", MAX_LOOKS)}; +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h new file mode 100644 index 0000000000..3284faf9c1 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h @@ -0,0 +1,81 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + * A closed loop over known screens: look at the game, take the one step that + * screen calls for, look again. Every step follows a look, and each state says + * how often it may come up and how long the screen may stay unchanged on it, + * so a dropped or wrong input shows up as a state that keeps coming back or a + * screen that stops changing. The looks are ScreenObservations, so the same + * machine runs on a console and on a scripted backend. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_H +#define PokemonAutomation_CommonTools_ScreenMachine_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/CancellableScope.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "ScreenObservation.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +// One recognizable screen: how to tell it, and the step to take on it. +struct ScreenState{ + std::string name; + // Whether a look shows this screen. States are tried in order; the first + // match wins. + std::function matches; + // The one step to take on this screen; none for a state that only + // waits. + std::function act = nullptr; + // Fail once the screen has been seen more than this many times. + size_t max_visits = SIZE_MAX; + // End the run after this state's step. + bool terminal = false; + // How long the screen may stay unchanged on this state. A waiting + // state (a fade, a still cutscene, a hold before a screen counts) sets + // it: its looks never count as a stall, and it fails only once the + // screen has not changed for longer than this. Zero: the state's step + // is input that should change the screen, and + // ScreenMachineChecks::MAX_UNCHANGED (ScreenMachine.cpp) such steps + // that do not are a stall. + Milliseconds max_still = Milliseconds::zero(); +}; + + +// How a run of states ended. +struct ScreenMachineResult{ + // The terminal state that acted; empty when the run failed. + std::string terminal; + // Why it failed: no state matched, a state seen too often, a stall, or + // the timeout. + std::string failure; + + bool succeeded() const{ + return failure.empty(); + } +}; + +// Runs `states` over looks from `observe` until a terminal one has acted: +// each look picks the first state that matches it, `trace` records it, the +// checks apply, then the state acts. Time is the looks' own (`when`): the +// run fails once `timeout` has passed since the first look, and after +// 10000 looks in any case. A failure is returned, not thrown. +ScreenMachineResult run_screen_machine( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp new file mode 100644 index 0000000000..18e082a63e --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp @@ -0,0 +1,289 @@ +/* Screen Machine Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The screen machine on scripted observations: a run that ends on its + * terminal state, the first matching state winning, the ways one fails (no state + * matches, a state comes up too often, the time or the looks run out, a + * waiting state stays still too long, steps that change nothing), and the + * trace's lines. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "ScreenMachine.h" +#include "ScreenMachine_Tests.h" + +namespace PokemonAutomation{ + + +namespace{ + +using namespace std::chrono_literals; + + +std::string joined(const std::vector& items, std::string_view separator){ + std::string text; + for (const std::string& item : items){ + text += text.empty() ? item : std::format("{}{}", separator, item); + } + return text; +} + + +// One scripted look: its time from the first, what it shows, and its frame +// as one uniform shade (none: no frame, as in a simulation). +struct Look{ + Milliseconds at = 0ms; + ScreenEvents events = {}; + std::optional shade = std::nullopt; +}; + +ScreenObservation observation_of(const Look& look, WallClock start){ + ScreenObservation observation; + observation.when = start + look.at; + observation.events = look.events; + if (look.shade){ + ImageRGB32 image(16, 16); + image.fill(*look.shade); + observation.frame = VideoSnapshot(std::move(image), observation.when); + } + return observation; +} + +// `count` looks a second apart from `from` s, each showing `events` on +// a frame of `shade`, if any. +std::vector looks_every_second(int from, int count, ScreenEvents events, std::optional shade){ + std::vector looks; + for (int second = from; second < from + count; second++){ + looks.push_back({.at = std::chrono::seconds(second), .events = events, .shade = shade}); + } + return looks; +} + +// `first`, then `then`. +std::vector concat(std::vector first, const std::vector& then){ + first.insert(first.end(), then.begin(), then.end()); + return first; +} + +constexpr uint32_t BLACK = 0xff000000; +constexpr uint32_t WHITE = 0xffffffff; + + +// The states every case runs, in this order: +// - "fade" on FADE_BLACK: waits, the screen may stay still up to 30 s; +// - "choice" on CHOICE_BOX: terminal; +// - "prompt" on INTERACT_PROMPT: a step, no visit limit; +// - "text" on DIALOG_BOX: a step, at most 10 visits. +// Each step's name goes to `acted`. +std::vector test_states(std::vector& acted){ + const auto state = [&acted](std::string name, ScreenEvent event){ + ScreenState result; + result.name = name; + result.matches = [event](const ScreenObservation& observation){ + return observation.events.has(event); + }; + result.act = [&acted, name]{ + acted.push_back(name); + }; + return result; + }; + ScreenState fade = state("fade", ScreenEvent::FADE_BLACK); + fade.act = nullptr; + fade.max_still = 30s; + ScreenState choice = state("choice", ScreenEvent::CHOICE_BOX); + choice.terminal = true; + ScreenState text = state("text", ScreenEvent::DIALOG_BOX); + text.max_visits = 10; + return {std::move(fade), std::move(choice), state("prompt", ScreenEvent::INTERACT_PROMPT), std::move(text)}; +} + + +struct RunCase{ + std::string name; + struct Input{ + // The looks, the last repeating. + std::vector looks; + Milliseconds timeout = 600s; + }; + Input input; + struct Expectation{ + // The terminal state, or "failed: ". + std::string end; + // The steps taken, joined by ", "; none: not checked. + std::optional acted = std::nullopt; + // The trace's lines, joined by "; "; none: not checked. + std::optional trace = std::nullopt; + }; + Expectation expectation; +}; + +const RunCase RUN_CASES[] = { + { + .name = "a run ends on its terminal state, after each step", + .input = {.looks = { + {.at = 0s, .events = {ScreenEvent::INTERACT_PROMPT}}, + {.at = 1s, .events = {ScreenEvent::DIALOG_BOX}}, + {.at = 2s, .events = {ScreenEvent::CHOICE_BOX}}, + }}, + .expectation = {.end = "choice", .acted = "prompt, text, choice"}, + }, + { + .name = "the first state that matches a look wins", + .input = {.looks = {{.at = 0s, .events = {ScreenEvent::DIALOG_BOX, ScreenEvent::CHOICE_BOX}}}}, + .expectation = {.end = "choice", .acted = "choice"}, + }, + { + .name = "the trace logs each change of screen once, timed from the first look", + .input = {.looks = { + {.at = 0s, .events = {ScreenEvent::INTERACT_PROMPT}}, + {.at = 1s, .events = {ScreenEvent::INTERACT_PROMPT}}, + {.at = 2500ms, .events = {ScreenEvent::DIALOG_BOX}}, + {.at = 3s, .events = {ScreenEvent::CHOICE_BOX}}, + }}, + .expectation = { + .end = "choice", + .trace = "[test] screen 1: prompt at 0.0s; [test] screen 2: text at 2.5s; [test] screen 3: choice at 3.0s", + }, + }, + { + .name = "no state matches", + .input = {.looks = {{.at = 0s, .events = {ScreenEvent::BATTLE_STARTED}}}}, + .expectation = {.end = "failed: The screen matches none of the known states.", .acted = ""}, + }, + { + .name = "a look with nothing on it matches no state", + .input = {.looks = {{.at = 0s, .events = {}}}}, + .expectation = {.end = "failed: The screen matches none of the known states.", .acted = ""}, + }, + { + .name = "a state comes up more often than its limit", + .input = {.looks = looks_every_second(/*from=*/0, /*count=*/1, {ScreenEvent::DIALOG_BOX}, std::nullopt)}, + .expectation = {.end = "failed: Screen \"text\" came up more than 10 times: the step for it is not working."}, + }, + { + .name = "the looks' time runs out", + .input = { + .looks = { + {.at = 0s, .events = {ScreenEvent::DIALOG_BOX}}, + {.at = 1s, .events = {ScreenEvent::INTERACT_PROMPT}}, + {.at = 2s, .events = {ScreenEvent::DIALOG_BOX}}, + }, + .timeout = 1s, + }, + .expectation = {.end = "failed: Did not finish within 1s.", .acted = "text, prompt"}, + }, + { + .name = "looks with no time and a state with no limit run out of looks", + .input = {.looks = {{.at = 0s, .events = {ScreenEvent::INTERACT_PROMPT}}}}, + .expectation = {.end = "failed: Did not finish within 10000 looks."}, + }, + { + .name = "a waiting state may stay still up to its limit", + .input = {.looks = concat( + looks_every_second(/*from=*/0, /*count=*/12, {ScreenEvent::FADE_BLACK}, BLACK), + {{.at = 12s, .events = {ScreenEvent::CHOICE_BOX}, .shade = WHITE}} + )}, + .expectation = {.end = "choice", .acted = "choice"}, + }, + { + .name = "a waiting state still past its limit fails", + .input = {.looks = looks_every_second(/*from=*/0, /*count=*/40, {ScreenEvent::FADE_BLACK}, BLACK)}, + .expectation = {.end = "failed: The screen stayed still on \"fade\" for more than 30s."}, + }, + { + .name = "steps that change nothing on screen stall", + .input = {.looks = looks_every_second(/*from=*/0, /*count=*/12, {ScreenEvent::DIALOG_BOX}, WHITE)}, + .expectation = { + .end = "failed: The screen stopped changing: 8 steps on \"text\" did nothing.", + .acted = "text, text, text, text, text, text, text, text", + }, + }, + { + .name = "a changed frame starts the stall count again", + .input = {.looks = concat( + concat( + looks_every_second(/*from=*/0, /*count=*/5, {ScreenEvent::DIALOG_BOX}, WHITE), + looks_every_second(/*from=*/5, /*count=*/5, {ScreenEvent::DIALOG_BOX}, BLACK) + ), + {{.at = 10s, .events = {ScreenEvent::CHOICE_BOX}, .shade = WHITE}} + )}, + .expectation = {.end = "choice"}, + }, + { + .name = "looks with no frame never stall", + .input = {.looks = concat( + looks_every_second(/*from=*/0, /*count=*/9, {ScreenEvent::DIALOG_BOX}, std::nullopt), + {{.at = 9s, .events = {ScreenEvent::CHOICE_BOX}}} + )}, + .expectation = {.end = "choice"}, + }, +}; + +UnitTestResult check_run(const RunCase& test, CancellableScope& scope){ + std::vector acted; + const std::vector states = test_states(acted); + std::vector lines; + StepTrace trace("test", [&lines](const std::string& line){ + lines.push_back(line); + }); + const WallClock start = current_time(); + size_t next = 0; + const std::function observe = [&]{ + const Look& look = test.input.looks[std::min(next, test.input.looks.size() - 1)]; + next++; + return observation_of(look, start); + }; + const ScreenMachineResult result = run_screen_machine(observe, scope, states, test.input.timeout, trace); + const std::string end = result.succeeded() ? result.terminal : std::format("failed: {}", result.failure); + if (end != test.expectation.end){ + return std::format("expected \"{}\", got \"{}\"", test.expectation.end, end); + } + if (test.expectation.acted && joined(acted, ", ") != *test.expectation.acted){ + return std::format("expected the steps \"{}\", got \"{}\"", *test.expectation.acted, joined(acted, ", ")); + } + if (test.expectation.trace && joined(lines, "; ") != *test.expectation.trace){ + return std::format("expected the trace \"{}\", got \"{}\"", *test.expectation.trace, joined(lines, "; ")); + } + return true; +} + + +class Test_RunScreenMachine : public UnitTest{ +public: + explicit Test_RunScreenMachine(const RunCase& test) + : UnitTest(std::format("CommonTools::ScreenMachine - run: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope& scope) const override{ + return check_run(m_test, scope); + } + +private: + const RunCase& m_test; +}; + + +} // namespace + + +void add_tests_ScreenMachine(UnitTestDatabase& database){ + for (const RunCase& test : RUN_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h new file mode 100644 index 0000000000..b0eac935f7 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h @@ -0,0 +1,22 @@ +/* Screen Machine Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_ScreenMachine_Tests_H +#define PokemonAutomation_CommonTools_ScreenMachine_ScreenMachine_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +// Registers the screen machine's tests: scripted observations, no frames +// from disk. +void add_tests_ScreenMachine(UnitTestDatabase& database); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp new file mode 100644 index 0000000000..ef42f33954 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp @@ -0,0 +1,39 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/PrettyPrint.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +StepTrace::StepTrace(std::string label, std::function log) + : m_log(std::move(log)) + , m_label(std::move(label)) +{} + +void StepTrace::screen(const std::string& name, WallClock when){ + if (!m_start){ + m_start = when; + } + if (m_count > 0 && name == m_last){ + return; + } + m_last = name; + m_count++; + if (!m_log){ + return; + } + const double seconds = std::chrono::duration(when - *m_start).count(); + m_log(std::format("[{}] screen {}: {} at {}s", m_label, m_count, name, tostr_fixed(seconds, 1))); +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h new file mode 100644 index 0000000000..34493a64d7 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h @@ -0,0 +1,44 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + * The step log of a screen machine (ScreenMachine.h): every change of + * recognized screen is logged once, numbered, with its time since the first + * screen, so a run's log shows the screens that led to where it ended. + * + */ + +#ifndef PokemonAutomation_CommonTools_StepTrace_H +#define PokemonAutomation_CommonTools_StepTrace_H + +#include +#include +#include +#include "Common/Cpp/Time.h" + +namespace PokemonAutomation{ + + +// Logs the screens a routine recognizes, each change once. +class StepTrace{ +public: + // Lines go to `log`, each starting "[