diff --git a/.github/workflows/runtime-map-reader.yml b/.github/workflows/runtime-map-reader.yml new file mode 100644 index 0000000000..1fd59940a7 --- /dev/null +++ b/.github/workflows/runtime-map-reader.yml @@ -0,0 +1,29 @@ +name: Runtime map reader +on: + pull_request: + workflow_dispatch: +jobs: + native-schema: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: + repository: dayfine/PokemonSimulation + ref: 906956e94fc912ae9adb0e44845da780f3bc7e4c + path: publisher + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - run: python devtools/runtime-map-tests/prepare_parity.py --publisher publisher --publisher-revision 906956e94fc912ae9adb0e44845da780f3bc7e4c --out runtime-map-pairs.json + - run: cmake -S devtools/runtime-map-tests -B build-runtime-maps -DCMAKE_BUILD_TYPE=Debug + - run: cmake --build build-runtime-maps --config Debug -j4 + - run: ctest --test-dir build-runtime-maps -C Debug --output-on-failure + - if: runner.os != 'Windows' + run: ./build-runtime-maps/RuntimeMapTests runtime-map-pairs.json + - if: runner.os == 'Windows' + run: ./build-runtime-maps/Debug/RuntimeMapTests.exe runtime-map-pairs.json diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index d135188804..62d006eb4c 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -228,6 +228,8 @@ add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES}) # Data files built into the program (see cmake/EmbedTextFiles.cmake). include(cmake/EmbedTextFiles.cmake) +pa_embed_text_files(SerialProgramsLib Source/Pokemon/Navigation/TestMaps + Pokemon/Navigation/Pokemon_NavigationTestMaps.h PokemonAutomation::Pokemon navigation_test_map_json) # Create binary target diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h index 3985131d56..73b4b12a77 100644 --- a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h @@ -14,6 +14,7 @@ #include #include #include +#include namespace PokemonAutomation{ @@ -82,8 +83,9 @@ class ScreenEvents{ uint32_t m_bits = 0; }; -// "dialog box, player can move"; "nothing" when empty. -std::string describe(const ScreenEvents& events); +// "dialog box, player can move" (joined by `separator`); "nothing" when +// empty. +std::string describe(const ScreenEvents& events, std::string_view separator = ", "); diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp index 1351e70343..8269377758 100644 --- a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp @@ -81,7 +81,7 @@ WarpFade fade_in(const ScreenEvents& events){ } return WarpFade::NONE; } -std::string describe(const ScreenEvents& events){ +std::string describe(const ScreenEvents& events, std::string_view separator){ std::string out; for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){ const ScreenEvent event = (ScreenEvent)c; @@ -89,7 +89,7 @@ std::string describe(const ScreenEvents& events){ continue; } if (!out.empty()){ - out += ", "; + out += separator; } out += screen_event_name(event); } diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp index 11c7771aa4..4be721658c 100644 --- a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include "CommonFramework/ImageTools/ImageDiff.h" #include "ScreenMachine.h" @@ -18,7 +20,7 @@ namespace{ const ScreenState* classify(const std::vector& states, const ImageViewRGB32& screen){ for (const ScreenState& state : states){ - if (state.matches(screen)){ + if (state.matches && state.matches(screen)){ return &state; } } @@ -28,6 +30,23 @@ const ScreenState* classify(const std::vector& states, const ImageV } // namespace +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b){ + if (!a || !b || a->width() != b->width() || a->height() != b->height()){ + return false; + } + return ImageMatch::pixel_RMSD(a, b) < 2.0; +} + +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation){ + for (const ScreenState& state : states){ + if (state.matches_observation && state.matches_observation(observation)){ + return &state; + } + } + return nullptr; +} + + const ScreenState& read_screen_state( VideoStream& stream, CancellableScope& scope, const std::vector& states, StepTrace& trace, VideoSnapshot& frame @@ -64,38 +83,93 @@ const ScreenState& read_screen_state( void ScreenMachineChecks::check(const ScreenState& state, VideoSnapshot frame){ - const bool same_size = m_last_frame && frame && - m_last_frame->width() == frame->width() && m_last_frame->height() == frame->height(); - if (same_size && ImageMatch::pixel_RMSD(m_last_frame, frame) < 2.0){ - if (++m_unchanged >= MAX_UNCHANGED){ - m_trace.fail(std::format( - "The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name - )); + const std::optional found = problem(state, std::move(frame), current_time()); + if (found){ + m_trace.fail(*found); + } +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, const ScreenObservation& observation){ + return problem(state, observation.frame ? *observation.frame : VideoSnapshot(), observation.when); +} + +std::optional ScreenMachineChecks::unchanged_problem(const ScreenState& state, WallClock when){ + if (state.max_still > Milliseconds::zero()){ + if (when - m_changed_at > state.max_still){ + return std::format("The screen stayed still on \"{}\" for more than {}s.", state.name, state.max_still.count() / 1000); + } + return std::nullopt; + } + if (++m_unchanged >= MAX_UNCHANGED){ + return std::format("The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name); + } + return std::nullopt; +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, VideoSnapshot frame, WallClock when){ + if (same_screen(m_last_frame, frame)){ + std::optional stalled = unchanged_problem(state, when); + if (stalled){ + return stalled; } }else{ m_unchanged = 0; + m_changed_at = when; } m_last_frame = std::move(frame); if (m_previous != nullptr && m_previous != &state && !m_previous->next.empty() && std::find(m_previous->next.begin(), m_previous->next.end(), state.name) == m_previous->next.end() ){ - m_trace.fail(std::format("Unexpected screen after \"{}\": \"{}\".", m_previous->name, state.name)); + return std::format("Unexpected screen after \"{}\": \"{}\".", m_previous->name, state.name); } m_in_a_row = m_previous == &state ? m_in_a_row + 1 : 1; if (m_in_a_row > state.max_in_a_row){ - m_trace.fail(std::format( - "Screen \"{}\" came up {} times in a row: the step for it is not getting anywhere.", - state.name, m_in_a_row - )); + return std::format("Screen \"{}\" came up {} times in a row: the step for it is not getting anywhere.", state.name, m_in_a_row); } if (++m_visits[state.name] > state.max_visits){ - m_trace.fail(std::format( - "Screen \"{}\" came up more than {} times: the step for it is not working.", - state.name, state.max_visits - )); + return std::format("Screen \"{}\" came up more than {} times: the step for it is not working.", state.name, state.max_visits); } m_previous = &state; + return std::nullopt; +} + + +ScreenMachineResult run_screen_machine_on_observations( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +){ + // Looks before the run stops even when nothing else ends it (looks with + // no time, states with no visit limit). + constexpr size_t MAX_LOOKS = 10000; + ScreenMachineChecks checks(trace); + std::optional start; + for (size_t looks = 0; looks < MAX_LOOKS; looks++){ + scope.throw_if_cancelled(); + const ScreenObservation observation = observe(); + if (!start){ + start = observation.when; + } + if (observation.when - *start > timeout){ + return {.terminal = "", .failure = std::format("Did not finish within {}s.", timeout.count() / 1000)}; + } + const ScreenState* state = match_screen_state(states, observation); + if (state == nullptr){ + return {.terminal = "", .failure = "The screen matches none of the known states."}; + } + trace.screen(state->name); + std::optional found = checks.problem(*state, observation); + if (found){ + return {.terminal = "", .failure = std::move(*found)}; + } + if (state->act){ + state->act(); + } + if (state->terminal){ + return {.terminal = state->name, .failure = ""}; + } + } + return {.terminal = "", .failure = std::format("Did not finish within {} looks.", MAX_LOOKS)}; } diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h index 5ed73a7bea..4f11d7066e 100644 --- a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include "Common/Cpp/Time.h" @@ -23,6 +24,7 @@ #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/Tools/VideoStream.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "ScreenObservation.h" #include "StepTrace.h" namespace PokemonAutomation{ @@ -34,6 +36,11 @@ struct ScreenState{ // Whether this is the screen. States are tried in order; the first match wins, so // the last state should match anything. std::function matches; + // Whether this is the screen, from what a look at the game observed + // instead of from its pixels, so the state also runs in a simulation. + // A state may have either or both; read_screen_state() uses `matches` + // and skips a state without it. + std::function matches_observation = nullptr; // The one step to take on this screen. std::function act; // The screens the step may lead to besides staying here. Empty: any. @@ -44,6 +51,13 @@ struct ScreenState{ bool terminal = false; // Fail once the screen has been seen this many times in a row. size_t max_in_a_row = SIZE_MAX; + // How long the screen may stay unchanged on this state. A waiting + // state (a fade, a still cutscene, a hold before a screen counts) sets + // it: its looks never count as a stall, and it fails only once the + // screen has not changed for longer than this. Zero: the state's step + // is input that should change the screen, and MAX_UNCHANGED such + // steps that do not are a stall. + Milliseconds max_still = Milliseconds::zero(); }; @@ -54,16 +68,30 @@ const ScreenState& read_screen_state( const std::vector& states, StepTrace& trace, VideoSnapshot& frame ); +// The first of `states` whose matches_observation() accepts `observation`; +// null when none does. States without one are skipped. +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation); + // The checks between reads: the screen must be one the previous state can lead -// to, no state may come up too often, and steps must keep changing the screen. +// to, no state may come up too often, and steps must keep changing the screen +// (a waiting state's only for its ScreenState::max_still). class ScreenMachineChecks{ public: ScreenMachineChecks(StepTrace& trace) : m_trace(trace) {} + // Fails through the trace on a problem. void check(const ScreenState& state, VideoSnapshot frame); + // The problem with `state` coming up on `observation`, if any. The + // screen counts as unchanged only when it and the look before both have + // a frame and the frames match; how long, by the looks' `when`. + std::optional problem(const ScreenState& state, const ScreenObservation& observation); private: + std::optional problem(const ScreenState& state, VideoSnapshot frame, WallClock when); + // The stall problem, if any, of `state` on a frame unchanged since `m_changed_at`. + std::optional unchanged_problem(const ScreenState& state, WallClock when); + // A step that changes nothing on screen is a dead end (e.g. a menu the states do // not know): fail after a few instead of running out the visit limit. static constexpr size_t MAX_UNCHANGED = 8; @@ -72,6 +100,8 @@ class ScreenMachineChecks{ std::map m_visits; const ScreenState* m_previous = nullptr; VideoSnapshot m_last_frame; + // When the screen last changed (or was first read). + WallClock m_changed_at = WallClock::min(); size_t m_unchanged = 0; size_t m_in_a_row = 0; }; @@ -85,6 +115,32 @@ std::string run_screen_machine( const std::vector& states, Milliseconds timeout, StepTrace& trace ); + +// How a run of states over observations ended. +struct ScreenMachineResult{ + // The terminal state that acted; empty when the run failed. + std::string terminal; + // Why it failed: no state matched, an unexpected screen, a state seen too + // often, a stall, or the timeout. + std::string failure; + + bool succeeded() const{ + return failure.empty(); + } +}; + +// The same loop as run_screen_machine() over looks from `observe` instead of +// frames, so it runs on a console and in a simulation alike: each look picks +// the first state whose matches_observation() accepts it (one look, no second +// read: an observation is already settled), the trace records it, the checks +// apply, then the state acts. Time is the looks' own (`when`): the run fails +// once `timeout` has passed since the first look, and after 10000 looks in +// any case. A failure is returned, not thrown. +ScreenMachineResult run_screen_machine_on_observations( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +); + template std::string run_screen_machine( VideoStream& stream, ControllerContext& context, diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h new file mode 100644 index 0000000000..ba1e6e84b3 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h @@ -0,0 +1,47 @@ +/* Screen Observation + * + * From: https://github.com/PokemonAutomation/ + * + * What one look at the game showed, as facts rather than pixels: the + * events on screen (a fade, a dialog box, a battle) and named readings. A + * console fills it from a frame with the game's detectors; a simulation + * fills it from its model and has no frame. A ScreenState that matches on + * an observation (ScreenState::matches_observation) therefore runs in + * both. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenObservation_H +#define PokemonAutomation_CommonTools_ScreenObservation_H + +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/ScreenEvents/ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// One look at the game. +struct ScreenObservation{ + WallClock when = WallClock::min(); + ScreenEvents events = {}; + // Named numbers a reader found: "bag.RARE_CANDY", "facing", + // "heading" (degrees clockwise from north, [0, 360)). + std::map> readings = {}; + // The frame looked at; none in a simulation. A state that needs it + // can only run on a console. + std::optional frame = std::nullopt; +}; + +// Whether `a` and `b` both have a frame of the same size within the +// still-screen RMSD threshold (2.0). +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp index b14224967e..b9a12614c8 100644 --- a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp @@ -6,9 +6,11 @@ #include #include +#include #include "Common/Cpp/PrettyPrint.h" #include "Common/Cpp/Filesystem/Filesystem.h" #include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" #include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "StepTrace.h" @@ -17,13 +19,40 @@ namespace PokemonAutomation{ StepTrace::StepTrace(VideoStream& stream, std::string label) - : m_stream(stream) + : m_stream(&stream) , m_label(std::move(label)) , m_folder(std::format("{}StepTrace/{}-{}/", SCREENSHOTS_PATH(), now_to_filestring(), m_label)) , m_start(current_time()) { Filesystem::create_directories(Filesystem::Path(m_folder)); - m_stream.log(std::format("[{}] Trace screenshots: {}", m_label, m_folder)); + log(std::format("[{}] Trace screenshots: {}", m_label, m_folder)); +} + +StepTrace::StepTrace(std::string label, std::function log) + : m_log(std::move(log)) + , m_label(std::move(label)) + , m_start(current_time()) +{} + +void StepTrace::log(const std::string& line) const{ + if (m_stream != nullptr){ + m_stream->log(line); + }else if (m_log){ + m_log(line); + } +} + +std::string StepTrace::save_screenshot(const std::string& name) const{ + if (m_stream == nullptr){ + return "(no video)"; + } + const std::string path = std::format("{}{:03}-{}.png", m_folder, m_count, name); + VideoSnapshot screen = m_stream->video().snapshot(); + if (!screen){ + return "(no video frame)"; + } + screen.frame->save(path); + return path; } void StepTrace::screen(const std::string& name){ @@ -37,20 +66,12 @@ void StepTrace::screen(const std::string& name){ } m_count++; const double seconds = std::chrono::duration(current_time() - m_start).count(); - std::string path = std::format("{}{:03}-{}.png", m_folder, m_count, name); - VideoSnapshot screen = m_stream.video().snapshot(); - if (screen){ - screen.frame->save(path); - }else{ - path = "(no video frame)"; - } - m_stream.log(std::format( - "[{}] screen {}: {} at {}s, {}", m_label, m_count, name, tostr_fixed(seconds, 1), path - )); + const std::string path = save_screenshot(name); + log(std::format("[{}] screen {}: {} at {}s, {}", m_label, m_count, name, tostr_fixed(seconds, 1), path)); } void StepTrace::action(const std::string& what){ - m_stream.log(std::format("[{}] {}", m_label, what)); + log(std::format("[{}] {}", m_label, what)); } std::string StepTrace::recent_screens() const{ @@ -65,10 +86,16 @@ std::string StepTrace::recent_screens() const{ } void StepTrace::fail(const std::string& message) const{ + if (m_stream == nullptr){ + throw OperationFailedException( + ErrorReportMode::NO_ERROR_REPORT, + std::format("{}: {} Recent screens: {}.", m_label, message, recent_screens()) + ); + } OperationFailedExceptionWithScreenshot::fire( ErrorReportMode::SEND_ERROR_REPORT, std::format("{}: {} Recent screens: {}. Trace: {}", m_label, message, recent_screens(), m_folder), - m_stream + *m_stream ); } diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h index ef266ff69d..7b125a37cf 100644 --- a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h @@ -4,7 +4,9 @@ * * A step log for closed-loop routines: every change of recognized screen is * logged and saved as a numbered screenshot, every input is logged, and a - * failure reports the screens that led up to it. + * failure reports the screens that led up to it. A trace of observations + * (ScreenObservation.h), which may come from a simulation, logs the same + * lines with no screenshots. * */ @@ -12,6 +14,7 @@ #define PokemonAutomation_CommonTools_StepTrace_H #include +#include #include #include "Common/Cpp/Time.h" #include "CommonFramework/Tools/VideoStream.h" @@ -25,6 +28,9 @@ class StepTrace{ // Screenshots go to /StepTrace/