diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index f8a299851d..412df7d233 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -245,9 +245,9 @@ jobs: uses: actions/checkout@v7 if: inputs.run-tests with: - # The fork's fixtures: upstream's plus the SystemKeyboardDetector screenshots. + # The fork's fixtures: upstream's plus the SystemKeyboardDetector and HomeTileReader screenshots. repository: 'dayfine/CommandLineTests' - ref: '5de4ff774f9141d904c4b5c557fa7b9dc85eaef1' + ref: 'eae4697738d5fe18975e77c05a4e01216bed68e7' path: 'CommandLineTests' # The unit tests assume the downloadable resources they need are already downloaded diff --git a/.github/workflows/runtime-map-reader.yml b/.github/workflows/runtime-map-reader.yml new file mode 100644 index 0000000000..1fd59940a7 --- /dev/null +++ b/.github/workflows/runtime-map-reader.yml @@ -0,0 +1,29 @@ +name: Runtime map reader +on: + pull_request: + workflow_dispatch: +jobs: + native-schema: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: + repository: dayfine/PokemonSimulation + ref: 906956e94fc912ae9adb0e44845da780f3bc7e4c + path: publisher + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - run: python devtools/runtime-map-tests/prepare_parity.py --publisher publisher --publisher-revision 906956e94fc912ae9adb0e44845da780f3bc7e4c --out runtime-map-pairs.json + - run: cmake -S devtools/runtime-map-tests -B build-runtime-maps -DCMAKE_BUILD_TYPE=Debug + - run: cmake --build build-runtime-maps --config Debug -j4 + - run: ctest --test-dir build-runtime-maps -C Debug --output-on-failure + - if: runner.os != 'Windows' + run: ./build-runtime-maps/RuntimeMapTests runtime-map-pairs.json + - if: runner.os == 'Windows' + run: ./build-runtime-maps/Debug/RuntimeMapTests.exe runtime-map-pairs.json diff --git a/.github/workflows/story-data.yml b/.github/workflows/story-data.yml new file mode 100644 index 0000000000..045f398256 --- /dev/null +++ b/.github/workflows/story-data.yml @@ -0,0 +1,34 @@ +name: Story data codec +on: + pull_request: + workflow_dispatch: +jobs: + native-codec: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: + repository: dayfine/PokemonSimulation + # Synthetic contract cases, not an approved map/resource pin. + ref: 4e023f3d57022c68018f95f6fc0a4c0722d53adf + path: publisher + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - name: Check the Python reader against the same cases + working-directory: publisher + env: + PYTHONPATH: src + run: python -m unittest pokesim.story.test_checkpoint_wire + - run: cmake -S devtools/story-data-tests -B build-story-data -DCMAKE_BUILD_TYPE=Debug + - run: cmake --build build-story-data --config Debug -j4 + - run: ctest --test-dir build-story-data -C Debug --output-on-failure + - if: runner.os != 'Windows' + run: ./build-story-data/StoryDataTests publisher/data/story/wire-conformance.json + - if: runner.os == 'Windows' + run: ./build-story-data/Debug/StoryDataTests.exe publisher/data/story/wire-conformance.json diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index d7fc0c3e76..62d006eb4c 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -226,6 +226,11 @@ include(cmake/CoreLib.cmake) # Create the GUI library add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES}) +# Data files built into the program (see cmake/EmbedTextFiles.cmake). +include(cmake/EmbedTextFiles.cmake) +pa_embed_text_files(SerialProgramsLib Source/Pokemon/Navigation/TestMaps + Pokemon/Navigation/Pokemon_NavigationTestMaps.h PokemonAutomation::Pokemon navigation_test_map_json) + # Create binary target if (APPLE) diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h new file mode 100644 index 0000000000..2d3039a5e7 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h @@ -0,0 +1,90 @@ +/* Screen Event Log + * + * From: https://github.com/PokemonAutomation/ + * + * The screen events of a stream of frames, in order and timed. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventLog_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventLog_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// Whether an event started or stopped showing. +enum class ScreenEventEdge : uint8_t{ + ROSE, + FELL, +}; + +// One event starting or stopping to show, at the frame that showed it. +struct ScreenEventChange{ + WallClock time; + ScreenEvent event; + ScreenEventEdge edge; + + bool operator==(const ScreenEventChange& other) const = default; +}; + +// How many changes a log keeps. A walk's events change a few times a +// second at most (a fade's black, a box, the screen going still), so this +// is a minute or more of a busy scene. +constexpr size_t SCREEN_EVENT_LOG_CAPACITY = 256; + +// The events of a stream of frames, in order and timed: what rose and fell +// when, not only what was seen (ScreenEventWatcher::seen()). The judges that +// care about order read it: a flash then a battle is an encounter, a box +// then a battle a trainer's sight. +// +// It keeps the newest `capacity` changes and counts the ones it dropped. +// Not thread-safe; ScreenEventWatcher guards its own. +class ScreenEventLog{ +public: + explicit ScreenEventLog(size_t capacity = SCREEN_EVENT_LOG_CAPACITY); + + // Adds a frame: a change for each event that rose or fell since the + // frame before (the first frame's events all rise). `now` must not go + // back in time. + void record(const ScreenEvents& frame, WallClock now); + + // The changes kept, oldest first. + const std::deque& changes() const{ + return m_changes; + } + // How many changes were dropped, oldest first, to keep the capacity. + size_t dropped() const{ + return m_dropped; + } + // The events of the last frame recorded. + ScreenEvents showing() const{ + return m_showing; + } + // When the last frame was recorded; none before the first. + std::optional last_frame() const{ + return m_last_frame; + } + void clear(); + +private: + size_t m_capacity; + std::deque m_changes; + size_t m_dropped = 0; + ScreenEvents m_showing; + std::optional m_last_frame; +}; +// "+1200 ms dialog box rose", times from `origin`, one change per line. +std::string describe(const ScreenEventLog& log, WallClock origin); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h new file mode 100644 index 0000000000..e9d05d5052 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h @@ -0,0 +1,94 @@ +/* Screen Event Types + * + * From: https://github.com/PokemonAutomation/ + * + * The words a program uses for what the screen shows: a fade, a dialogue + * box, a battle starting. ScreenEvents.h watches a stream for them. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H + +#include +#include +#include +#include +#include + +namespace PokemonAutomation{ + + +// One thing a frame shows. A game reports the ones its detectors can see. +enum class ScreenEvent : uint8_t{ + FADE_BLACK, // A uniform dark screen: a door, a cave mouth, a scene cut. + FADE_WHITE, // A uniform white screen: a cave exit. + LOCATION_BANNER, // A location banner. An observer given an expected + // map reports only that map's. + DIALOG_BOX, // A dialogue box. + CHOICE_BOX, // A choice (Yes/No, a menu of answers). + INTERACT_PROMPT, // A button prompt to interact ("A Talk"). + BATTLE_STARTED, // A battle shows (its menu, its HUD or its opening). + BATTLE_ENDED, // A battle's closing screen. + PLAYER_CAN_MOVE, // The overworld with the player in control. + SCREEN_STILL, // The screen barely changed since the frame before. + SWITCH_HOME_MENU, // The Switch's HOME menu: the game is not on the screen. +}; +// How many ScreenEvent values there are. +constexpr size_t SCREEN_EVENT_COUNT = 11; +static_assert((size_t)ScreenEvent::SWITCH_HOME_MENU + 1 == SCREEN_EVENT_COUNT, "SWITCH_HOME_MENU is the last ScreenEvent"); +// "dialog box", for the log. +std::string screen_event_name(ScreenEvent event); + + +// How a screen's fade looks: the kind of FADE_BLACK / FADE_WHITE. +enum class WarpFade{ + NONE, + BLACK, // a door's black, or a cave mouth's dim fade + WHITE, // a cave exit's white, with no battle after it +}; +// "black fade", for the log. +std::string warp_fade_name(WarpFade fade); + + +// The events of one frame, or of all the frames a watcher saw. +class ScreenEvents{ +public: + ScreenEvents() = default; + ScreenEvents(std::initializer_list events); + + bool has(ScreenEvent event) const{ + return (m_bits & bit(event)) != 0; + } + // Whether any of `events` is in this set. + bool any_of(const ScreenEvents& events) const{ + return (m_bits & events.m_bits) != 0; + } + // Adds `event`, or removes it when `on` is false (as std::bitset::set). + void set(ScreenEvent event, bool on); + void merge(const ScreenEvents& events){ + m_bits |= events.m_bits; + } + + uint32_t bits() const{ + return m_bits; + } + static ScreenEvents from_bits(uint32_t bits); + + bool operator==(const ScreenEvents& other) const = default; + +private: + static uint32_t bit(ScreenEvent event){ + return uint32_t(1) << (uint32_t)event; + } + + uint32_t m_bits = 0; +}; +// "dialog box, player can move" (joined by `separator`); "nothing" when +// empty. +std::string describe(const ScreenEvents& events, std::string_view separator = ", "); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp new file mode 100644 index 0000000000..8269377758 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp @@ -0,0 +1,188 @@ +/* Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "ScreenEventLog.h" +#include "ScreenEventTypes.h" +#include "ScreenFades.h" +#include "ScreenEvents.h" + +namespace PokemonAutomation{ + + +std::string screen_event_name(ScreenEvent event){ + switch (event){ + case ScreenEvent::FADE_BLACK: return "black fade"; + case ScreenEvent::FADE_WHITE: return "white fade"; + case ScreenEvent::LOCATION_BANNER: return "location banner"; + case ScreenEvent::DIALOG_BOX: return "dialog box"; + case ScreenEvent::CHOICE_BOX: return "choice box"; + case ScreenEvent::INTERACT_PROMPT: return "interact prompt"; + case ScreenEvent::BATTLE_STARTED: return "battle started"; + case ScreenEvent::BATTLE_ENDED: return "battle ended"; + case ScreenEvent::PLAYER_CAN_MOVE: return "player can move"; + case ScreenEvent::SCREEN_STILL: return "screen still"; + case ScreenEvent::SWITCH_HOME_MENU: return "Switch HOME menu"; + } + return "?"; +} + +std::string warp_fade_name(WarpFade fade){ + switch (fade){ + case WarpFade::NONE: return "no fade"; + case WarpFade::BLACK: return "black fade"; + case WarpFade::WHITE: return "white fade"; + } + return "?"; +} + + +ScreenEvents::ScreenEvents(std::initializer_list events){ + for (ScreenEvent event : events){ + set(event, true); + } +} +void ScreenEvents::set(ScreenEvent event, bool on){ + if (on){ + m_bits |= bit(event); + }else{ + m_bits &= ~bit(event); + } +} +ScreenEvents ScreenEvents::from_bits(uint32_t bits){ + ScreenEvents out; + out.m_bits = bits & ((uint32_t(1) << SCREEN_EVENT_COUNT) - 1); + return out; +} + +ScreenEvents fade_events(WarpFade fade){ + switch (fade){ + case WarpFade::NONE: + return {}; + case WarpFade::BLACK: + return {ScreenEvent::FADE_BLACK}; + case WarpFade::WHITE: + return {ScreenEvent::FADE_WHITE}; + } + return {}; +} +WarpFade fade_in(const ScreenEvents& events){ + if (events.has(ScreenEvent::FADE_BLACK)){ + return WarpFade::BLACK; + } + if (events.has(ScreenEvent::FADE_WHITE)){ + return WarpFade::WHITE; + } + return WarpFade::NONE; +} +std::string describe(const ScreenEvents& events, std::string_view separator){ + std::string out; + for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){ + const ScreenEvent event = (ScreenEvent)c; + if (!events.has(event)){ + continue; + } + if (!out.empty()){ + out += separator; + } + out += screen_event_name(event); + } + return out.empty() ? "nothing" : out; +} + + +WarpFade fade_of(const ImageStats& middle){ + if (middle.stddev.sum() > FADE_MAX_STDDEV_SUM){ + return WarpFade::NONE; + } + const double sum = middle.average.sum(); + return sum <= FADE_MAX_DARK_SUM ? WarpFade::BLACK : sum >= FADE_MIN_WHITE_SUM ? WarpFade::WHITE : WarpFade::NONE; +} +WarpFade fade_of_screen(const ImageViewRGB32& screen){ + return fade_of(image_stats(extract_box_reference(screen, ImageFloatBox(0.1, 0.1, 0.8, 0.8)))); +} + + +ScreenEventLog::ScreenEventLog(size_t capacity) + : m_capacity(capacity) +{} +void ScreenEventLog::record(const ScreenEvents& frame, WallClock now){ + const uint32_t changed = frame.bits() ^ m_showing.bits(); + for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){ + if ((changed & (uint32_t(1) << c)) == 0){ + continue; + } + const ScreenEvent event = (ScreenEvent)c; + m_changes.push_back({now, event, frame.has(event) ? ScreenEventEdge::ROSE : ScreenEventEdge::FELL}); + } + while (m_changes.size() > m_capacity){ + m_changes.pop_front(); + m_dropped++; + } + m_showing = frame; + m_last_frame = now; +} +void ScreenEventLog::clear(){ + m_changes.clear(); + m_dropped = 0; + m_showing = ScreenEvents(); + m_last_frame.reset(); +} +std::string describe(const ScreenEventLog& log, WallClock origin){ + std::string out; + if (log.dropped() > 0){ + out += std::format("({} older changes dropped)\n", log.dropped()); + } + for (const ScreenEventChange& change : log.changes()){ + out += std::format( + "{:+} ms {} {}\n", + std::chrono::duration_cast(change.time - origin).count(), + screen_event_name(change.event), + change.edge == ScreenEventEdge::ROSE ? "rose" : "fell" + ); + } + return out.empty() ? "no changes" : out; +} + + +ScreenEventWatcher::ScreenEventWatcher(ScreenEventObserver& observer, ScreenEvents stop_on) + : VisualInferenceCallback("ScreenEventWatcher") + , m_observer(observer) + , m_stop_on(stop_on) +{} +void ScreenEventWatcher::make_overlays(VideoOverlaySet& items) const{ + m_observer.make_overlays(items); +} +bool ScreenEventWatcher::process_frame(const VideoSnapshot& frame){ + const ScreenEvents events = m_observer.observe(frame, frame.timestamp); + m_seen.fetch_or(events.bits(), std::memory_order_acq_rel); + { + LockGuard lg(m_log_lock); + m_log.record(events, frame.timestamp); + } + if (!events.any_of(m_stop_on)){ + return false; + } + m_stopped_on.store(events.bits(), std::memory_order_release); + return true; +} +ScreenEventLog ScreenEventWatcher::log() const{ + LockGuard lg(m_log_lock); + return m_log; +} +void ScreenEventWatcher::reset(){ + m_seen.store(0, std::memory_order_release); + m_stopped_on.store(0, std::memory_order_release); + LockGuard lg(m_log_lock); + m_log.clear(); +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.h new file mode 100644 index 0000000000..3e90b8d85d --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.h @@ -0,0 +1,83 @@ +/* Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + * What an autonomous program can see on the screen, in words every game + * shares: a fade, a location banner, a dialogue box, a battle starting. + * Each game implements ScreenEventObserver with its own detectors; the + * judgements built on these events are then the same for all of them. + * + * An observer runs during a controller action as a ScreenEventWatcher + * (a VisualInferenceCallback for run_until), or is asked frame by frame. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_H + +#include +#include +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/Time.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" +#include "ScreenEventLog.h" +#include "ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// A game's detectors, as events. Observers may keep state between frames +// (the frame before, for SCREEN_STILL; when a banner was last read, to +// gate OCR), so one observer watches one stream at a time. +class ScreenEventObserver{ +public: + virtual ~ScreenEventObserver() = default; + + // What `frame` shows. `now` is the caller's clock, for observers that + // rate-limit expensive reads. A null frame shows nothing. + virtual ScreenEvents observe(const VideoSnapshot& frame, WallClock now) = 0; + + // The boxes the detectors read, for the video overlay. + virtual void make_overlays(VideoOverlaySet& items) const = 0; +}; + + +// An observer as a run_until callback: it records every event it sees, in +// order in a log, and stops the action on any of `stop_on` (none: it only +// records). +class ScreenEventWatcher : public VisualInferenceCallback{ +public: + ScreenEventWatcher(ScreenEventObserver& observer, ScreenEvents stop_on = {}); + + virtual void make_overlays(VideoOverlaySet& items) const override; + using VisualInferenceCallback::process_frame; + virtual bool process_frame(const VideoSnapshot& frame) override; + + // Every event seen since construction (or the last reset()). + ScreenEvents seen() const{ + return ScreenEvents::from_bits(m_seen.load(std::memory_order_acquire)); + } + // The events of the frame that stopped the action ("nothing" if none did). + ScreenEvents stopped_on() const{ + return ScreenEvents::from_bits(m_stopped_on.load(std::memory_order_acquire)); + } + // A copy of the timed log of every frame since construction (or the + // last reset()). + ScreenEventLog log() const; + void reset(); + +private: + ScreenEventObserver& m_observer; + const ScreenEvents m_stop_on; + std::atomic m_seen{0}; + std::atomic m_stopped_on{0}; + mutable Mutex m_log_lock; + ScreenEventLog m_log; +}; + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.cpp b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.cpp new file mode 100644 index 0000000000..7b61ab9227 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.cpp @@ -0,0 +1,281 @@ +/* Screen Events Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The shared event vocabulary and the watcher form of an observer. + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "ScreenEventLog.h" +#include "ScreenEventTypes.h" +#include "ScreenFades.h" +#include "ScreenEvents.h" +#include "ScreenEvents_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + +using namespace std::chrono_literals; + +UnitTestResult expect_equal(const std::string& what, const std::string& got, const std::string& expected){ + if (got == expected){ + return true; + } + return std::format("{}: expected \"{}\", got \"{}\"", what, expected, got); +} + + +UnitTestResult test_set_and_describe(){ + ScreenEvents events{ScreenEvent::DIALOG_BOX, ScreenEvent::PLAYER_CAN_MOVE}; + if (!events.has(ScreenEvent::DIALOG_BOX) || events.has(ScreenEvent::CHOICE_BOX)){ + return "has() disagrees with the constructor's events"; + } + events.set(ScreenEvent::DIALOG_BOX, false); + events.merge(fade_events(WarpFade::WHITE)); + events.merge(fade_events(WarpFade::NONE)); + if (fade_in(events) != WarpFade::WHITE){ + return "fade_events(WHITE) did not give a white fade"; + } + if (ScreenEvents::from_bits(events.bits()) != events){ + return "from_bits(bits()) is not the same set"; + } + if (describe(ScreenEvents()) != "nothing"){ + return "an empty set does not describe as nothing"; + } + return expect_equal("describe()", describe(events), "white fade, player can move"); +} + +// A screen filled with one colour, or a checkerboard of two. +ImageRGB32 screen_of(uint32_t a, uint32_t b){ + ImageRGB32 out(64, 36); + for (size_t y = 0; y < out.height(); y++){ + for (size_t x = 0; x < out.width(); x++){ + out.pixel(x, y) = (x + y) % 2 == 0 ? a : b; + } + } + return out; +} + +UnitTestResult test_fade_of_screen(){ + struct Row{ + std::string name; + uint32_t a; + uint32_t b; + WarpFade fade; + }; + const std::vector rows = { + {"black", 0xff000000, 0xff000000, WarpFade::BLACK}, + {"a cave mouth's dim fade (60 a channel)", 0xff3c3c3c, 0xff3c3c3c, WarpFade::BLACK}, + {"white", 0xffffffff, 0xffffffff, WarpFade::WHITE}, + {"grey, neither", 0xff787878, 0xff787878, WarpFade::NONE}, + {"a dark textured floor", 0xff101010, 0xff505050, WarpFade::NONE}, + }; + for (const Row& row : rows){ + const WarpFade got = fade_of_screen(screen_of(row.a, row.b)); + if (got != row.fade){ + return std::format("{}: expected {}, got {}", row.name, warp_fade_name(row.fade), warp_fade_name(got)); + } + } + return true; +} + +// Replays a fixed list of event sets, one per frame. +class ScriptedObserver : public ScreenEventObserver{ +public: + explicit ScriptedObserver(std::vector frames) + : m_frames(std::move(frames)) + {} + virtual ScreenEvents observe(const VideoSnapshot&, WallClock) override{ + return m_next < m_frames.size() ? m_frames[m_next++] : ScreenEvents(); + } + virtual void make_overlays(VideoOverlaySet&) const override{} + +private: + std::vector m_frames; + size_t m_next = 0; +}; + +UnitTestResult test_watcher(){ + ScriptedObserver observer({ + {ScreenEvent::PLAYER_CAN_MOVE}, + {ScreenEvent::FADE_BLACK}, + {ScreenEvent::LOCATION_BANNER, ScreenEvent::PLAYER_CAN_MOVE}, + {ScreenEvent::DIALOG_BOX}, + }); + ScreenEventWatcher watcher(observer, {ScreenEvent::LOCATION_BANNER}); + const VideoSnapshot frame; + std::vector stops; + for (size_t c = 0; c < 4; c++){ + stops.push_back(watcher.process_frame(frame)); + } + if (stops != std::vector{false, false, true, false}){ + return "the watcher did not stop exactly on the banner frame"; + } + UnitTestResult result = expect_equal("seen()", describe(watcher.seen()), + "black fade, location banner, dialog box, player can move"); + if (result.result != UnitTestResult::PASSED){ + return result; + } + result = expect_equal("stopped_on()", describe(watcher.stopped_on()), "location banner, player can move"); + if (result.result != UnitTestResult::PASSED){ + return result; + } + watcher.reset(); + return expect_equal("seen() after reset()", describe(watcher.seen()), "nothing"); +} + + +struct ScreenEventsCase{ + struct Input{ + std::function body; + }; + struct Expectation{ + }; + + std::string name; + Input input; + Expectation expectation; +}; + +class Test_ScreenEvents : public UnitTest{ +public: + explicit Test_ScreenEvents(ScreenEventsCase test_case) + : UnitTest(std::format("CommonTools::ScreenEvents - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_case.input.body(); + } + +private: + ScreenEventsCase m_case; +}; + +} // namespace + + +const ScreenEventsCase SCREEN_EVENTS_CASES[] = { + { + .name = "a set of events, its fade and its log text", + .input = {.body = test_set_and_describe}, + .expectation = {}, + }, + { + .name = "a fade is uniform, dark or white; a textured floor is none", + .input = {.body = test_fade_of_screen}, + .expectation = {}, + }, + { + .name = "an observer as a watcher records and stops", + .input = {.body = test_watcher}, + .expectation = {}, + }, +}; + +namespace{ + +// A stream of frames into a ScreenEventLog, and its describe() text. +struct ScreenEventLogCase{ + struct Frame{ + int ms = 0; + std::vector events; + }; + struct Input{ + size_t capacity = 0; + std::vector frames; + }; + struct Expectation{ + // describe() of the log, times from the first frame. + std::string text; + size_t dropped = 0; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +class Test_ScreenEventLog : public UnitTest{ +public: + explicit Test_ScreenEventLog(ScreenEventLogCase test_case) + : UnitTest(std::format("CommonTools::ScreenEventLog - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + ScreenEventLog log(m_case.input.capacity); + const WallClock origin = current_time(); + for (const ScreenEventLogCase::Frame& frame : m_case.input.frames){ + ScreenEvents events; + for (ScreenEvent event : frame.events){ + events.set(event, true); + } + log.record(events, origin + Milliseconds(frame.ms)); + } + if (log.dropped() != m_case.expectation.dropped){ + return std::format("dropped(): expected {}, got {}", m_case.expectation.dropped, log.dropped()); + } + return expect_equal("describe()", describe(log, origin), m_case.expectation.text); + } + +private: + ScreenEventLogCase m_case; +}; + +} // namespace + +const ScreenEventLogCase SCREEN_EVENT_LOG_CASES[] = { + { + .name = "no frames", + .input = {.capacity = 8, .frames = {}}, + .expectation = {.text = "no changes", .dropped = 0}, + }, + { + .name = "the first frame's events all rise", + .input = {.capacity = 8, .frames = {{.ms = 0, .events = {ScreenEvent::DIALOG_BOX, ScreenEvent::SCREEN_STILL}}}}, + .expectation = {.text = "+0 ms dialog box rose\n+0 ms screen still rose\n", .dropped = 0}, + }, + { + .name = "only changes are kept, rising and falling", + .input = {.capacity = 8, .frames = { + {.ms = 0, .events = {ScreenEvent::PLAYER_CAN_MOVE}}, + {.ms = 100, .events = {ScreenEvent::PLAYER_CAN_MOVE}}, + {.ms = 200, .events = {ScreenEvent::FADE_BLACK}}, + {.ms = 900, .events = {}}, + }}, + .expectation = { + .text = "+0 ms player can move rose\n+200 ms black fade rose\n+200 ms player can move fell\n+900 ms black fade fell\n", + .dropped = 0, + }, + }, + { + .name = "the oldest changes drop past the capacity", + .input = {.capacity = 2, .frames = { + {.ms = 0, .events = {ScreenEvent::DIALOG_BOX}}, + {.ms = 100, .events = {}}, + {.ms = 200, .events = {ScreenEvent::CHOICE_BOX}}, + }}, + .expectation = {.text = "(1 older changes dropped)\n+100 ms dialog box fell\n+200 ms choice box rose\n", .dropped = 1}, + }, +}; + +void add_tests_ScreenEvents(UnitTestDatabase& database){ + for (const auto& item : SCREEN_EVENTS_CASES){ + database.add(item); + } + for (const auto& item : SCREEN_EVENT_LOG_CASES){ + database.add(item); + } +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.h new file mode 100644 index 0000000000..3d1c84b44f --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.h @@ -0,0 +1,21 @@ +/* Screen Events Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_Tests_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +// Registers unit tests for the shared screen events. +void add_tests_ScreenEvents(UnitTestDatabase& database); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenFades.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenFades.h new file mode 100644 index 0000000000..a153f49246 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenFades.h @@ -0,0 +1,43 @@ +/* Screen Fades + * + * From: https://github.com/PokemonAutomation/ + * + * A fade to black or white, read from a screen and as screen events. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenFades_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenFades_H + +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// The event of a fade: FADE_BLACK, FADE_WHITE, or none for WarpFade::NONE. +ScreenEvents fade_events(WarpFade fade); +// The fade `events` show: BLACK before WHITE if they have both. +WarpFade fade_in(const ScreenEvents& events); + + +// A fade is uniform over the middle 80% of the screen, the same stddev +// test as the shared Black/WhiteScreenWatcher: a dim cave floor (mean +// about 46 a channel) is textured, so it is no fade. A tighter bound +// needs checking against such dim floors first. +constexpr double FADE_MAX_STDDEV_SUM = 10; +// Dark goes to an rgb sum of 225, as a cave mouth's fade is dim, 45 to 64 +// a channel, not black. +constexpr double FADE_MAX_DARK_SUM = 225; +// White from an rgb sum of 500, as the shared WhiteScreenWatcher's. +constexpr double FADE_MIN_WHITE_SUM = 500; +// The fade `middle` (the stats of the screen's middle 80%) shows, if any. +WarpFade fade_of(const ImageStats& middle); +// fade_of() for a screen. +WarpFade fade_of_screen(const ImageViewRGB32& screen); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp new file mode 100644 index 0000000000..4be721658c --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp @@ -0,0 +1,177 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "ScreenMachine.h" + +namespace PokemonAutomation{ + +using namespace std::chrono_literals; + + +namespace{ + +const ScreenState* classify(const std::vector& states, const ImageViewRGB32& screen){ + for (const ScreenState& state : states){ + if (state.matches && state.matches(screen)){ + return &state; + } + } + return nullptr; +} + +} // namespace + + +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b){ + if (!a || !b || a->width() != b->width() || a->height() != b->height()){ + return false; + } + return ImageMatch::pixel_RMSD(a, b) < 2.0; +} + +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation){ + for (const ScreenState& state : states){ + if (state.matches_observation && state.matches_observation(observation)){ + return &state; + } + } + return nullptr; +} + + +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, + const std::vector& states, StepTrace& trace, VideoSnapshot& frame +){ + const ScreenState* last = nullptr; + WallClock video_lost = WallClock::max(); + for (size_t c = 0; c < 20; c++){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + // The app resets a frozen capture by itself, which takes a second or two. + const WallClock now = current_time(); + video_lost = std::min(video_lost, now); + if (now - video_lost > std::chrono::seconds(10)){ + trace.fail("No video from the capture card for 10s."); + } + c--; + scope.wait_for(500ms); + continue; + } + video_lost = WallClock::max(); + const ScreenState* now = classify(states, screen); + frame = std::move(screen); + if (now != nullptr && now == last){ + return *now; + } + last = now; + scope.wait_for(150ms); + } + if (last == nullptr){ + trace.fail("The screen matches none of the known states."); + } + return *last; // Still changing after 3s: go with the latest read. +} + + +void ScreenMachineChecks::check(const ScreenState& state, VideoSnapshot frame){ + const std::optional found = problem(state, std::move(frame), current_time()); + if (found){ + m_trace.fail(*found); + } +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, const ScreenObservation& observation){ + return problem(state, observation.frame ? *observation.frame : VideoSnapshot(), observation.when); +} + +std::optional ScreenMachineChecks::unchanged_problem(const ScreenState& state, WallClock when){ + if (state.max_still > Milliseconds::zero()){ + if (when - m_changed_at > state.max_still){ + return std::format("The screen stayed still on \"{}\" for more than {}s.", state.name, state.max_still.count() / 1000); + } + return std::nullopt; + } + if (++m_unchanged >= MAX_UNCHANGED){ + return std::format("The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name); + } + return std::nullopt; +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, VideoSnapshot frame, WallClock when){ + if (same_screen(m_last_frame, frame)){ + std::optional stalled = unchanged_problem(state, when); + if (stalled){ + return stalled; + } + }else{ + m_unchanged = 0; + m_changed_at = when; + } + m_last_frame = std::move(frame); + + if (m_previous != nullptr && m_previous != &state && !m_previous->next.empty() && + std::find(m_previous->next.begin(), m_previous->next.end(), state.name) == m_previous->next.end() + ){ + return std::format("Unexpected screen after \"{}\": \"{}\".", m_previous->name, state.name); + } + m_in_a_row = m_previous == &state ? m_in_a_row + 1 : 1; + if (m_in_a_row > state.max_in_a_row){ + return std::format("Screen \"{}\" came up {} times in a row: the step for it is not getting anywhere.", state.name, m_in_a_row); + } + if (++m_visits[state.name] > state.max_visits){ + return std::format("Screen \"{}\" came up more than {} times: the step for it is not working.", state.name, state.max_visits); + } + m_previous = &state; + return std::nullopt; +} + + +ScreenMachineResult run_screen_machine_on_observations( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +){ + // Looks before the run stops even when nothing else ends it (looks with + // no time, states with no visit limit). + constexpr size_t MAX_LOOKS = 10000; + ScreenMachineChecks checks(trace); + std::optional start; + for (size_t looks = 0; looks < MAX_LOOKS; looks++){ + scope.throw_if_cancelled(); + const ScreenObservation observation = observe(); + if (!start){ + start = observation.when; + } + if (observation.when - *start > timeout){ + return {.terminal = "", .failure = std::format("Did not finish within {}s.", timeout.count() / 1000)}; + } + const ScreenState* state = match_screen_state(states, observation); + if (state == nullptr){ + return {.terminal = "", .failure = "The screen matches none of the known states."}; + } + trace.screen(state->name); + std::optional found = checks.problem(*state, observation); + if (found){ + return {.terminal = "", .failure = std::move(*found)}; + } + if (state->act){ + state->act(); + } + if (state->terminal){ + return {.terminal = state->name, .failure = ""}; + } + } + return {.terminal = "", .failure = std::format("Did not finish within {} looks.", MAX_LOOKS)}; +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h new file mode 100644 index 0000000000..4f11d7066e --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h @@ -0,0 +1,169 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + * A closed loop over known screens: read the screen, take the one step that screen + * calls for, read again. Every step follows a read, a screen counts only once two + * frames agree, and each state lists the screens it may lead to, so a dropped or + * wrong input shows up as an unexpected screen or a state that keeps coming back. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_H +#define PokemonAutomation_CommonTools_ScreenMachine_H + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/CancellableScope.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "ScreenObservation.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +// Single recognizable screen state, including detection predicate and handler action. +struct ScreenState{ + std::string name; + // Whether this is the screen. States are tried in order; the first match wins, so + // the last state should match anything. + std::function matches; + // Whether this is the screen, from what a look at the game observed + // instead of from its pixels, so the state also runs in a simulation. + // A state may have either or both; read_screen_state() uses `matches` + // and skips a state without it. + std::function matches_observation = nullptr; + // The one step to take on this screen. + std::function act; + // The screens the step may lead to besides staying here. Empty: any. + std::vector next; + // Fail once the screen has been seen this many times. + size_t max_visits; + // End the run after this state's step. + bool terminal = false; + // Fail once the screen has been seen this many times in a row. + size_t max_in_a_row = SIZE_MAX; + // How long the screen may stay unchanged on this state. A waiting + // state (a fade, a still cutscene, a hold before a screen counts) sets + // it: its looks never count as a stall, and it fails only once the + // screen has not changed for longer than this. Zero: the state's step + // is input that should change the screen, and MAX_UNCHANGED such + // steps that do not are a stall. + Milliseconds max_still = Milliseconds::zero(); +}; + + +// The state on screen, once two frames 150ms apart agree. Waits out a video +// gap of up to 10s. Fails through `trace` if no state matches. +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, + const std::vector& states, StepTrace& trace, VideoSnapshot& frame +); + +// The first of `states` whose matches_observation() accepts `observation`; +// null when none does. States without one are skipped. +const ScreenState* match_screen_state(const std::vector& states, const ScreenObservation& observation); + +// The checks between reads: the screen must be one the previous state can lead +// to, no state may come up too often, and steps must keep changing the screen +// (a waiting state's only for its ScreenState::max_still). +class ScreenMachineChecks{ +public: + ScreenMachineChecks(StepTrace& trace) + : m_trace(trace) + {} + // Fails through the trace on a problem. + void check(const ScreenState& state, VideoSnapshot frame); + // The problem with `state` coming up on `observation`, if any. The + // screen counts as unchanged only when it and the look before both have + // a frame and the frames match; how long, by the looks' `when`. + std::optional problem(const ScreenState& state, const ScreenObservation& observation); + +private: + std::optional problem(const ScreenState& state, VideoSnapshot frame, WallClock when); + // The stall problem, if any, of `state` on a frame unchanged since `m_changed_at`. + std::optional unchanged_problem(const ScreenState& state, WallClock when); + + // A step that changes nothing on screen is a dead end (e.g. a menu the states do + // not know): fail after a few instead of running out the visit limit. + static constexpr size_t MAX_UNCHANGED = 8; + + StepTrace& m_trace; + std::map m_visits; + const ScreenState* m_previous = nullptr; + VideoSnapshot m_last_frame; + // When the screen last changed (or was first read). + WallClock m_changed_at = WallClock::min(); + size_t m_unchanged = 0; + size_t m_in_a_row = 0; +}; + + +// Runs the states until a terminal one has acted. Returns its name. Fails through +// `trace` on an unexpected screen, a state seen too often, a stall or the timeout. +template +std::string run_screen_machine( + VideoStream& stream, ControllerContext& context, + const std::vector& states, Milliseconds timeout, StepTrace& trace +); + + +// How a run of states over observations ended. +struct ScreenMachineResult{ + // The terminal state that acted; empty when the run failed. + std::string terminal; + // Why it failed: no state matched, an unexpected screen, a state seen too + // often, a stall, or the timeout. + std::string failure; + + bool succeeded() const{ + return failure.empty(); + } +}; + +// The same loop as run_screen_machine() over looks from `observe` instead of +// frames, so it runs on a console and in a simulation alike: each look picks +// the first state whose matches_observation() accepts it (one look, no second +// read: an observation is already settled), the trace records it, the checks +// apply, then the state acts. Time is the looks' own (`when`): the run fails +// once `timeout` has passed since the first look, and after 10000 looks in +// any case. A failure is returned, not thrown. +ScreenMachineResult run_screen_machine_on_observations( + const std::function& observe, CancellableScope& scope, + const std::vector& states, Milliseconds timeout, StepTrace& trace +); + +template +std::string run_screen_machine( + VideoStream& stream, ControllerContext& context, + const std::vector& states, Milliseconds timeout, StepTrace& trace +){ + ScreenMachineChecks checks(trace); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + context.wait_for_all_requests(); + VideoSnapshot frame; + const ScreenState& state = read_screen_state(stream, context, states, trace, frame); + trace.screen(state.name); + checks.check(state, std::move(frame)); + state.act(); + if (state.terminal){ + context.wait_for_all_requests(); + return state.name; + } + } + trace.fail(std::format("Did not finish within {}s.", timeout.count() / 1000)); +} + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h new file mode 100644 index 0000000000..ba1e6e84b3 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h @@ -0,0 +1,47 @@ +/* Screen Observation + * + * From: https://github.com/PokemonAutomation/ + * + * What one look at the game showed, as facts rather than pixels: the + * events on screen (a fade, a dialog box, a battle) and named readings. A + * console fills it from a frame with the game's detectors; a simulation + * fills it from its model and has no frame. A ScreenState that matches on + * an observation (ScreenState::matches_observation) therefore runs in + * both. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenObservation_H +#define PokemonAutomation_CommonTools_ScreenObservation_H + +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/ScreenEvents/ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// One look at the game. +struct ScreenObservation{ + WallClock when = WallClock::min(); + ScreenEvents events = {}; + // Named numbers a reader found: "bag.RARE_CANDY", "facing", + // "heading" (degrees clockwise from north, [0, 360)). + std::map> readings = {}; + // The frame looked at; none in a simulation. A state that needs it + // can only run on a console. + std::optional frame = std::nullopt; +}; + +// Whether `a` and `b` both have a frame of the same size within the +// still-screen RMSD threshold (2.0). +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp new file mode 100644 index 0000000000..b9a12614c8 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp @@ -0,0 +1,104 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/PrettyPrint.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +StepTrace::StepTrace(VideoStream& stream, std::string label) + : m_stream(&stream) + , m_label(std::move(label)) + , m_folder(std::format("{}StepTrace/{}-{}/", SCREENSHOTS_PATH(), now_to_filestring(), m_label)) + , m_start(current_time()) +{ + Filesystem::create_directories(Filesystem::Path(m_folder)); + log(std::format("[{}] Trace screenshots: {}", m_label, m_folder)); +} + +StepTrace::StepTrace(std::string label, std::function log) + : m_log(std::move(log)) + , m_label(std::move(label)) + , m_start(current_time()) +{} + +void StepTrace::log(const std::string& line) const{ + if (m_stream != nullptr){ + m_stream->log(line); + }else if (m_log){ + m_log(line); + } +} + +std::string StepTrace::save_screenshot(const std::string& name) const{ + if (m_stream == nullptr){ + return "(no video)"; + } + const std::string path = std::format("{}{:03}-{}.png", m_folder, m_count, name); + VideoSnapshot screen = m_stream->video().snapshot(); + if (!screen){ + return "(no video frame)"; + } + screen.frame->save(path); + return path; +} + +void StepTrace::screen(const std::string& name){ + if (name == m_last){ + return; + } + m_last = name; + m_recent.push_back(name); + if (m_recent.size() > 12){ + m_recent.pop_front(); + } + m_count++; + const double seconds = std::chrono::duration(current_time() - m_start).count(); + const std::string path = save_screenshot(name); + log(std::format("[{}] screen {}: {} at {}s, {}", m_label, m_count, name, tostr_fixed(seconds, 1), path)); +} + +void StepTrace::action(const std::string& what){ + log(std::format("[{}] {}", m_label, what)); +} + +std::string StepTrace::recent_screens() const{ + std::string text; + for (const std::string& name : m_recent){ + if (!text.empty()){ + text += " -> "; + } + text += name; + } + return text.empty() ? "(none)" : text; +} + +void StepTrace::fail(const std::string& message) const{ + if (m_stream == nullptr){ + throw OperationFailedException( + ErrorReportMode::NO_ERROR_REPORT, + std::format("{}: {} Recent screens: {}.", m_label, message, recent_screens()) + ); + } + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("{}: {} Recent screens: {}. Trace: {}", m_label, message, recent_screens(), m_folder), + *m_stream + ); +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h new file mode 100644 index 0000000000..7b125a37cf --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h @@ -0,0 +1,67 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + * A step log for closed-loop routines: every change of recognized screen is + * logged and saved as a numbered screenshot, every input is logged, and a + * failure reports the screens that led up to it. A trace of observations + * (ScreenObservation.h), which may come from a simulation, logs the same + * lines with no screenshots. + * + */ + +#ifndef PokemonAutomation_CommonTools_StepTrace_H +#define PokemonAutomation_CommonTools_StepTrace_H + +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/Tools/VideoStream.h" + +namespace PokemonAutomation{ + + +// Recorder that captures screen transitions and recent actions for failure diagnostics. +class StepTrace{ +public: + // Screenshots go to /StepTrace/