diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 1c698ab707..4ab69e6c79 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -81,6 +81,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ Pokemon::add_tests_MapData(ret); Pokemon::add_tests_RuntimeMap(ret); Pokemon::add_tests_RouteLog(ret); + Pokemon::add_tests_ScreenEvents(ret); Pokemon::add_tests_StoryEvents(ret); Pokemon::add_tests_StorySegments(ret); Pokemon::add_tests_StorySteps(ret); diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h index 5d0bbe1172..237cd643d7 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h @@ -15,6 +15,8 @@ namespace PokemonAutomation{ namespace Pokemon{ +// Registers unit tests for the shared screen events and the judgements over them. +void add_tests_ScreenEvents(UnitTestDatabase& database); // Registers unit tests for story event sequencing and execution. void add_tests_StoryEvents(UnitTestDatabase& database); // Registers unit tests for story segment running, verification, and branching. diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_SceneJudges.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_SceneJudges.cpp new file mode 100644 index 0000000000..628be668dc --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_SceneJudges.cpp @@ -0,0 +1,85 @@ +/* Scene Judges + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Pokemon_SceneJudges.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +WarpSeen note_warp(WarpSeen seen, const ScreenEvents& events, const std::string& banner_read){ + if (seen.fade == WarpFade::NONE){ + seen.fade = fade_in(events); + } + if (seen.banner.empty() && events.has(ScreenEvent::LOCATION_BANNER)){ + seen.banner = banner_read; + } + return seen; +} + +std::string describe(const WarpSeen& seen){ + if (!seen.any()){ + return "nothing"; + } + std::string out = seen.fade == WarpFade::NONE ? "" : warp_fade_name(seen.fade); + if (!seen.banner.empty()){ + out += (out.empty() ? "banner \"" : ", banner \"") + seen.banner + "\""; + } + return out; +} + + +SceneMove TalkOver::look(const ScreenEvents& look, WallClock now){ + if (look.has(ScreenEvent::BATTLE_STARTED)){ + m_quiet_since = WallClock::max(); + m_calm_since = WallClock::max(); + return SceneMove::BATTLE; + } + if (fade_in(look) != WarpFade::NONE){ + m_fade = fade_in(look); + } + m_banner |= look.has(ScreenEvent::LOCATION_BANNER); + const bool box = look.has(ScreenEvent::DIALOG_BOX); + const bool calm = !box && look.has(ScreenEvent::PLAYER_CAN_MOVE) && warped(); + const bool quiet = calm && look.has(ScreenEvent::SCREEN_STILL); + if (!calm){ + m_calm_since = WallClock::max(); + }else if (m_calm_since == WallClock::max()){ + m_calm_since = now; + } + if (!quiet){ + m_quiet_since = WallClock::max(); + }else if (m_quiet_since == WallClock::max()){ + m_quiet_since = now; + } + const bool over = + (quiet && now - m_quiet_since >= TALK_OVER_QUIET) || + (calm && now - m_calm_since >= TALK_OVER_CALM); + if (over && m_rule.end_on_quiet){ + return SceneMove::DONE; + } + return box ? SceneMove::NEXT : SceneMove::B; +} + +bool TalkOver::warped() const{ + return !m_rule.warp || m_fade != WarpFade::NONE || m_banner; +} + +std::string TalkOver::describe() const{ + if (m_fade == WarpFade::NONE && !m_banner){ + return "no fade or banner"; + } + std::string out = m_fade == WarpFade::NONE ? "" : warp_fade_name(m_fade); + if (m_banner){ + out += out.empty() ? "banner read" : ", banner read"; + } + return out; +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_SceneJudges.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_SceneJudges.h new file mode 100644 index 0000000000..704ecf9a86 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_SceneJudges.h @@ -0,0 +1,99 @@ +/* Scene Judges + * + * From: https://github.com/PokemonAutomation/ + * + * Judgements a story step makes from screen events (Pokemon_ScreenEvents.h) + * over time, not from one look (#268's design note): a warp is a fade or + * the banner of the map arrived on, and a talk or a scene is over when the + * overworld has been quiet for a while. Pure: tests drive them with + * scripted scenes, and every game shares them. + * + */ + +#ifndef PokemonAutomation_Pokemon_SceneJudges_H +#define PokemonAutomation_Pokemon_SceneJudges_H + +#include +#include "Common/Cpp/Time.h" +#include "Pokemon_ScreenEvents.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// What a walk or a step saw of a warp (#268): a fade or the location +// banner of the map arrived on. Either one alone is the warp. +struct WarpSeen{ + WarpFade fade = WarpFade::NONE; + // The banner read, "" if none. + std::string banner = {}; + + bool any() const{ + return fade != WarpFade::NONE || !banner.empty(); + } +}; +// `seen` after a frame's events: their fade, and `banner_read` when they +// show LOCATION_BANNER. The first of each counts. +WarpSeen note_warp(WarpSeen seen, const ScreenEvents& events, const std::string& banner_read); +// How it was seen, for the log: a black fade, banner "Route 203". +std::string describe(const WarpSeen& seen); + + +// What to do after a look. +enum class SceneMove{ + NEXT, // A box shows: press the talk's button. + B, // No box: press B. It never starts a talk on the overworld + // (A would, and answers Yes on a prompt). + BATTLE, // A battle started. + DONE, // The talk or scene is over. +}; + +// What ends a talk-over, besides the quiet. +struct TalkOverRule{ + // A map change must come first: a fade or the banner. Only the quiet + // after it counts: r30's scene went quiet for 5 s before the fade home. + bool warp = false; + // False when a battle is to follow (a trainer's lines pause for more + // than 4 s before it, r210d): no quiet ends the talk then. + bool end_on_quiet = true; +}; + +// The overworld with no box and a still screen this long ends it: a scene +// pauses while someone walks off (Dawn and Barry after Rowan leaves, r23), +// a scripted walk-in ran 5 s with no text before Mom spoke (r31), and +// Roark's badge speech pauses for the fanfare (lab-B 2026-10-31 09:45 and +// 2026-11-08 15:06). The clock is the last look that was not quiet. +constexpr Milliseconds TALK_OVER_QUIET = Milliseconds(5000); +// The overworld with no box this long ends it, still or not: wanderers or +// grass can keep the screen from ever being still. +constexpr Milliseconds TALK_OVER_CALM = Milliseconds(15000); + +// When a talk or a scene is over, from looks alone. A look is a frame's +// events: DIALOG_BOX, PLAYER_CAN_MOVE, SCREEN_STILL, a fade, +// LOCATION_BANNER (the expected map's) and BATTLE_STARTED are read. +class TalkOver{ +public: + explicit TalkOver(TalkOverRule rule = {}) + : m_rule(rule) + {} + + SceneMove look(const ScreenEvents& look, WallClock now); + + // Whether the warp the rule asks for has shown (true without one). + bool warped() const; + // What it has seen, for the log: "black fade, banner read". + std::string describe() const; + +private: + TalkOverRule m_rule; + WarpFade m_fade = WarpFade::NONE; + bool m_banner = false; + WallClock m_quiet_since = WallClock::max(); + WallClock m_calm_since = WallClock::max(); +}; + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents.cpp new file mode 100644 index 0000000000..d8ea3b5beb --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents.cpp @@ -0,0 +1,132 @@ +/* Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "Pokemon_ScreenEvents.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +const char* screen_event_name(ScreenEvent event){ + switch (event){ + case ScreenEvent::FADE_BLACK: return "black fade"; + case ScreenEvent::FADE_WHITE: return "white fade"; + case ScreenEvent::LOCATION_BANNER: return "location banner"; + case ScreenEvent::DIALOG_BOX: return "dialog box"; + case ScreenEvent::CHOICE_BOX: return "choice box"; + case ScreenEvent::INTERACT_PROMPT: return "interact prompt"; + case ScreenEvent::BATTLE_STARTED: return "battle started"; + case ScreenEvent::BATTLE_ENDED: return "battle ended"; + case ScreenEvent::PLAYER_CAN_MOVE: return "player can move"; + case ScreenEvent::SCREEN_STILL: return "screen still"; + } + return "?"; +} + +const char* warp_fade_name(WarpFade fade){ + switch (fade){ + case WarpFade::NONE: return "no fade"; + case WarpFade::BLACK: return "black fade"; + case WarpFade::WHITE: return "white fade"; + } + return "?"; +} + + +ScreenEvents::ScreenEvents(std::initializer_list events){ + for (ScreenEvent event : events){ + set(event, true); + } +} +void ScreenEvents::set(ScreenEvent event, bool on){ + if (on){ + m_bits |= bit(event); + }else{ + m_bits &= ~bit(event); + } +} +ScreenEvents ScreenEvents::from_bits(uint32_t bits){ + ScreenEvents out; + out.m_bits = bits & ((uint32_t(1) << SCREEN_EVENT_COUNT) - 1); + return out; +} + +ScreenEvents fade_events(WarpFade fade){ + switch (fade){ + case WarpFade::NONE: + return {}; + case WarpFade::BLACK: + return {ScreenEvent::FADE_BLACK}; + case WarpFade::WHITE: + return {ScreenEvent::FADE_WHITE}; + } + return {}; +} +WarpFade fade_in(const ScreenEvents& events){ + if (events.has(ScreenEvent::FADE_BLACK)){ + return WarpFade::BLACK; + } + if (events.has(ScreenEvent::FADE_WHITE)){ + return WarpFade::WHITE; + } + return WarpFade::NONE; +} +std::string describe(const ScreenEvents& events){ + std::string out; + for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){ + const ScreenEvent event = (ScreenEvent)c; + if (!events.has(event)){ + continue; + } + if (!out.empty()){ + out += ", "; + } + out += screen_event_name(event); + } + return out.empty() ? "nothing" : out; +} + + +WarpFade fade_of(const ImageStats& middle){ + if (middle.stddev.sum() > FADE_MAX_STDDEV_SUM){ + return WarpFade::NONE; + } + const double sum = middle.average.sum(); + return sum <= FADE_MAX_DARK_SUM ? WarpFade::BLACK : sum >= FADE_MIN_WHITE_SUM ? WarpFade::WHITE : WarpFade::NONE; +} +WarpFade fade_of_screen(const ImageViewRGB32& screen){ + return fade_of(image_stats(extract_box_reference(screen, ImageFloatBox(0.1, 0.1, 0.8, 0.8)))); +} + + +ScreenEventWatcher::ScreenEventWatcher(ScreenEventObserver& observer, ScreenEvents stop_on) + : VisualInferenceCallback("ScreenEventWatcher") + , m_observer(observer) + , m_stop_on(stop_on) +{} +void ScreenEventWatcher::make_overlays(VideoOverlaySet& items) const{ + m_observer.make_overlays(items); +} +bool ScreenEventWatcher::process_frame(const VideoSnapshot& frame){ + const ScreenEvents events = m_observer.observe(frame, frame.timestamp); + m_seen.fetch_or(events.bits(), std::memory_order_acq_rel); + if (!events.any_of(m_stop_on)){ + return false; + } + m_stopped_on.store(events.bits(), std::memory_order_release); + return true; +} +void ScreenEventWatcher::reset(){ + m_seen.store(0, std::memory_order_release); + m_stopped_on.store(0, std::memory_order_release); +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents.h new file mode 100644 index 0000000000..1bf6ca6649 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents.h @@ -0,0 +1,168 @@ +/* Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + * What a story step can see on the screen, in words every game shares: a + * fade, a location banner, a dialogue box, a battle starting. Each game + * implements ScreenEventObserver with its own detectors; the judgements + * built on these events (Pokemon_SceneJudges.h) are then the same for all + * of them. + * + * An observer runs during a controller action as a ScreenEventWatcher + * (a VisualInferenceCallback for run_until), or is asked frame by frame. + * + */ + +#ifndef PokemonAutomation_Pokemon_ScreenEvents_H +#define PokemonAutomation_Pokemon_ScreenEvents_H + +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// One thing a frame shows. A game reports the ones its detectors can see. +enum class ScreenEvent : uint8_t{ + FADE_BLACK, // A uniform dark screen: a door, a cave mouth, a scene cut. + FADE_WHITE, // A uniform white screen: a cave exit. + LOCATION_BANNER, // A location banner. An observer given an expected + // map reports only that map's. + DIALOG_BOX, // A dialogue box. + CHOICE_BOX, // A choice (Yes/No, a menu of answers). + INTERACT_PROMPT, // A button prompt to interact ("A Talk"). + BATTLE_STARTED, // A battle shows (its menu, its HUD or its opening). + BATTLE_ENDED, // A battle's closing screen. + PLAYER_CAN_MOVE, // The overworld with the player in control. + SCREEN_STILL, // The screen barely changed since the frame before. +}; +// How many ScreenEvent values there are. +constexpr size_t SCREEN_EVENT_COUNT = 10; +// "dialog box", for the log. +const char* screen_event_name(ScreenEvent event); + + +// How a screen's fade looks: the kind of FADE_BLACK / FADE_WHITE. +enum class WarpFade{ + NONE, + BLACK, // a door's black, or a cave mouth's dim fade + WHITE, // a cave exit's white, with no battle after it +}; +// "black fade", for the log. +const char* warp_fade_name(WarpFade fade); + + +// The events of one frame, or of all the frames a watcher saw. +class ScreenEvents{ +public: + ScreenEvents() = default; + ScreenEvents(std::initializer_list events); + + bool has(ScreenEvent event) const{ + return (m_bits & bit(event)) != 0; + } + // Whether any of `events` is in this set. + bool any_of(const ScreenEvents& events) const{ + return (m_bits & events.m_bits) != 0; + } + // Adds `event`, or removes it when `on` is false (as std::bitset::set). + void set(ScreenEvent event, bool on); + void merge(const ScreenEvents& events){ + m_bits |= events.m_bits; + } + + uint32_t bits() const{ + return m_bits; + } + static ScreenEvents from_bits(uint32_t bits); + + bool operator==(const ScreenEvents& other) const = default; + +private: + static uint32_t bit(ScreenEvent event){ + return uint32_t(1) << (uint32_t)event; + } + + uint32_t m_bits = 0; +}; +// The event of a fade: FADE_BLACK, FADE_WHITE, or none for WarpFade::NONE. +ScreenEvents fade_events(WarpFade fade); +// The fade `events` show: BLACK before WHITE if they have both. +WarpFade fade_in(const ScreenEvents& events); +// "dialog box, player can move"; "nothing" when empty. +std::string describe(const ScreenEvents& events); + + +// A fade is uniform over the middle 80% of the screen, the shared +// Black/WhiteScreenWatcher's stddev test (upstream #1717): a dim cave floor +// is textured, so it is no fade (BDSP's Oreburgh Gate, mean 46, lab-B +// 2026-11-08). Check against the Gate's frames on the console before +// tightening. +constexpr double FADE_MAX_STDDEV_SUM = 10; +// Dark goes to an rgb sum of 225, as a cave mouth's fade is dim, 45 to 64 +// a channel, not black. +constexpr double FADE_MAX_DARK_SUM = 225; +// White from an rgb sum of 500, as the shared WhiteScreenWatcher's. +constexpr double FADE_MIN_WHITE_SUM = 500; +// The fade `middle` (the stats of the screen's middle 80%) shows, if any. +WarpFade fade_of(const ImageStats& middle); +// fade_of() for a screen. +WarpFade fade_of_screen(const ImageViewRGB32& screen); + + +// A game's detectors, as events. Observers may keep state between frames +// (the frame before, for SCREEN_STILL; when a banner was last read, to +// gate OCR), so one observer watches one stream at a time. +class ScreenEventObserver{ +public: + virtual ~ScreenEventObserver() = default; + + // What `frame` shows. `now` is the caller's clock, for observers that + // rate-limit expensive reads. A null frame shows nothing. + virtual ScreenEvents observe(const VideoSnapshot& frame, WallClock now) = 0; + + // The boxes the detectors read, for the video overlay. + virtual void make_overlays(VideoOverlaySet& items) const = 0; +}; + + +// An observer as a run_until callback: it records every event it sees and +// stops the action on any of `stop_on` (none: it only records). +class ScreenEventWatcher : public VisualInferenceCallback{ +public: + ScreenEventWatcher(ScreenEventObserver& observer, ScreenEvents stop_on = {}); + + virtual void make_overlays(VideoOverlaySet& items) const override; + using VisualInferenceCallback::process_frame; + virtual bool process_frame(const VideoSnapshot& frame) override; + + // Every event seen since construction (or the last reset()). + ScreenEvents seen() const{ + return ScreenEvents::from_bits(m_seen.load(std::memory_order_acquire)); + } + // The events of the frame that stopped the action ("nothing" if none did). + ScreenEvents stopped_on() const{ + return ScreenEvents::from_bits(m_stopped_on.load(std::memory_order_acquire)); + } + void reset(); + +private: + ScreenEventObserver& m_observer; + const ScreenEvents m_stop_on; + std::atomic m_seen{0}; + std::atomic m_stopped_on{0}; +}; + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents_Tests.cpp new file mode 100644 index 0000000000..2d0db31cfd --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_ScreenEvents_Tests.cpp @@ -0,0 +1,269 @@ +/* Screen Events Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The shared event vocabulary, the watcher form of an observer, and the + * judgements over events (warp seen, talk-over). + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "Pokemon_AutoStory_Tests.h" +#include "Pokemon_SceneJudges.h" +#include "Pokemon_ScreenEvents.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + +namespace{ + +using namespace std::chrono_literals; + +UnitTestResult expect_equal(const std::string& what, const std::string& got, const std::string& expected){ + if (got == expected){ + return true; + } + return std::format("{}: expected \"{}\", got \"{}\"", what, expected, got); +} + + +UnitTestResult test_set_and_describe(){ + ScreenEvents events{ScreenEvent::DIALOG_BOX, ScreenEvent::PLAYER_CAN_MOVE}; + if (!events.has(ScreenEvent::DIALOG_BOX) || events.has(ScreenEvent::CHOICE_BOX)){ + return "has() disagrees with the constructor's events"; + } + events.set(ScreenEvent::DIALOG_BOX, false); + events.merge(fade_events(WarpFade::WHITE)); + events.merge(fade_events(WarpFade::NONE)); + if (fade_in(events) != WarpFade::WHITE){ + return "fade_events(WHITE) did not give a white fade"; + } + if (ScreenEvents::from_bits(events.bits()) != events){ + return "from_bits(bits()) is not the same set"; + } + if (describe(ScreenEvents()) != "nothing"){ + return "an empty set does not describe as nothing"; + } + return expect_equal("describe()", describe(events), "white fade, player can move"); +} + +// A screen filled with one colour, or a checkerboard of two. +ImageRGB32 screen_of(uint32_t a, uint32_t b){ + ImageRGB32 out(64, 36); + for (size_t y = 0; y < out.height(); y++){ + for (size_t x = 0; x < out.width(); x++){ + out.pixel(x, y) = (x + y) % 2 == 0 ? a : b; + } + } + return out; +} + +UnitTestResult test_fade_of_screen(){ + struct Row{ + const char* name; + uint32_t a; + uint32_t b; + WarpFade fade; + }; + const std::vector rows = { + {"black", 0xff000000, 0xff000000, WarpFade::BLACK}, + {"a cave mouth's dim fade (60 a channel)", 0xff3c3c3c, 0xff3c3c3c, WarpFade::BLACK}, + {"white", 0xffffffff, 0xffffffff, WarpFade::WHITE}, + {"grey, neither", 0xff787878, 0xff787878, WarpFade::NONE}, + {"a dark textured floor", 0xff101010, 0xff505050, WarpFade::NONE}, + }; + for (const Row& row : rows){ + const WarpFade got = fade_of_screen(screen_of(row.a, row.b)); + if (got != row.fade){ + return std::format("{}: expected {}, got {}", row.name, warp_fade_name(row.fade), warp_fade_name(got)); + } + } + return true; +} + +UnitTestResult test_warp_seen(){ + WarpSeen seen = note_warp({}, {ScreenEvent::DIALOG_BOX}, "Route 203"); + if (seen.any()){ + return "a dialog box with no banner event counted as a warp"; + } + seen = note_warp(seen, {ScreenEvent::FADE_BLACK}, ""); + seen = note_warp(seen, {ScreenEvent::FADE_WHITE, ScreenEvent::LOCATION_BANNER}, "Route 203"); + seen = note_warp(seen, {ScreenEvent::LOCATION_BANNER}, "Oreburgh Gate"); + return expect_equal("describe(WarpSeen)", describe(seen), "black fade, banner \"Route 203\""); +} + + +// A scripted scene: a look per beat, each `every` apart. Returns the beat +// TalkOver first says `want` at, or -1. +int first_beat(TalkOver& judge, const std::vector& beats, SceneMove want, Milliseconds every = 1000ms){ + WallClock now = WallClock(); + for (size_t c = 0; c < beats.size(); c++){ + if (judge.look(beats[c], now) == want){ + return (int)c; + } + now += every; + } + return -1; +} + +const ScreenEvents QUIET{ScreenEvent::PLAYER_CAN_MOVE, ScreenEvent::SCREEN_STILL}; +const ScreenEvents CALM{ScreenEvent::PLAYER_CAN_MOVE}; +const ScreenEvents BOX{ScreenEvent::DIALOG_BOX, ScreenEvent::PLAYER_CAN_MOVE, ScreenEvent::SCREEN_STILL}; + +std::vector repeat(const ScreenEvents& look, size_t count){ + return std::vector(count, look); +} +std::vector concat(std::vector a, const std::vector& b){ + a.insert(a.end(), b.begin(), b.end()); + return a; +} + +UnitTestResult test_talk_over(){ + struct Row{ + const char* name; + TalkOverRule rule; + std::vector beats; + SceneMove want; + int beat; + }; + const std::vector rows = { + {"5 s of quiet ends it", {}, repeat(QUIET, 8), SceneMove::DONE, 5}, + {"a box restarts the quiet", {}, concat(concat(repeat(QUIET, 3), {BOX}), repeat(QUIET, 8)), SceneMove::DONE, 9}, + {"a box is answered", {}, {BOX}, SceneMove::NEXT, 0}, + {"no box is a B press", {}, {QUIET}, SceneMove::B, 0}, + {"15 s calm but moving ends it", {}, repeat(CALM, 20), SceneMove::DONE, 15}, + {"a warp rule waits for the fade", {.warp = true}, + concat(concat(repeat(QUIET, 8), {ScreenEvents{ScreenEvent::FADE_BLACK}}), repeat(QUIET, 8)), + SceneMove::DONE, 14}, + {"a banner is the warp too", {.warp = true}, + concat({ScreenEvents{ScreenEvent::LOCATION_BANNER, ScreenEvent::PLAYER_CAN_MOVE}}, repeat(QUIET, 8)), + SceneMove::DONE, 6}, + {"no quiet ends a talk before a battle", {.end_on_quiet = false}, repeat(QUIET, 30), SceneMove::DONE, -1}, + {"a battle", {.end_on_quiet = false}, concat(repeat(QUIET, 6), {ScreenEvents{ScreenEvent::BATTLE_STARTED}}), SceneMove::BATTLE, 6}, + }; + for (const Row& row : rows){ + TalkOver judge(row.rule); + const int got = first_beat(judge, row.beats, row.want); + if (got != row.beat){ + return std::format("{}: expected beat {}, got {} ({})", row.name, row.beat, got, judge.describe()); + } + } + return true; +} + + +// Replays a fixed list of event sets, one per frame. +class ScriptedObserver : public ScreenEventObserver{ +public: + explicit ScriptedObserver(std::vector frames) + : m_frames(std::move(frames)) + {} + virtual ScreenEvents observe(const VideoSnapshot&, WallClock) override{ + return m_next < m_frames.size() ? m_frames[m_next++] : ScreenEvents(); + } + virtual void make_overlays(VideoOverlaySet&) const override{} + +private: + std::vector m_frames; + size_t m_next = 0; +}; + +UnitTestResult test_watcher(){ + ScriptedObserver observer({ + {ScreenEvent::PLAYER_CAN_MOVE}, + {ScreenEvent::FADE_BLACK}, + {ScreenEvent::LOCATION_BANNER, ScreenEvent::PLAYER_CAN_MOVE}, + {ScreenEvent::DIALOG_BOX}, + }); + ScreenEventWatcher watcher(observer, {ScreenEvent::LOCATION_BANNER}); + const VideoSnapshot frame; + std::vector stops; + for (size_t c = 0; c < 4; c++){ + stops.push_back(watcher.process_frame(frame)); + } + if (stops != std::vector{false, false, true, false}){ + return "the watcher did not stop exactly on the banner frame"; + } + UnitTestResult result = expect_equal("seen()", describe(watcher.seen()), + "black fade, location banner, dialog box, player can move"); + if (result.result != UnitTestResult::PASSED){ + return result; + } + result = expect_equal("stopped_on()", describe(watcher.stopped_on()), "location banner, player can move"); + if (result.result != UnitTestResult::PASSED){ + return result; + } + watcher.reset(); + return expect_equal("seen() after reset()", describe(watcher.seen()), "nothing"); +} + + +struct ScreenEventsInput{ + std::string name; + std::function body; +}; + +struct ScreenEventsExpectation{ +}; + +struct ScreenEventsCase{ + ScreenEventsInput input; + ScreenEventsExpectation expectation; +}; + +class Test_ScreenEvents : public UnitTest{ +public: + explicit Test_ScreenEvents(ScreenEventsCase test_case) + : UnitTest("Pokemon::ScreenEvents - " + test_case.input.name) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_case.input.body(); + } + +private: + ScreenEventsCase m_case; +}; + +} // namespace + + +const ScreenEventsCase SCREEN_EVENTS_CASES[] = { + { + .input = {"a set of events, its fade and its log text", test_set_and_describe}, + .expectation = {}, + }, + { + .input = {"a fade is uniform, dark or white; a textured floor is none", test_fade_of_screen}, + .expectation = {}, + }, + { + .input = {"a warp is the first fade and the first banner seen", test_warp_seen}, + .expectation = {}, + }, + { + .input = {"talk-over from scripted events", test_talk_over}, + .expectation = {}, + }, + { + .input = {"an observer as a watcher records and stops", test_watcher}, + .expectation = {}, + }, +}; + +void add_tests_ScreenEvents(UnitTestDatabase& database){ + for (const auto& item : SCREEN_EVENTS_CASES){ + database.add(item); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_TalkOver.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_TalkOver.cpp index 5b51629ff2..4c371b495b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_TalkOver.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_TalkOver.cpp @@ -7,12 +7,8 @@ #include #include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" -#include "CommonTools/VisualDetectors/BlackScreenDetector.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" -#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" -#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" -#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" -#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.h" #include "PokemonBDSP_AutoStory_WildBattle.h" #include "PokemonBDSP_AutoStory_TalkOver.h" @@ -21,105 +17,14 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ -SceneMove TalkOver::look(const SceneLook& look, WallClock now){ - if (look.battle){ - m_quiet_since = WallClock::max(); - m_calm_since = WallClock::max(); - return SceneMove::BATTLE; - } - if (look.fade != WarpFade::NONE){ - m_fade = look.fade; - } - m_banner |= look.banner; - const bool calm = !look.box && look.overworld && warped(); - const bool quiet = calm && look.still; - if (!calm){ - m_calm_since = WallClock::max(); - }else if (m_calm_since == WallClock::max()){ - m_calm_since = now; - } - if (!quiet){ - m_quiet_since = WallClock::max(); - }else if (m_quiet_since == WallClock::max()){ - m_quiet_since = now; - } - const bool over = - (quiet && now - m_quiet_since >= TALK_OVER_QUIET) || - (calm && now - m_calm_since >= TALK_OVER_CALM); - if (over && m_rule.end_on_quiet){ - return SceneMove::DONE; - } - return look.box ? SceneMove::NEXT : SceneMove::B; -} - -bool TalkOver::warped() const{ - return !m_rule.warp || m_fade != WarpFade::NONE || m_banner; -} - -std::string TalkOver::describe() const{ - if (m_fade == WarpFade::NONE && !m_banner){ - return "no fade or banner"; - } - std::string out = m_fade == WarpFade::NONE ? "" : warp_fade_name(m_fade); - if (m_banner){ - out += out.empty() ? "banner read" : ", banner read"; - } - return out; -} - - -namespace{ - -// The detectors talk_over() looks with. -class SceneEyes{ -public: - SceneEyes(TalkOverWatch watch, const std::string& banner) - : m_watch(watch) - , m_banner(banner) - , m_wild_menu(BattleType::STANDARD) - , m_trainer_menu(BattleType::TRAINER) - {} - - SceneLook look(const VideoSnapshot& frame, WallClock now){ - SceneLook out; - if (!frame){ - m_previous = VideoSnapshot(); - return out; - } - const ImageViewRGB32& screen = *frame.frame; - out.battle = m_wild_menu.detect(screen) || (m_watch == TalkOverWatch::TRAINER && - (m_trainer_menu.detect(screen) || m_black.detect(screen))); - out.box = m_dialog.detect(screen); - out.overworld = m_overworld.detect(screen); - out.still = m_previous && walk_screen_change(*m_previous.frame, screen) < WALK_TAP_BLOCKED_CHANGE; - out.fade = fade_of_screen(screen); - out.banner = !m_banner.empty() && location_banner_matches(m_reads.read(screen, now), m_banner); - m_previous = frame; - return out; - } - -private: - TalkOverWatch m_watch; - const std::string& m_banner; - ShortDialogDetector m_dialog; - OverworldDetector m_overworld; - BattleMenuDetector m_wild_menu; - BattleMenuDetector m_trainer_menu; - BlackScreenDetector m_black; - LocationBannerReads m_reads; - VideoSnapshot m_previous; -}; - -} // namespace - TalkOverEnd talk_over(VideoStream& stream, ProControllerContext& context, const std::string& label, const TalkOverSpec& spec){ const WallClock deadline = current_time() + TALK_OVER_LIMIT; TalkOver judge(spec.rule); - SceneEyes eyes(spec.watch, spec.banner); + BdspScreenObserver eyes(spec.watch == TalkOverWatch::TRAINER ? BattleCue::TRAINER : BattleCue::WILD_MENU, spec.banner); while (current_time() < deadline){ context.wait_for_all_requests(); const WallClock now = current_time(); - switch (judge.look(eyes.look(stream.video().snapshot(), now), now)){ + switch (judge.look(eyes.observe(stream.video().snapshot(), now), now)){ case SceneMove::DONE: stream.log(std::format("[Talk-over] {}: over; {}.", label, judge.describe())); return TalkOverEnd::QUIET; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_TalkOver.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_TalkOver.h index 9961a86d8e..b4e5c6debe 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_TalkOver.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_TalkOver.h @@ -18,6 +18,7 @@ #include "Common/Cpp/Time.h" #include "CommonFramework/Tools/VideoStream.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "Pokemon/AutoStory/Pokemon_SceneJudges.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h" namespace PokemonAutomation{ @@ -25,50 +26,15 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ -// What one look at the screen showed. -struct SceneLook{ - // A dialogue box. - bool box = false; - // The overworld, as OverworldDetector reads it. - bool overworld = false; - // The screen barely changed since the look before (WALK_TAP_BLOCKED_CHANGE). - bool still = false; - // A fade, by fade_of(): uniform, so a dim cave floor is none. - WarpFade fade = WarpFade::NONE; - // The expected map's location banner read. - bool banner = false; - // A battle started (TalkOverWatch says what counts). - bool battle = false; -}; - +// The judgement is shared (Pokemon_SceneJudges.h): it reads the screen +// events BdspScreenObserver reports. // What to do after a look. -enum class SceneMove{ - NEXT, // A box shows: press the talk's button. - B, // No box: press B. It never starts a talk on the overworld - // (A would, and answers Yes on a prompt). - BATTLE, // A battle started. - DONE, // The talk or scene is over. -}; - +using Pokemon::SceneMove; // What ends a talk-over, besides the quiet. -struct TalkOverRule{ - // A map change must come first: a fade or the banner. Only the quiet - // after it counts: r30's scene went quiet for 5 s before the fade home. - bool warp = false; - // False when a battle is to follow (a trainer's lines pause for more - // than 4 s before it, r210d): no quiet ends the talk then. - bool end_on_quiet = true; -}; +using Pokemon::TalkOverRule; +// When a talk or a scene is over, from looks alone. +using Pokemon::TalkOver; -// The overworld with no box and a still screen this long ends it: a scene -// pauses while someone walks off (Dawn and Barry after Rowan leaves, r23), -// a scripted walk-in ran 5 s with no text before Mom spoke (r31), and -// Roark's badge speech pauses for the fanfare (lab-B 2026-10-31 09:45 and -// 2026-11-08 15:06). The clock is the last look that was not quiet. -constexpr Milliseconds TALK_OVER_QUIET = Milliseconds(5000); -// The overworld with no box this long ends it, still or not: wanderers or -// grass can keep the screen from ever being still. -constexpr Milliseconds TALK_OVER_CALM = Milliseconds(15000); // One press is held this long, constexpr Milliseconds TALK_OVER_HOLD = Milliseconds(80); // then released this long before the next look. @@ -76,28 +42,6 @@ constexpr Milliseconds TALK_OVER_RELEASE = Milliseconds(1200); // How long a talk-over may take before the step fails. constexpr Milliseconds TALK_OVER_LIMIT = Milliseconds(120000); -// The judgement, from looks alone: tests drive it with scripted scenes. -class TalkOver{ -public: - explicit TalkOver(TalkOverRule rule = {}) - : m_rule(rule) - {} - - SceneMove look(const SceneLook& look, WallClock now); - - // Whether the warp the rule asks for has shown (true without one). - bool warped() const; - // What it has seen, for the log: "black fade, banner read". - std::string describe() const; - -private: - TalkOverRule m_rule; - WarpFade m_fade = WarpFade::NONE; - bool m_banner = false; - WallClock m_quiet_since = WallClock::max(); - WallClock m_calm_since = WallClock::max(); -}; - // What counts as a battle starting during a talk-over. enum class TalkOverWatch{ diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.cpp new file mode 100644 index 0000000000..6473bf3a2b --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.cpp @@ -0,0 +1,54 @@ +/* BDSP AutoStory - Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h" +#include "PokemonBDSP_AutoStory_ScreenEvents.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using Pokemon::ScreenEvent; +using Pokemon::ScreenEvents; + + +BdspScreenObserver::BdspScreenObserver(BattleCue cue, std::string banner) + : m_cue(cue) + , m_banner(std::move(banner)) + , m_wild_menu(BattleType::STANDARD) + , m_trainer_menu(BattleType::TRAINER) +{} + +ScreenEvents BdspScreenObserver::observe(const VideoSnapshot& frame, WallClock now){ + ScreenEvents out; + if (!frame){ + m_previous = VideoSnapshot(); + return out; + } + const ImageViewRGB32& screen = *frame.frame; + out.set(ScreenEvent::BATTLE_STARTED, m_wild_menu.detect(screen) || (m_cue == BattleCue::TRAINER && + (m_trainer_menu.detect(screen) || m_black.detect(screen)))); + out.set(ScreenEvent::DIALOG_BOX, m_dialog.detect(screen)); + out.set(ScreenEvent::PLAYER_CAN_MOVE, m_overworld.detect(screen)); + out.set(ScreenEvent::SCREEN_STILL, m_previous && walk_screen_change(*m_previous.frame, screen) < WALK_TAP_BLOCKED_CHANGE); + out.merge(Pokemon::fade_events(fade_of_screen(screen))); + out.set(ScreenEvent::LOCATION_BANNER, !m_banner.empty() && location_banner_matches(m_reads.read(screen, now), m_banner)); + m_previous = frame; + return out; +} + +void BdspScreenObserver::make_overlays(VideoOverlaySet& items) const{ + m_dialog.make_overlays(items); + m_overworld.make_overlays(items); + m_wild_menu.make_overlays(items); +} + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.h new file mode 100644 index 0000000000..f5255e5948 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.h @@ -0,0 +1,64 @@ +/* BDSP AutoStory - Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + * BDSP's detectors as the shared screen events (Pokemon_ScreenEvents.h): + * the dialogue box, the overworld, a still screen, a fade, the expected + * map's location banner and a battle's start. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_ScreenEvents_H +#define PokemonAutomation_PokemonBDSP_AutoStory_ScreenEvents_H + +#include +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" +#include "Pokemon/AutoStory/Pokemon_ScreenEvents.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_OverworldDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// What BATTLE_STARTED is. +enum class BattleCue{ + // A wild battle's menu. A black screen is a fade. + WILD_MENU, + // Also a trainer's battle: its menu (Roark's read as STANDARD, r210d) + // or the black screen it opens with. + TRAINER, +}; + +// The shared events from BDSP's detectors. Not the choice box, the +// interact prompt or a battle's end: no AutoStory step reads them yet. +class BdspScreenObserver : public Pokemon::ScreenEventObserver{ +public: + // `banner`: the expected map's banner name; LOCATION_BANNER is only it + // ("" for none). + BdspScreenObserver(BattleCue cue, std::string banner); + + virtual Pokemon::ScreenEvents observe(const VideoSnapshot& frame, WallClock now) override; + virtual void make_overlays(VideoOverlaySet& items) const override; + +private: + BattleCue m_cue; + std::string m_banner; + ShortDialogDetector m_dialog; + OverworldDetector m_overworld; + BattleMenuDetector m_wild_menu; + BattleMenuDetector m_trainer_menu; + BlackScreenDetector m_black; + LocationBannerReads m_reads; + VideoSnapshot m_previous; +}; + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Simulation/PokemonBDSP_AutoStory_StorySim.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Simulation/PokemonBDSP_AutoStory_StorySim.cpp index 04ec9ce644..4aca2d5f4a 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Simulation/PokemonBDSP_AutoStory_StorySim.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Simulation/PokemonBDSP_AutoStory_StorySim.cpp @@ -58,10 +58,14 @@ std::string play_scene(const SimScene& scene, const TalkOverRule& rule){ beat_start += scene[beat].length; beat++; } - SceneLook look{.overworld = true, .still = true}; + Pokemon::ScreenEvents look{Pokemon::ScreenEvent::PLAYER_CAN_MOVE, Pokemon::ScreenEvent::SCREEN_STILL}; if (beat < scene.size()){ const SimBeat& shown = scene[beat]; - look = {.box = shown.line, .overworld = shown.overworld, .still = !shown.moving, .fade = shown.fade}; + look = {}; + look.set(Pokemon::ScreenEvent::DIALOG_BOX, shown.line); + look.set(Pokemon::ScreenEvent::PLAYER_CAN_MOVE, shown.overworld); + look.set(Pokemon::ScreenEvent::SCREEN_STILL, !shown.moving); + look.merge(Pokemon::fade_events(shown.fade)); } switch (judge.look(look, now)){ case SceneMove::DONE: diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.cpp index f1a18078fb..72fdbbfd50 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.cpp @@ -42,36 +42,6 @@ const char* walk_stop_name(WalkStop stop){ return "?"; } -const char* warp_fade_name(WarpFade fade){ - switch (fade){ - case WarpFade::NONE: return "no fade"; - case WarpFade::BLACK: return "black fade"; - case WarpFade::WHITE: return "white fade"; - } - return "?"; -} - -WarpFade fade_of(const ImageStats& middle){ - if (middle.stddev.sum() > FADE_MAX_STDDEV_SUM){ - return WarpFade::NONE; - } - const double sum = middle.average.sum(); - return sum <= FADE_MAX_DARK_SUM ? WarpFade::BLACK : sum >= FADE_MIN_WHITE_SUM ? WarpFade::WHITE : WarpFade::NONE; -} -WarpFade fade_of_screen(const ImageViewRGB32& screen){ - return fade_of(image_stats(extract_box_reference(screen, ImageFloatBox(0.1, 0.1, 0.8, 0.8)))); -} - -std::string describe(const WarpSeen& seen){ - if (!seen.any()){ - return "nothing"; - } - std::string out = seen.fade == WarpFade::NONE ? "" : warp_fade_name(seen.fade); - if (!seen.banner.empty()){ - out += (out.empty() ? "banner \"" : ", banner \"") + seen.banner + "\""; - } - return out; -} double walk_screen_change(const ImageViewRGB32& before, const ImageViewRGB32& after){ // Small copies: only the whole-view scroll matters, not detail. diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h index 311e3c784e..d3d61b6743 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h @@ -19,6 +19,7 @@ #include "CommonFramework/Tools/VideoStream.h" #include "Controllers/Joystick.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "Pokemon/AutoStory/Pokemon_SceneJudges.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -50,44 +51,22 @@ using WalkArrived = std::function; // hold where we are. enum class WalkOrigin{ CURRENT_LEG, MAY_PRECEDE_LEG }; -// How a screen's fade looks (#268's design note, slice 3). -enum class WarpFade{ - NONE, - BLACK, // a door's black, or a cave mouth's dim fade - WHITE, // a cave exit's white, with no battle after it -}; +// The fade and warp judgements are shared (Pokemon_ScreenEvents.h, +// Pokemon_SceneJudges.h); BDSP's walks and steps fill them. +// How a screen's fade looks. +using Pokemon::WarpFade; // "black fade", for the log. -const char* warp_fade_name(WarpFade fade); - -// A fade is uniform over the middle 80% of the screen, the shared -// Black/WhiteScreenWatcher's stddev test (upstream #1717): a dim cave floor -// is textured, so it is no fade (the Oreburgh Gate's, mean 46, lab-B -// 2026-11-08). Check against the Gate's frames on the console before -// tightening. -constexpr double FADE_MAX_STDDEV_SUM = 10; -// Dark goes to an rgb sum of 225, as a cave mouth's fade is dim, 45 to 64 -// a channel, not black. -constexpr double FADE_MAX_DARK_SUM = 225; -// White from an rgb sum of 500, as the shared WhiteScreenWatcher's. -constexpr double FADE_MIN_WHITE_SUM = 500; -// The fade `middle` (the stats of the screen's middle 80%) shows, if any. -WarpFade fade_of(const ImageStats& middle); -// fade_of() for a screen. -WarpFade fade_of_screen(const ImageViewRGB32& screen); - -// What a walk or a step saw of a warp (#268): a fade or the location -// banner of the map arrived on. Either one alone is the warp. -struct WarpSeen{ - WarpFade fade = WarpFade::NONE; - // The banner read, "" if none. - std::string banner = {}; - - bool any() const{ - return fade != WarpFade::NONE || !banner.empty(); - } -}; -// How it was seen, for the log: a black fade, banner "Route 203". -std::string describe(const WarpSeen& seen); +using Pokemon::warp_fade_name; +// A fade's most texture (the walk's black-screen watcher uses it too). +using Pokemon::FADE_MAX_STDDEV_SUM; +// A dark fade's brightest rgb sum (likewise). +using Pokemon::FADE_MAX_DARK_SUM; +// The fade a screen's middle shows, from its stats. +using Pokemon::fade_of; +// The fade a screen shows. +using Pokemon::fade_of_screen; +// What a walk or a step saw of a warp: a fade or a banner. +using Pokemon::WarpSeen; // Result metrics and terminal state returned by automated walking execution. struct WalkResult{ diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/PokemonFRLG_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/PokemonFRLG_Tests.cpp index d734e275f7..68ab2a274b 100644 --- a/SerialPrograms/Source/VideoGames/PokemonFRLG/PokemonFRLG_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/PokemonFRLG_Tests.cpp @@ -8,6 +8,7 @@ #include "Common/Cpp/Filesystem/Filesystem.h" #include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/StaticGlobals.h" +#include "Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.h" #include "Inference/Box/PokemonFRLG_BoxCellOccupiedDetector.h" #include "Inference/Box/PokemonFRLG_BoxCellReader.h" #include "Inference/Box/PokemonFRLG_BoxGenderReader.h" @@ -288,6 +289,7 @@ void add_tests(UnitTestDatabase& database){ add_tests_BattleMoveSelectionArrow(database); add_tests_BagDetector(database); add_tests_ExitWildBattle(database); + add_tests_AutoStoryScreenEvents(database); } diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.cpp new file mode 100644 index 0000000000..85edb74b36 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.cpp @@ -0,0 +1,52 @@ +/* FRLG AutoStory - Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "PokemonFRLG_AutoStoryScreenEvents.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonFRLG{ + +using Pokemon::ScreenEvent; +using Pokemon::ScreenEvents; + + +FrlgScreenObserver::FrlgScreenObserver() + : m_dialog(COLOR_RED) + , m_choice(COLOR_RED) + , m_battle_menu(COLOR_RED) + , m_battle_dialog(COLOR_RED) +{} + +ScreenEvents FrlgScreenObserver::observe(const VideoSnapshot& frame, WallClock){ + if (!frame){ + return {}; + } + const ImageViewRGB32& screen = *frame.frame; + ScreenEvents out = Pokemon::fade_events(Pokemon::fade_of_screen(screen)); + if (Pokemon::fade_in(out) != Pokemon::WarpFade::NONE){ + // A uniform screen has no box or menu to read. + return out; + } + const bool battle = m_battle_menu.detect(screen) || m_battle_dialog.detect(screen); + out.set(ScreenEvent::BATTLE_STARTED, battle); + out.set(ScreenEvent::DIALOG_BOX, !battle && m_dialog.detect(screen)); + out.set(ScreenEvent::CHOICE_BOX, !battle && m_choice.detect(screen)); + return out; +} + +void FrlgScreenObserver::make_overlays(VideoOverlaySet& items) const{ + m_dialog.make_overlays(items); + m_choice.make_overlays(items); + m_battle_menu.make_overlays(items); + m_battle_dialog.make_overlays(items); +} + + + +} // namespace PokemonFRLG +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.h b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.h new file mode 100644 index 0000000000..d6403a765f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.h @@ -0,0 +1,53 @@ +/* FRLG AutoStory - Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + * FRLG's detectors as the shared screen events (Pokemon_ScreenEvents.h), + * so the shared judgements (Pokemon_SceneJudges.h) read FRLG frames too. + * + */ + +#ifndef PokemonAutomation_PokemonFRLG_AutoStoryScreenEvents_H +#define PokemonAutomation_PokemonFRLG_AutoStoryScreenEvents_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Pokemon/AutoStory/Pokemon_ScreenEvents.h" +#include "VideoGames/PokemonFRLG/Inference/Dialogs/PokemonFRLG_BattleDialogs.h" +#include "VideoGames/PokemonFRLG/Inference/Dialogs/PokemonFRLG_DialogDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonFRLG{ + + +// The shared events from FRLG's detectors: +// - DIALOG_BOX: the overworld's white text box (WhiteDialogDetector); +// - CHOICE_BOX: a Yes/No or answer box beside it (SelectionDialogDetector); +// - BATTLE_STARTED: the battle menu or a battle's text box; +// - FADE_BLACK / FADE_WHITE: fade_of_screen(). +// Not reported yet, as FRLG has no detector for them here: the map-name +// popup (LOCATION_BANNER), PLAYER_CAN_MOVE, SCREEN_STILL, the interact +// prompt (FRLG has none) and a battle's end. +class FrlgScreenObserver : public Pokemon::ScreenEventObserver{ +public: + FrlgScreenObserver(); + + virtual Pokemon::ScreenEvents observe(const VideoSnapshot& frame, WallClock now) override; + virtual void make_overlays(VideoOverlaySet& items) const override; + +private: + WhiteDialogDetector m_dialog; + SelectionDialogDetector m_choice; + BattleMenuDetector m_battle_menu; + BattleDialogDetector m_battle_dialog; +}; + +// Registers the adapter's unit tests (synthetic frames). +void add_tests_AutoStoryScreenEvents(UnitTestDatabase& database); + + + +} // namespace PokemonFRLG +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents_Tests.cpp new file mode 100644 index 0000000000..7f8182bccc --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents_Tests.cpp @@ -0,0 +1,97 @@ +/* FRLG AutoStory - Screen Events Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The adapter on synthetic frames: a uniform screen is a fade and nothing + * else, and no frame shows nothing. Frames of real boxes and battles are + * the detectors' own image tests (CommandLineTests); a flat grey frame is + * not one of those (the battle text box reads it as a box). + * + */ + +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonFRLG_AutoStoryScreenEvents.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonFRLG{ + +using Pokemon::ScreenEvent; +using Pokemon::ScreenEvents; + +namespace{ + +struct ScreenEventsInput{ + std::string name; + // The frame's colour; 0 for no frame. + uint32_t color; +}; + +struct ScreenEventsExpectation{ + ScreenEvents events; +}; + +struct ScreenEventsCase{ + ScreenEventsInput input; + ScreenEventsExpectation expectation; +}; + +class Test_AutoStoryScreenEvents : public UnitTest{ +public: + explicit Test_AutoStoryScreenEvents(ScreenEventsCase test_case) + : UnitTest("PokemonFRLG::AutoStoryScreenEvents - " + test_case.input.name) + , m_case(std::move(test_case)) + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + VideoSnapshot frame; + if (m_case.input.color != 0){ + ImageRGB32 image(1920, 1080); + image.fill(m_case.input.color); + frame = VideoSnapshot(std::move(image), WallClock()); + } + FrlgScreenObserver observer; + const ScreenEvents got = observer.observe(frame, WallClock()); + if (got != m_case.expectation.events){ + return std::format("expected {}, got {}", describe(m_case.expectation.events), describe(got)); + } + return true; + } + +private: + ScreenEventsCase m_case; +}; + +} // namespace + + +const ScreenEventsCase SCREEN_EVENTS_CASES[] = { + { + .input = {"no frame shows nothing", 0}, + .expectation = {{}}, + }, + { + .input = {"a black screen is a black fade only", 0xff000000}, + .expectation = {{ScreenEvent::FADE_BLACK}}, + }, + { + .input = {"a white screen is a white fade only", 0xffffffff}, + .expectation = {{ScreenEvent::FADE_WHITE}}, + }, +}; + +void add_tests_AutoStoryScreenEvents(UnitTestDatabase& database){ + for (const auto& item : SCREEN_EVENTS_CASES){ + database.add(item); + } +} + + + +} // namespace PokemonFRLG +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index e548845993..e7c140bbdb 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1495,6 +1495,11 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Pokemon_AutoStorySelect_Tests.h Source/Pokemon/AutoStory/Pokemon_AutoStoryStats.h Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h + Source/Pokemon/AutoStory/Pokemon_SceneJudges.cpp + Source/Pokemon/AutoStory/Pokemon_SceneJudges.h + Source/Pokemon/AutoStory/Pokemon_ScreenEvents.cpp + Source/Pokemon/AutoStory/Pokemon_ScreenEvents.h + Source/Pokemon/AutoStory/Pokemon_ScreenEvents_Tests.cpp Source/Pokemon/AutoStory/Pokemon_StoryEvents.cpp Source/Pokemon/AutoStory/Pokemon_StoryEvents.h Source/Pokemon/AutoStory/Pokemon_StoryEvents_Tests.cpp @@ -1687,6 +1692,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.h Source/PokemonBDSP/PokemonBDSP_Tests.cpp Source/PokemonBDSP/PokemonBDSP_Tests.h Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.cpp @@ -1954,6 +1961,9 @@ file(GLOB LIBRARY_SOURCES Source/VideoGames/PokemonFRLG/Programs/Farming/PokemonFRLG_NuggetBridgeFarmer.h Source/VideoGames/PokemonFRLG/Programs/Farming/PokemonFRLG_PickupFarmer.cpp Source/VideoGames/PokemonFRLG/Programs/Farming/PokemonFRLG_PickupFarmer.h + Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.cpp + Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents.h + Source/VideoGames/PokemonFRLG/Programs/AutoStory/PokemonFRLG_AutoStoryScreenEvents_Tests.cpp Source/VideoGames/PokemonFRLG/Programs/PokemonFRLG_BattleMenuNavigation.cpp Source/VideoGames/PokemonFRLG/Programs/PokemonFRLG_BattleMenuNavigation.h Source/VideoGames/PokemonFRLG/Programs/PokemonFRLG_SafariOptimalAction.cpp