From 2c987a558cefa4189c8cfe8e1b871f903bd2b7e0 Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 13 Nov 2026 01:57:39 -0800 Subject: [PATCH 1/8] BDSP AutoStory rebuild: BF1 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../Inference/PokemonBDSP_MenuDetector.cpp | 98 +++--- .../PokemonBDSP_NameListDetector.cpp | 96 ++++++ .../Inference/PokemonBDSP_NameListDetector.h | 42 +++ .../PokemonBDSP_TransitionWatchers.cpp | 45 +++ .../PokemonBDSP_TransitionWatchers.h | 51 ++++ ...PokemonBDSP_AutoStory_BattleMenuReader.cpp | 182 +++++++++++ .../PokemonBDSP_AutoStory_BattleMenuReader.h | 55 ++++ ...okemonBDSP_AutoStory_IntroMenuDetector.cpp | 118 ++++++++ .../PokemonBDSP_AutoStory_IntroMenuDetector.h | 48 +++ ...kemonBDSP_AutoStory_MenuCursorDetector.cpp | 286 ++++++++++++++++++ ...PokemonBDSP_AutoStory_MenuCursorDetector.h | 152 ++++++++++ ...okemonBDSP_AutoStory_OverworldDetector.cpp | 136 +++++++++ .../PokemonBDSP_AutoStory_OverworldDetector.h | 101 +++++++ SerialPrograms/cmake/SourceFiles.cmake | 12 + 14 files changed, 1378 insertions(+), 44 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp index 6d23f8fb83..d4af9f47d1 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp @@ -28,10 +28,14 @@ constexpr double STRIPE_BOTTOM = 0.598; // Each column sits in the gap right of an icon, clear of the "NEW" tag on // its top right corner. The tag on TOWN MAP extends into a column at 0.160. constexpr double STRIPE_LEFT = 0.166; +// The selected icon glows about 0.008 past its left edge, into the gap left +// of it. A column of 0.015 reached the glow from 0.177 (cursor on the first +// icon of a row); 0.010 ends just short of it. +constexpr double STRIPE_WIDTH = 0.010; ImageFloatBox menu_stripe(size_t index, bool skip_new_banner){ double top = skip_new_banner ? STRIPE_TOP_BELOW_BANNER : STRIPE_TOP; - return ImageFloatBox(STRIPE_LEFT + 0.166 * (double)index, top, 0.015, STRIPE_BOTTOM - top); + return ImageFloatBox(STRIPE_LEFT + 0.166 * (double)index, top, STRIPE_WIDTH, STRIPE_BOTTOM - top); } } // namespace @@ -149,52 +153,58 @@ class Test_MenuDetector : public ImageUnitTest{ MenuExpectation m_expectation; }; +// Every image runs with both column modes. +constexpr std::array MENU_CASES{{ + {.input = {.image = "PokemonBDSP/MenuDetector/FrenchCursorOnMap_True.png", .columns = Columns::FromTop}, + .expectation = {.detected = true}}, + {.input = {.image = "PokemonBDSP/MenuDetector/FrenchCursorOnMap_True.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = true}}, + {.input = {.image = "PokemonBDSP/MenuDetector/NewGameBagOptions_True.png", .columns = Columns::FromTop}, + .expectation = {.detected = true}}, + {.input = {.image = "PokemonBDSP/MenuDetector/NewGameBagOptions_True.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = true}}, + {.input = {.image = "PokemonBDSP/MenuDetector/JubilifeTownMapNew_True.png", .columns = Columns::FromTop}, + .expectation = {.detected = true}}, + {.input = {.image = "PokemonBDSP/MenuDetector/JubilifeTownMapNew_True.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = true}}, + {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaMenuThreeNew_True.png", .columns = Columns::FromTop}, + .expectation = {.detected = true}}, + {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaMenuThreeNew_True.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = true}}, + {.input = {.image = "PokemonBDSP/MenuDetector/Home_False.png", .columns = Columns::FromTop}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/Home_False.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/OverworldSandgemMart_False.png", .columns = Columns::FromTop}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/OverworldSandgemMart_False.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/OverworldOreburghMine_False.png", .columns = Columns::FromTop}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/OverworldOreburghMine_False.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/IntroDialog_False.png", .columns = Columns::FromTop}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/IntroDialog_False.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/IntroBarry_False.png", .columns = Columns::FromTop}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/IntroBarry_False.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaOldManDialog_False.png", .columns = Columns::FromTop}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaOldManDialog_False.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaCloudShadows_False.png", .columns = Columns::FromTop}, + .expectation = {.detected = false}}, + {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaCloudShadows_False.png", .columns = Columns::BelowNewTags}, + .expectation = {.detected = false}}, +}}; + } // namespace void add_tests_MenuDetector(UnitTestDatabase& database){ - static constexpr std::array CASES{{ - {.input = {.image = "PokemonBDSP/MenuDetector/NewGameBagOptions_True.png", .columns = Columns::FromTop}, - .expectation = {.detected = true}}, - {.input = {.image = "PokemonBDSP/MenuDetector/NewGameBagOptions_True.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = true}}, - {.input = {.image = "PokemonBDSP/MenuDetector/JubilifeTownMapNew_True.png", .columns = Columns::FromTop}, - .expectation = {.detected = true}}, - {.input = {.image = "PokemonBDSP/MenuDetector/JubilifeTownMapNew_True.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = true}}, - {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaMenuThreeNew_True.png", .columns = Columns::FromTop}, - .expectation = {.detected = true}}, - {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaMenuThreeNew_True.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = true}}, - {.input = {.image = "PokemonBDSP/MenuDetector/Home_False.png", .columns = Columns::FromTop}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/Home_False.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/OverworldSandgemMart_False.png", .columns = Columns::FromTop}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/OverworldSandgemMart_False.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/OverworldOreburghMine_False.png", .columns = Columns::FromTop}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/OverworldOreburghMine_False.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/IntroDialog_False.png", .columns = Columns::FromTop}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/IntroDialog_False.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/IntroBarry_False.png", .columns = Columns::FromTop}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/IntroBarry_False.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaOldManDialog_False.png", .columns = Columns::FromTop}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaOldManDialog_False.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaCloudShadows_False.png", .columns = Columns::FromTop}, - .expectation = {.detected = false}}, - {.input = {.image = "PokemonBDSP/MenuDetector/FloaromaCloudShadows_False.png", .columns = Columns::BelowNewTags}, - .expectation = {.detected = false}}, - }}; - for (const MenuCase& test : CASES){ + for (const MenuCase& test : MENU_CASES){ database.add(test); } } diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp new file mode 100644 index 0000000000..0ee0a55f55 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp @@ -0,0 +1,96 @@ +/* Name List Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Kernels/Waterfill/Kernels_Waterfill.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "CommonTools/ImageMatch/ExactImageMatcher.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "PokemonBDSP_NameListDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ +// Row centers, as a fraction of the screen height (1080p: 307, 370, 433, ...). +const double FIRST_ROW_Y = 0.284; +const double ROW_STEP_Y = 0.0587; +const int ROWS = 5; + +// The menu's selection arrows in `image`, as PokemonBDSP_SelectionArrow.cpp +// finds them: that file keeps its finder out of its header, so the same +// waterfill and template match are repeated here rather than changing it. +const ImageMatch::ExactImageMatcher& name_list_arrow(){ + static ImageMatch::ExactImageMatcher matcher(RESOURCE_PATH() + "PokemonBDSP/SelectionArrow.png"); + return matcher; +} +std::vector find_name_list_arrows(const ImageViewRGB32& image){ + using namespace Kernels; + using namespace Kernels::Waterfill; + PackedBinaryMatrix matrix = compress_rgb32_to_binary_max(image, 200, 200, 200); + std::vector objects = find_objects_inplace(matrix, 200); + std::vector ret; + for (const WaterfillObject& object : objects){ + if (object.width() > object.height() || object.height() > 3 * object.width()){ + continue; + } + if (name_list_arrow().rmsd(extract_box_reference(image, object)) <= 90){ + ret.emplace_back(object.min_x, object.min_y, object.max_x, object.max_y); + } + } + return ret; +} +} // namespace + + +NameListDetector::NameListDetector(Color color) + : m_color(color) + , m_arrows(0.137, 0.245, 0.030, 0.310) + , m_panel(0.140, 0.245, 0.140, 0.300) +{} +void NameListDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_arrows); + items.add(m_color, m_panel); +} +bool NameListDetector::detect(const ImageViewRGB32& screen){ + return selected_row(screen) >= 0; +} +int NameListDetector::selected_row(const ImageViewRGB32& screen) const{ + // The list is a white panel with a little dark text. A thin margin strip is not + // enough: the border's antialiasing reaches into it on some frames. + const ImageViewRGB32 panel = extract_box_reference(screen, m_panel); + size_t white = 0; + for (size_t y = 0; y < panel.height(); y += 2){ + for (size_t x = 0; x < panel.width(); x += 2){ + const uint32_t p = panel.pixel(x, y); + white += ((p >> 16) & 0xff) > 220 && ((p >> 8) & 0xff) > 220 && (p & 0xff) > 220; + } + } + const size_t samples = ((panel.width() + 1) / 2) * ((panel.height() + 1) / 2); + if (samples == 0 || white < samples * 6 / 10){ + return -1; + } + ImageViewRGB32 area = extract_box_reference(screen, m_arrows); + std::vector arrows = find_name_list_arrows(area); + if (arrows.size() != 1){ + return -1; + } + const double center_y = m_arrows.y + + m_arrows.height * (arrows[0].min_y + arrows[0].max_y) / 2.0 / (double)area.height(); + const int row = (int)std::lround((center_y - FIRST_ROW_Y) / ROW_STEP_Y); + return row >= 0 && row < ROWS ? row : -1; +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h new file mode 100644 index 0000000000..9e40fb77d0 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h @@ -0,0 +1,42 @@ +/* Name List Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The list of preset names shown for the rival: "I'll tell you...", then four names. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_NameListDetector_H +#define PokemonAutomation_PokemonBDSP_NameListDetector_H + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Detects the rival preset name selection menu and reads the selected row. +class NameListDetector : public StaticScreenDetector{ +public: + NameListDetector(Color color = COLOR_MAGENTA); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + + // The row the arrow is on: 0 is "I'll tell you...", 1 the first preset name. + // -1 if the list is not showing. + int selected_row(const ImageViewRGB32& screen) const; + +private: + Color m_color; + ImageFloatBox m_arrows; + ImageFloatBox m_panel; +}; + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.cpp new file mode 100644 index 0000000000..d5e37c567f --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.cpp @@ -0,0 +1,45 @@ +/* BDSP Transition Watchers + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + */ + +#include "PokemonBDSP_TransitionWatchers.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +WhiteScreenOverWatcher::WhiteScreenOverWatcher(Color color, const ImageFloatBox& box) + : VisualInferenceCallback("WhiteScreenOverWatcher") + , m_on(color, box, WhiteScreenWatcher::FinderType::PRESENT, std::chrono::milliseconds(100)) + , m_off(color, box, WhiteScreenWatcher::FinderType::GONE, std::chrono::milliseconds(100)) +{} +void WhiteScreenOverWatcher::make_overlays(VideoOverlaySet& items) const{ + m_on.make_overlays(items); +} +bool WhiteScreenOverWatcher::process_frame(const ImageViewRGB32& frame, WallClock timestamp){ + if (m_white_is_over.load(std::memory_order_acquire)){ + return true; + } + if (!m_has_been_white){ + m_has_been_white = m_on.process_frame(frame, timestamp); + return false; + } + bool is_over = m_off.process_frame(frame, timestamp); + if (!is_over){ + return false; + } + m_white_is_over.store(true, std::memory_order_release); + return true; +} +bool WhiteScreenOverWatcher::white_is_over(const ImageViewRGB32& frame){ + return m_white_is_over.load(std::memory_order_acquire); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.h b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.h new file mode 100644 index 0000000000..2dd7e59d7c --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.h @@ -0,0 +1,51 @@ +/* BDSP Transition Watchers + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + */ + +#ifndef PokemonAutomation_PokemonBDSP_TransitionWatchers_H +#define PokemonAutomation_PokemonBDSP_TransitionWatchers_H + +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// AutoStory-specific transition handling; shared watchers retain upstream behavior. +// A white screen (RGB sum over 500, stddev sum under 10) in `box`, +// present or gone for `duration`. +class WhiteScreenWatcher : public DetectorToFinder{ +public: + WhiteScreenWatcher(Color color, const ImageFloatBox& box, FinderType finder_type, std::chrono::milliseconds duration) + : DetectorToFinder("WhiteScreenWatcher", finder_type, duration, color, box, 500, 10) + {} +}; + +// The screen in `box` went white for 100 ms, then stopped being white for +// 100 ms: an encounter's flash. +class WhiteScreenOverWatcher : public VisualInferenceCallback{ +public: + WhiteScreenOverWatcher(Color color, const ImageFloatBox& box); + + bool white_is_over(const ImageViewRGB32& frame); + + virtual void make_overlays(VideoOverlaySet& items) const override; + + virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override; + +private: + WhiteScreenWatcher m_on; + WhiteScreenWatcher m_off; + bool m_has_been_white = false; + std::atomic m_white_is_over = false; +}; + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp new file mode 100644 index 0000000000..d5f91e6914 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp @@ -0,0 +1,182 @@ +/* BDSP AutoStory - Battle Menu Reader + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonBDSP_AutoStory_BattleMenuReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// Whether at least 15% of the box's pixels are white (min channel over +// `low`, spread under `spread`): a cursor bracket's corner. +bool white_corner(const ImageViewRGB32& screen, const ImageFloatBox& box, unsigned low_min, unsigned spread_max){ + const ImageViewRGB32 patch = extract_box_reference(screen, box); + size_t white = 0; + const size_t total = patch.width() * patch.height(); + for (size_t y = 0; y < patch.height(); y++){ + for (size_t x = 0; x < patch.width(); x++){ + const uint32_t pixel = patch.pixel(x, y); + const unsigned r = (pixel >> 16) & 255; + const unsigned g = (pixel >> 8) & 255; + const unsigned b = pixel & 255; + const unsigned low = std::min(r, std::min(g, b)); + const unsigned high = std::max(r, std::max(g, b)); + white += low > low_min && high - low < spread_max; + } + } + return total != 0 && static_cast(white) / total >= 0.15; +} + +// Whether the command menu's row `row` has the cursor's corner brackets. +// They wiggle; all four must agree on one small offset. +bool row_has_brackets(const ImageViewRGB32& screen, int row){ + const double offset = row * 0.1075; + const double offsets[] = {0.0, -0.003, 0.003, -0.006, 0.006}; + for (double dy : offsets){ + for (double dx : offsets){ + const ImageFloatBox corners[] = { + {0.793 + dx, 0.562 + offset + dy, 0.018, 0.023}, + {0.976 + dx, 0.562 + offset + dy, 0.018, 0.023}, + {0.793 + dx, 0.652 + offset + dy, 0.018, 0.023}, + {0.976 + dx, 0.652 + offset + dy, 0.018, 0.023}, + }; + if (std::all_of(std::begin(corners), std::end(corners), [&](const ImageFloatBox& box){ + return white_corner(screen, box, 210, 35); + })){ + return true; + } + } + } + return false; +} + +// The white PP panels in `patch` (the search box), top to bottom: nothing +// for no panel, more than four, or geometry that is not a move list. +std::vector find_pp_panels(const ImageViewRGB32& screen, const ImageViewRGB32& patch){ + cv::Mat white((int)patch.height(), (int)patch.width(), CV_8UC1); + for (size_t y = 0; y < patch.height(); y++){ + for (size_t x = 0; x < patch.width(); x++){ + uint32_t p = patch.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + white.at((int)y, (int)x) = std::min(r, std::min(g, b)) > 225 && + std::max(r, std::max(g, b)) - std::min(r, std::min(g, b)) < 30 ? 255 : 0; + } + } + cv::Mat labels, stats, centers; + int components = cv::connectedComponentsWithStats(white, labels, stats, centers); + std::vector panels; + for (int component = 1; component < components; component++){ + cv::Rect box(stats.at(component, cv::CC_STAT_LEFT), stats.at(component, cv::CC_STAT_TOP), + stats.at(component, cv::CC_STAT_WIDTH), stats.at(component, cv::CC_STAT_HEIGHT)); + const double area = (double)stats.at(component, cv::CC_STAT_AREA); + if (box.width < patch.width() * 0.60 || box.height < screen.height() * 0.055 || + box.height > screen.height() * 0.13 || area / box.area() < 0.55){ + continue; + } + panels.push_back(box); + } + if (panels.empty() || panels.size() > 4){ + return {}; + } + std::sort(panels.begin(), panels.end(), [](const cv::Rect& a, const cv::Rect& b){ return a.y < b.y; }); + // Reject inconsistent/overlapping geometry rather than count noise as moves. + for (size_t i = 1; i < panels.size(); i++){ + if (panels[i].y <= panels[i - 1].y + panels[i - 1].height || + std::abs(panels[i].height - panels[0].height) > screen.height() * 0.025){ + return {}; + } + } + return panels; +} + +// Whether the move button from `top` to `bottom` (of the screen) has the +// cursor's four corners, at a small offset. +bool move_button_selected(const ImageViewRGB32& screen, double top, double bottom){ + auto is_corner = [&](double x, double y){ + if (x < 0 || y < 0 || x + 0.018 > 1 || y + 0.023 > 1){ + return false; + } + return white_corner(screen, ImageFloatBox{x, y, 0.018, 0.023}, 210, 35); + }; + for (double dy : {0.0, -0.003, 0.003, -0.006, 0.006, -0.010, 0.010}){ + for (double dx : {0.0, -0.003, 0.003}){ + if (is_corner(0.663 + dx, top - 0.010 + dy) && + is_corner(0.976 + dx, top - 0.010 + dy) && + is_corner(0.663 + dx, bottom - 0.014 + dy) && + is_corner(0.976 + dx, bottom - 0.014 + dy)){ + return true; + } + } + } + return false; +} + +} // namespace + + +BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen){ + if (!screen || !menu.detect(screen)){ + return BattleMenuOption::UNKNOWN; + } + int selected = -1; + for (int row = 0; row < 4; row++){ + if (!row_has_brackets(screen, row)){ + continue; + } + if (selected >= 0){ + return BattleMenuOption::UNKNOWN; + } + selected = row; + } + return static_cast(selected); +} + +BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen){ + BattleMoveMenuState result; + if (!screen){ + return result; + } + // Every populated move has a white PP panel, irrespective of move type or + // selected row. Find connected panels rather than assume a 4-row layout. + const ImageFloatBox search{0.90, 0.48, 0.075, 0.505}; + const std::vector panels = find_pp_panels(screen, extract_box_reference(screen, search)); + if (panels.empty()){ + return result; + } + result.move_count = panels.size(); + for (size_t slot = 0; slot < panels.size(); slot++){ + double top = search.y + (double)panels[slot].y / screen.height(); + double bottom = top + (double)panels[slot].height / screen.height(); + result.buttons[slot] = ImageFloatBox{0.665, std::max(0.0, top - 0.01), 0.333, + std::min(1.0, bottom + 0.01) - std::max(0.0, top - 0.01)}; + if (!move_button_selected(screen, top, bottom)){ + continue; + } + if (result.selected_slot){ + result.selected_slot = 0; + return result; + } + result.selected_slot = slot + 1; + } + return result; +} + + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h new file mode 100644 index 0000000000..1e053c4953 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h @@ -0,0 +1,55 @@ +/* BDSP AutoStory - Battle Menu Reader + * + * From: https://github.com/PokemonAutomation/ + * + * Which battle command and which move the cursor is on. Adapted from + * PokemonAutomation/Arduino-Source#1717 by Developer-Butters, which added + * them to PokemonBDSP_BattleMenuDetector; here they sit beside it, so the + * upstream detector stays as it is. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_BattleMenuReader_H +#define PokemonAutomation_PokemonBDSP_AutoStory_BattleMenuReader_H + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Vertical command order in the BDSP battle menu. +enum class BattleMenuOption{ + UNKNOWN = -1, + BATTLE = 0, + POKEMON = 1, + BAG = 2, + RUN = 3, +}; + +// The command under the cursor: `menu` must see its battle menu, and white +// selection brackets must sit at all four corners of exactly one command. +// Missing or ambiguous brackets read UNKNOWN; the cursor is never guessed. +BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen); + +// Logical move slots are ordered top to bottom among the populated move +// buttons. `selected_slot` is 1 to 4, or 0 when missing or ambiguous. +struct BattleMoveMenuState{ + size_t move_count = 0; + size_t selected_slot = 0; + std::array buttons; +}; +// Detects the populated move slots and the currently selected slot from the battle move menu. +BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp new file mode 100644 index 0000000000..14be3abf8a --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp @@ -0,0 +1,118 @@ +/* BDSP AutoStory - Intro Menu Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonBDSP_AutoStory_IntroMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// Cards are flat: the frames' strips have stddev sums of 2 to 18. +const double MAX_STDDEV_SUM = 25; + +enum class CardLook{ GREY, BLUE, OTHER }; + +// Grey on the frames: (225, 233, 238) language buttons, (210, 218, 223) +// portraits. Blue: (167, 206, 255) and (152, 195, 251). The background +// between the cards is a bluer (203, 232, 253), neither. +CardLook strip_look(const ImageViewRGB32& screen, const ImageFloatBox& strip){ + const ImageStats stats = image_stats(extract_box_reference(screen, strip)); + if (stats.stddev.sum() > MAX_STDDEV_SUM){ + return CardLook::OTHER; + } + const FloatPixel& c = stats.average; + const double low = std::min({c.r, c.g, c.b}); + const double high = std::max({c.r, c.g, c.b}); + if (low >= 195 && high - low <= 25){ + return CardLook::GREY; + } + if (c.b >= 230 && c.r >= 120 && c.b - c.r >= 50){ + return CardLook::BLUE; + } + return CardLook::OTHER; +} + +// A card's look: all its strips must agree. +CardLook card_look(const ImageViewRGB32& screen, const std::vector& strips){ + const CardLook first = strip_look(screen, strips[0]); + for (size_t c = 1; c < strips.size(); c++){ + if (strip_look(screen, strips[c]) != first){ + return CardLook::OTHER; + } + } + return first; +} + +// The one blue card among grey ones; nullopt otherwise. +std::optional grid_cursor(const ImageViewRGB32& screen, const std::vector>& cards){ + std::optional cursor; + for (size_t card = 0; card < cards.size(); card++){ + const CardLook look = card_look(screen, cards[card]); + if (look == CardLook::OTHER || (look == CardLook::BLUE && cursor)){ + return std::nullopt; + } + if (look == CardLook::BLUE){ + cursor = card; + } + } + return cursor; +} + +// A strip inside each button, left of its text (1080p: x 330-460 on the left +// column, 1040-1170 on the right; 60 px tall around each row's centre). +const std::vector>& language_buttons(){ + static const std::vector> cards = []{ + const double column_x[2] = {0.172, 0.542}; + const double row_center_y[5] = {0.154, 0.280, 0.404, 0.528, 0.652}; + std::vector> ret; + for (size_t item = 0; item < LANGUAGE_SELECT_BUTTONS; item++){ + ret.push_back({ImageFloatBox(column_x[item % 2], row_center_y[item / 2] - 0.028, 0.068, 0.056)}); + } + return ret; + }(); + return cards; +} + +// Two strips of background beside each face (1080p: portraits from x 168, +// 576, 984, 1392 and y 124, 456; the strips 12-44 and 320-356 px in, 80-180 +// px down). +const std::vector>& look_portraits(){ + static const std::vector> cards = []{ + const double column_x[4] = {0.0875, 0.3000, 0.5125, 0.7250}; + const double row_y[2] = {0.1148, 0.4222}; + std::vector> ret; + for (size_t item = 0; item < LOOK_SELECT_PORTRAITS; item++){ + const double x = column_x[item % 4]; + const double y = row_y[item / 4] + 0.0741; + ret.push_back({ImageFloatBox(x + 0.0063, y, 0.0167, 0.0926), ImageFloatBox(x + 0.1667, y, 0.0188, 0.0926)}); + } + return ret; + }(); + return cards; +} + +} // namespace + + +std::optional language_select_cursor(const ImageViewRGB32& screen){ + return grid_cursor(screen, language_buttons()); +} + +std::optional look_select_cursor(const ImageViewRGB32& screen){ + return grid_cursor(screen, look_portraits()); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h new file mode 100644 index 0000000000..f413659405 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h @@ -0,0 +1,48 @@ +/* BDSP AutoStory - Intro Menu Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The two menus of a new game's intro, each a grid of flat cards over a + * crystal background, the cursor's card blue and the others light grey + * (lab-B 2026-11-09, 1080p): + * - the language select: ten buttons in two columns of five, "Select the + * language you would like to play in." below; + * - the look select: two rows of four portraits, "Choose your photo." + * below. Top row: Lucas light, Lucas dark, Dawn dark-haired, Dawn dark; + * bottom row: Lucas blond, Lucas brown, Dawn blond, Dawn brown-haired. + * Cards are numbered row by row, as MenuGrid reads them. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_IntroMenuDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_IntroMenuDetector_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The language select's buttons: English 0, its right neighbour 1, the next +// row 2 and 3, and so on. +inline constexpr size_t LANGUAGE_SELECT_BUTTONS = 10; +// The look select's portraits: the top row 0 to 3, the bottom row 4 to 7. +inline constexpr size_t LOOK_SELECT_PORTRAITS = 8; + +// The button the cursor is on, or nullopt when `screen` is not the language +// select: every button must read as plain grey but exactly one, blue. +std::optional language_select_cursor(const ImageViewRGB32& screen); + +// The portrait the cursor is on, or nullopt when `screen` is not the look +// select: the background of every portrait must read as plain grey but +// exactly one, blue. +std::optional look_select_cursor(const ImageViewRGB32& screen); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp new file mode 100644 index 0000000000..48bb59b8d5 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp @@ -0,0 +1,286 @@ +/* BDSP AutoStory - Menu Cursor Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include + +#include "CommonFramework/VideoPipeline/VideoFeed.h" + +#include "CommonTools/Images/ImageFilter.h" + +#include "PokemonBDSP_AutoStory_MenuCursorDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +std::string menu_cursor_to_string(MenuCursorPosition pos){ + switch (pos){ + case MenuCursorPosition::POKEDEX: + return "POKEDEX"; + case MenuCursorPosition::POKEMON: + return "POKEMON"; + case MenuCursorPosition::BAG: + return "BAG"; + case MenuCursorPosition::CARD: + return "CARD"; + case MenuCursorPosition::OPTIONS: + return "OPTIONS"; + case MenuCursorPosition::MAP: + return "MAP"; + case MenuCursorPosition::CAPSULES: + return "CAPSULES"; + case MenuCursorPosition::MYSTERYGIFT: + return "MYSTERYGIFT"; + default: + return "UNKNOWN"; + } +} + + +MenuCursorDetector::MenuCursorDetector(Color color) : + m_color(color) + // Top row, left to right. + , m_pokedex_box{ + ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), + }, + m_pokemon_box{ + ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), + }, + m_bag_box{ + ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), + }, + m_card_box{ + ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), + } + // Bottom row — left to right + // 5-icon layout: OPTIONS is the only bottom icon (far-left position). + // 8-icon layout: MAP, CAPSULES, OPTIONS, MYSTERYGIFT from left to right. + // OPTIONS sits at a different x-position in each layout — use a separate + // MenuCursorDetector instance if you need to detect OPTIONS in both. + , m_options_box{ + ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), + }, + m_map_box{ + ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), + }, + m_capsules_box{ + ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), + }, + m_mysterygift_box{ + ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), + } + // Approach 2 search boxes — one per icon, same area as the dark-pixel box. + // The template PNG is slid across this area at its native size. + , m_icon_match_box{ + ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), // 0 POKEDEX + ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), // 1 POKEMON + ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), // 2 BAG + ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), // 3 CARD + ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), // 4 OPTIONS (8-icon layout) + ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), // 5 MAP + ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), // 6 CAPSULES + ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), // 7 MYSTERYGIFT + } { +} + + +void MenuCursorDetector::make_overlays(VideoOverlaySet& items) const { + for (const ImageFloatBox* icon_boxes : { + m_pokedex_box, + m_pokemon_box, + m_bag_box, + m_card_box, + m_options_box, + m_map_box, + m_capsules_box, + m_mysterygift_box, + }){ + for (int i = 0; i < BOXES_PER_ICON; i++){ + if (icon_boxes[i].width > 0.0 && icon_boxes[i].height > 0.0){ + items.add(m_color, icon_boxes[i]); + } + } + } + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + if (m_icon_match_box[i].width > 0.0 && m_icon_match_box[i].height > 0.0){ + items.add(m_color, m_icon_match_box[i]); + } + } +} + + +namespace{ + +// Count how many of `n` boxes contain >= MIN_DARK_PIXEL_COUNT pixels with +// R, G, B all <= BLACK_CHANNEL_MAX. Boxes with zero width are skipped. +int count_dark_boxes(const ImageViewRGB32& screen, const ImageFloatBox* boxes, int n){ + const uint32_t ch = uint32_t(MenuCursorDetector::BLACK_CHANNEL_MAX); + const uint32_t maxs = 0xff000000u | (ch << 16) | (ch << 8) | ch; + + int count = 0; + for (int i = 0; i < n; i++){ + if (boxes[i].width == 0.0){ + continue; + } + size_t dark_pixels = 0; + filter_rgb32_range(dark_pixels, + extract_box_reference(screen, boxes[i]), + 0xff000000u, + maxs, + Color(0xffffffff), + false); + if (dark_pixels >= MenuCursorDetector::MIN_DARK_PIXEL_COUNT){ + count++; + } + } + return count; +} + +// Pixel-wise RMSD between two same-size images over RGB channels only. +double compute_rmsd(const ImageViewRGB32& a, const ImageViewRGB32& b){ + size_t n = a.width() * a.height(); + if (n == 0){ + return 0.0; + } + double sum_sq = 0.0; + for (size_t y = 0; y < a.height(); y++){ + for (size_t x = 0; x < a.width(); x++){ + uint32_t pa = a.pixel(x, y); + uint32_t pb = b.pixel(x, y); + int dr = int((pa >> 16) & 0xff) - int((pb >> 16) & 0xff); + int dg = int((pa >> 8) & 0xff) - int((pb >> 8) & 0xff); + int db = int(pa & 0xff) - int(pb & 0xff); + sum_sq += double(dr * dr + dg * dg + db * db); + } + } + return std::sqrt(sum_sq / (3.0 * n)); +} + +// Slide `tmpl` (at its native pixel size) across the region of `screen` +// defined by `box`. Returns the minimum RMSD found over all window positions. +double template_match_rmsd(const ImageViewRGB32& screen, const ImageFloatBox& box, const ImageViewRGB32& tmpl){ + ImageViewRGB32 region = extract_box_reference(screen, box); + size_t rw = tmpl.width(), rh = tmpl.height(); + size_t lw = region.width(), lh = region.height(); + if (rw == 0 || rh == 0 || rw > lw || rh > lh){ + return 0.0; + } + + double min_rmsd = std::numeric_limits::max(); + for (size_t dy = 0; dy <= lh - rh; dy++){ + for (size_t dx = 0; dx <= lw - rw; dx++){ + double r = compute_rmsd(region.sub_image(dx, dy, rw, rh), tmpl); + if (r < min_rmsd){ + min_rmsd = r; + } + } + } + return min_rmsd; +} + +} // namespace + + +MenuCursorPosition MenuCursorDetector::detect_dark_pixels(const ImageViewRGB32& screen) const { + struct IconEntry { + MenuCursorPosition position; + const ImageFloatBox* boxes; + }; + const IconEntry icons[] = { + {MenuCursorPosition::POKEDEX, m_pokedex_box}, + {MenuCursorPosition::POKEMON, m_pokemon_box}, + {MenuCursorPosition::BAG, m_bag_box}, + {MenuCursorPosition::CARD, m_card_box}, + {MenuCursorPosition::OPTIONS, m_options_box}, + {MenuCursorPosition::MAP, m_map_box}, + {MenuCursorPosition::CAPSULES, m_capsules_box}, + {MenuCursorPosition::MYSTERYGIFT, m_mysterygift_box}, + }; + + MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; + int best_count = 0; + + for (const auto& entry : icons){ + int count = count_dark_boxes(screen, entry.boxes, BOXES_PER_ICON); + if (count > best_count){ + best_count = count; + best_pos = entry.position; + } + } + + return (best_count >= REQUIRED_VOTES) ? best_pos : MenuCursorPosition::UNKNOWN; +} + + +MenuCursorPosition MenuCursorDetector::detect_image_match(const ImageViewRGB32& screen) const { + MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; + double best_rmsd = MATCH_RMSD_THRESHOLD; // must exceed this + + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + if (!m_icon_references[i].has_value()){ + continue; + } + if (m_icon_match_box[i].width == 0.0){ + continue; + } + double r = template_match_rmsd(screen, m_icon_match_box[i], *m_icon_references[i]); + if (r > best_rmsd){ + best_rmsd = r; + best_pos = static_cast(i); + } + } + return best_pos; +} + + +MenuCursorPosition MenuCursorDetector::detect(const ImageViewRGB32& screen) const { + MenuCursorPosition p1 = detect_dark_pixels(screen); + MenuCursorPosition p2 = detect_image_match(screen); + + if (p1 == p2){ + return p1; // both agree (or both UNKNOWN) + } + if (p1 == MenuCursorPosition::UNKNOWN){ + return p2; // only approach 2 fired + } + if (p2 == MenuCursorPosition::UNKNOWN){ + return p1; // only approach 1 fired + } + return p1; // conflict: prefer approach 1 +} + + +void MenuCursorDetector::set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference){ + m_icon_references[static_cast(icon)].emplace(std::move(reference)); +} + + +double MenuCursorDetector::icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const { + int idx = static_cast(icon); + if (!m_icon_references[idx].has_value() || m_icon_match_box[idx].width == 0.0){ + return -1.0; + } + return template_match_rmsd(screen, m_icon_match_box[idx], *m_icon_references[idx]); +} + + +MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& /*context*/){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + return MenuCursorPosition::UNKNOWN; + } + MenuCursorDetector detector; + return detector.detect(screen); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h new file mode 100644 index 0000000000..cadbf1e4e4 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h @@ -0,0 +1,152 @@ +/* BDSP AutoStory - Menu Cursor Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Detects which icon the cursor is hovering over in the X-menu. + * + * Two independent detection approaches are provided for testing: + * + * Approach 1 — dark pixel count (detect_dark_pixels): + * Each icon has BOXES_PER_ICON small boxes placed over a region that gains + * dark pixels when hovered. A box fires if >= MIN_DARK_PIXEL_COUNT pixels + * in it have R, G, B all <= BLACK_CHANNEL_MAX. An icon is selected when its + * fire count reaches REQUIRED_VOTES. + * Tune BLACK_CHANNEL_MAX upward if real dark pixels are being missed. + * Tune it downward (or raise MIN_DARK_PIXEL_COUNT) to reduce false positives. + * + * Approach 2 — template matching (detect_image_match): + * Each icon has one larger search box and a hovered-state reference PNG loaded + * at runtime via set_icon_reference(). The reference PNG is slid (at its native + * pixel size, without resizing) across the search box and the minimum RMSD over + * all window positions is computed. The icon with the lowest min-RMSD that is + * also <= MATCH_RMSD_THRESHOLD is considered selected. + * Tune MATCH_RMSD_THRESHOLD downward to require a closer match. + * + * detect() is the production method and currently delegates to approach 1. + * It will become a combined vote once both approaches are validated. + * + * Menu layouts: + * 5-icon (early game): POKEDEX POKEMON BAG CARD / OPTIONS + * 8-icon (late game): POKEDEX POKEMON BAG CARD / MAP CAPSULES OPTIONS MYSTERYGIFT + * + * Note: OPTIONS sits at a different screen position in each layout. + * Use separate MenuCursorDetector instances with different box coordinates + * if you need to detect OPTIONS reliably in both layouts. + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H + +#include +#include +#include + +#include "Common/Cpp/Color.h" + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" + +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Possible icon positions for the BDSP pause/X menu cursor. +enum class MenuCursorPosition { + UNKNOWN = -1, + POKEDEX = 0, + POKEMON = 1, + BAG = 2, + CARD = 3, + OPTIONS = 4, + MAP = 5, + CAPSULES = 6, + MYSTERYGIFT = 7, +}; + +// Total number of icons in the pause menu layout. +static constexpr int MENU_CURSOR_NUM_ICONS = 8; + +// Returns the debug name of a MenuCursorPosition. +std::string menu_cursor_to_string(MenuCursorPosition pos); + + +// Detects which icon in the BDSP pause menu is currently focused by the cursor. +class MenuCursorDetector { +public: + // ── Approach 1: dark pixel detection ───────────────────────────────────── + static constexpr int BOXES_PER_ICON = 3; + static constexpr int REQUIRED_VOTES = 1; // out of BOXES_PER_ICON + static constexpr uint8_t BLACK_CHANNEL_MAX = 50; // R,G,B <= this → dark pixel + static constexpr size_t MIN_DARK_PIXEL_COUNT = 1; // dark pixels per box to fire + + // ── Approach 2: template match ─────────────────────────────────────────── + // Reference PNGs are the UN-hovered state of each icon. When hovered, the + // icon gains dark pixels and diverges from the reference, so RMSD increases. + // detect_image_match() selects the icon with the HIGHEST RMSD >= threshold. + // Tune upward to reduce false positives; downward if the correct icon is missed. + static constexpr double MATCH_RMSD_THRESHOLD = 53.0; + + MenuCursorDetector(Color color = COLOR_RED); + + void make_overlays(VideoOverlaySet& items) const; + + // Approach 1: votes across dark-pixel boxes, returns icon with highest + // vote count >= REQUIRED_VOTES, or UNKNOWN. + MenuCursorPosition detect_dark_pixels(const ImageViewRGB32& screen) const; + + // Approach 2: returns the icon with the highest RMSD against its reference, + // provided that RMSD >= MATCH_RMSD_THRESHOLD (high RMSD = hovered). + // Returns UNKNOWN if no references are loaded or nothing exceeds threshold. + MenuCursorPosition detect_image_match(const ImageViewRGB32& screen) const; + + // Production method — currently delegates to detect_dark_pixels(). + MenuCursorPosition detect(const ImageViewRGB32& screen) const; + + // Load a hovered-state reference image for one icon (enables approach 2 for + // that icon). The match box for that icon must already be set in the + // constructor (m_icon_match_box); this method just attaches the reference. + void set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference); + + // Returns the raw RMSD of `icon` against its reference on the given frame. + // Returns -1.0 if no reference is loaded for that icon. + double icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const; + +private: + Color m_color; + + // ── Approach 1 boxes ───────────────────────────────────────────────────── + // 3 boxes per icon targeting regions that gain dark pixels when hovered. + // All initialized to (0,0,0,0) — replace with real normalized coordinates. + ImageFloatBox m_pokedex_box[BOXES_PER_ICON]; + ImageFloatBox m_pokemon_box[BOXES_PER_ICON]; + ImageFloatBox m_bag_box[BOXES_PER_ICON]; + ImageFloatBox m_card_box[BOXES_PER_ICON]; + ImageFloatBox m_options_box[BOXES_PER_ICON]; + ImageFloatBox m_map_box[BOXES_PER_ICON]; + ImageFloatBox m_capsules_box[BOXES_PER_ICON]; + ImageFloatBox m_mysterygift_box[BOXES_PER_ICON]; + + // ── Approach 2 search boxes + references ──────────────────────────────── + // One search box per icon (the area to slide the template across). + // References are populated via set_icon_reference() at runtime. + ImageFloatBox m_icon_match_box[MENU_CURSOR_NUM_ICONS]; + std::array, MENU_CURSOR_NUM_ICONS> m_icon_references; +}; + + +// Captures the current frame from `stream` and runs detect() on it. +MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& context); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp new file mode 100644 index 0000000000..effcc50af0 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp @@ -0,0 +1,136 @@ +/* BDSP AutoStory - Overworld Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + */ + +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" + +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/Images/SolidColorTest.h" + +#include "PokemonBDSP_AutoStory_OverworldDetector.h" + +using namespace std::chrono_literals; + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// --------------------------------------------------------------------------- +// OverworldDetector +// --------------------------------------------------------------------------- + +OverworldDetector::OverworldDetector(Color color) + : m_color(color) + , m_battle_menu(BattleType::STANDARD, color){ +} + +void OverworldDetector::make_overlays(VideoOverlaySet& items) const { + m_dialog.make_overlays(items); + m_battle_menu.make_overlays(items); + m_menu.make_overlays(items); +} + +bool OverworldDetector::detect(const ImageViewRGB32& screen){ + // We are in the overworld when none of the overlay screens are present. + if (m_dialog.detect(screen)){ + return false; + } + if (m_battle_menu.detect(screen)){ + return false; + } + if (m_menu.detect(screen)){ + return false; + } + return true; +} + + +// --------------------------------------------------------------------------- +// OverworldWatcher +// --------------------------------------------------------------------------- + +OverworldWatcher::OverworldWatcher(Color color) + : OverworldDetector(color) + , VisualInferenceCallback("OverworldWatcher"){ +} + +void OverworldWatcher::make_overlays(VideoOverlaySet& items) const { + OverworldDetector::make_overlays(items); +} + +bool OverworldWatcher::process_frame(const ImageViewRGB32& frame, WallClock /*timestamp*/){ + return detect(frame); +} + + +// --------------------------------------------------------------------------- +// PoketchButtonDetector +// --------------------------------------------------------------------------- + +PoketchButtonDetector::PoketchButtonDetector(Color color) + : m_color(color) + , m_button(0.968, 0.170, 0.022, 0.120){ +} + +void PoketchButtonDetector::make_overlays(VideoOverlaySet& items) const { + items.add(m_color, m_button); +} + +bool PoketchButtonDetector::detect(const ImageViewRGB32& screen){ + ImageStats stats = image_stats(extract_box_reference(screen, m_button)); + // The button is a glossy red with a highlight, so allow a wide stddev. + return is_solid(stats, {0.74, 0.10, 0.16}, 0.12, 80); +} + +PoketchButtonWatcher::PoketchButtonWatcher(Color color) + : PoketchButtonDetector(color) + , VisualInferenceCallback("PoketchButtonWatcher"){ +} + +void PoketchButtonWatcher::make_overlays(VideoOverlaySet& items) const { + PoketchButtonDetector::make_overlays(items); +} + +bool PoketchButtonWatcher::process_frame(const ImageViewRGB32& frame, WallClock){ + return detect(frame); +} + + +static bool look_for_poketch_button(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + PoketchButtonWatcher watcher; + return wait_until(stream, context, 2s, {{watcher}}) == 0; +} + +bool ensure_poketch_visible(VideoStream& stream, ProControllerContext& context){ + for (size_t attempt = 0; attempt < 3; attempt++){ + if (look_for_poketch_button(stream, context)){ + return true; + } + stream.log("[AutoStory] Poketch button not found. Attempt " + std::to_string(attempt + 1) + " of 3.", + COLOR_ORANGE); + + if (attempt == 0){ + // Most likely hidden: one tap shows it. + pbf_press_button(context, BUTTON_R, 205ms, 0ms); + }else{ + // Long press hides it, then a tap shows it at the normal size. + pbf_press_button(context, BUTTON_R, 1548ms, 0ms); + pbf_wait(context, 1418ms); + pbf_press_button(context, BUTTON_R, 143ms, 0ms); + } + } + + // Check the result of the last press sequence. + return look_for_poketch_button(stream, context); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h new file mode 100644 index 0000000000..f2717765b6 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h @@ -0,0 +1,101 @@ +/* BDSP AutoStory - Overworld Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Detects that the player is in the free-roaming overworld state: + * no dialog box, no battle menu, and no in-game menu open. + * + * BDSP does not ship a standalone OverworldDetector in PokemonBDSP/Inference/, + * so this composite detector is defined here for AutoStory use. + * + * Reuses: + * - ShortDialogDetector (PokemonBDSP_DialogDetector.h) — absence required + * - BattleMenuDetector (PokemonBDSP_BattleMenuDetector.h) — absence required + * - MenuDetector (PokemonBDSP_MenuDetector.h) — absence required + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_OverworldDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_OverworldDetector_H + +#include "Common/Cpp/Color.h" + +#include "CommonFramework/Tools/VideoStream.h" + +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" +#include "CommonTools/VisualDetector.h" + +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" + +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Static (single-frame) version — use when you already have a frame. +class OverworldDetector : public StaticScreenDetector { +public: + OverworldDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + + // Returns true when none of: short dialog, battle menu, or in-game menu + // are present on the given frame. + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ShortDialogDetector m_dialog; + BattleMenuDetector m_battle_menu; + MenuDetector m_menu; +}; + + +// Watcher (callback) version — use with run_until<> or wait_until(). +class OverworldWatcher : public OverworldDetector, public VisualInferenceCallback { +public: + OverworldWatcher(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override; +}; + + +// Detects the big red button on the right edge of the Poketch overlay. +// Present only while the Poketch is shown at its normal size. +class PoketchButtonDetector : public StaticScreenDetector { +public: + PoketchButtonDetector(Color color = COLOR_RED); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_button; +}; + +// Inference watcher tracking the visibility of the Poketch button on screen. +class PoketchButtonWatcher : public PoketchButtonDetector, public VisualInferenceCallback { +public: + PoketchButtonWatcher(Color color = COLOR_RED); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override; +}; + +// Tries up to 3 times to get the Poketch button on screen using R presses. +// Returns true if the button was seen. +bool ensure_poketch_visible(VideoStream& stream, ProControllerContext& context); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 5c38ef8678..8a1cac9439 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1762,6 +1762,16 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/Resources/Pokemon_PokemonNames.h Source/Pokemon/Resources/Pokemon_PokemonSlugs.cpp Source/Pokemon/Resources/Pokemon_PokemonSlugs.h + Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp + Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h Source/PokemonBDSP/Inference/Battles/PokemonBDSP_BattleBallReader.cpp Source/PokemonBDSP/Inference/Battles/PokemonBDSP_BattleBallReader.h Source/PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.cpp @@ -1788,6 +1798,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Inference/PokemonBDSP_MapDetector.h Source/PokemonBDSP/Inference/PokemonBDSP_MarkFinder.cpp Source/PokemonBDSP/Inference/PokemonBDSP_MarkFinder.h + Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.cpp + Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.h Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h Source/PokemonBDSP/Inference/PokemonBDSP_PokeballSpriteMatcher.cpp From 72de561cf831917d9ce9e070caf7f5e6f5083af9 Mon Sep 17 00:00:00 2001 From: Pokemon Automation Dev Bot Date: Fri, 13 Nov 2026 09:58:35 -0800 Subject: [PATCH 2/8] Sockets: add a per-connection id to server socket sends (#1910) * Sockets: add a per-connection id to server socket sends send() can now be given the connection_id() the caller captured when it started writing. The id increments on every accept, under the socket's lock, and send() compares it against the current id at transport selection and on every loop iteration (queue time and run time in the Qt backend). A write that outlived its connection - the peer disconnected and a new client was accepted in the meantime - is refused instead of being delivered into the new client's socket. Motivated by the in-app HTTP server: its per-connection task may be inside a long handler call when the client disconnects, and a fast reconnect can be accepted before the task notices; without the id, its final response could be written into the new client's stream. Default argument (0 = no check) keeps existing callers unchanged. Test: Sockets - A send for a replaced connection is refused (runs on the Qt, POSIX, and WinSocket backends). * Sockets: report connection_id() as 0 when no connection is active The documented contract ("0 when no connection is active") now holds in all three backends: the accessor reports 0 whenever the socket is not CONNECTED. The stored counter stays monotonic - resetting it on close would let a queued write from the previous connection match the id of the next one - and send() compares against the counter, so the stale-send refusal is unaffected. Test_Sockets_StaleSend now also checks the idle state before and after the first connection. * Sockets: move the connection_id() monotonicity note to the base class Suggested by review: the note explaining why the counter is never reset was duplicated in all three backends; it now lives once in the abstract class next to the connection_id() contract. Comment-only change. --------- Co-authored-by: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> --- Common/Cpp/Sockets/AbstractServerSocket.h | 18 ++- Common/Cpp/Sockets/ServerSocket_POSIX.cpp | 30 +++-- Common/Cpp/Sockets/ServerSocket_POSIX.h | 14 ++- Common/Cpp/Sockets/ServerSocket_Qt.cpp | 35 +++++- Common/Cpp/Sockets/ServerSocket_Qt.h | 20 ++- Common/Cpp/Sockets/ServerSocket_WinSocket.cpp | 27 ++++- Common/Cpp/Sockets/ServerSocket_WinSocket.h | 14 ++- Common/Cpp/Sockets/Sockets_Tests.cpp | 114 ++++++++++++++++++ 8 files changed, 249 insertions(+), 23 deletions(-) diff --git a/Common/Cpp/Sockets/AbstractServerSocket.h b/Common/Cpp/Sockets/AbstractServerSocket.h index 4139a0f120..1c2da259ef 100644 --- a/Common/Cpp/Sockets/AbstractServerSocket.h +++ b/Common/Cpp/Sockets/AbstractServerSocket.h @@ -13,6 +13,7 @@ #define PokemonAutomation_Sockets_AbstractServerSocket_H #include +#include #include #include "Common/Cpp/ListenerSet.h" @@ -58,6 +59,18 @@ class AbstractServerSocket{ return m_state.load(std::memory_order_relaxed); } + // A token identifying the connection the socket is currently + // serving. It changes every time a new client is accepted, and it + // is 0 when no connection is active. Capture it when a write + // starts and pass it to "send()", so that a write that outlives its + // connection (the peer disconnected and a new client was accepted + // in the meantime) refuses to reach the new client's socket. + // Implementations keep the underlying counter monotonic (never + // reset it when a connection closes, or a queued write for the old + // connection could match the next connection's id); 0 is only the + // "no connection is active" report, never a counter value. + virtual uint64_t connection_id() = 0; + // Binds to "address" and listens on "port". // Use "0.0.0.0" to accept connections on all interfaces. virtual void start(const std::string& address, uint16_t port) = 0; @@ -69,7 +82,10 @@ class AbstractServerSocket{ virtual size_t send( const void* data, size_t bytes, - std::chrono::milliseconds timeout = std::chrono::milliseconds::max() + std::chrono::milliseconds timeout = std::chrono::milliseconds::max(), + // The connection_id() captured when this write started. 0 skips + // the check: the call applies to whatever connection is active. + uint64_t connection = 0 ) = 0; diff --git a/Common/Cpp/Sockets/ServerSocket_POSIX.cpp b/Common/Cpp/Sockets/ServerSocket_POSIX.cpp index 75b2a14ead..ad38549eb6 100644 --- a/Common/Cpp/Sockets/ServerSocket_POSIX.cpp +++ b/Common/Cpp/Sockets/ServerSocket_POSIX.cpp @@ -83,6 +83,14 @@ void ServerSocket_POSIX::start(const std::string& address, uint16_t port){ } } +uint64_t ServerSocket_POSIX::connection_id(){ + std::lock_guard lg(m_lock); + return m_state.load(std::memory_order_relaxed) == State::CONNECTED + ? m_connection_id + : 0; +} + + void ServerSocket_POSIX::close_connection(){ int socket; { @@ -159,16 +167,16 @@ void ServerSocket_POSIX::close_and_wait() noexcept{ } } -size_t ServerSocket_POSIX::send(const void* data, size_t bytes, std::chrono::milliseconds timeout){ +size_t ServerSocket_POSIX::send(const void* data, size_t bytes, std::chrono::milliseconds timeout, uint64_t connection){ int socket; { std::lock_guard lg(m_lock); - socket = m_state.load(std::memory_order_relaxed) == State::CONNECTED - ? m_client - : -1; - } - if (socket == -1){ - return 0; + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ + return 0; + } + socket = m_client; } const char* ptr = (const char*)data; @@ -184,7 +192,12 @@ size_t ServerSocket_POSIX::send(const void* data, size_t bytes, std::chrono::mil while (bytes > 0){ { std::lock_guard lg(m_lock); - if (m_state.load(std::memory_order_relaxed) != State::CONNECTED){ + // The connection may have been replaced (or closed) since the + // last iteration: in that case this write must stop rather + // than reach the new client's socket. + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ return sent; } } @@ -359,6 +372,7 @@ void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port, fcntl(client_socket, F_SETFL, O_NONBLOCK); m_client = client_socket; m_close_requested = false; + m_connection_id++; m_state.store(State::CONNECTED, std::memory_order_release); accepted = true; } diff --git a/Common/Cpp/Sockets/ServerSocket_POSIX.h b/Common/Cpp/Sockets/ServerSocket_POSIX.h index 6786a631fc..f08b83ca68 100644 --- a/Common/Cpp/Sockets/ServerSocket_POSIX.h +++ b/Common/Cpp/Sockets/ServerSocket_POSIX.h @@ -42,9 +42,15 @@ class ServerSocket_POSIX final : public AbstractServerSocket{ virtual size_t send( const void* data, size_t bytes, - std::chrono::milliseconds timeout = std::chrono::milliseconds::max() + std::chrono::milliseconds timeout = std::chrono::milliseconds::max(), + uint64_t connection = 0 ) override; + // A token identifying the connection the socket is currently + // serving. It changes every time a new client is accepted, and it + // is 0 when no connection is active. + virtual uint64_t connection_id() override; + private: @@ -87,6 +93,12 @@ class ServerSocket_POSIX final : public AbstractServerSocket{ int m_client = -1; bool m_close_requested = false; + // A connection token, incremented (under m_lock) on every accept. + // send() compares it against the token the caller captured, so a + // write that outlived its connection cannot reach the client that + // replaced it. 0 means no connection is active. + uint64_t m_connection_id = 0; + // One task per start() call. start() must never destroy the previous // task's AsyncTask here: its destructor blocks until the task has // finished, and a task from an earlier attempt may still be running diff --git a/Common/Cpp/Sockets/ServerSocket_Qt.cpp b/Common/Cpp/Sockets/ServerSocket_Qt.cpp index fd48c72001..359a45b1f1 100644 --- a/Common/Cpp/Sockets/ServerSocket_Qt.cpp +++ b/Common/Cpp/Sockets/ServerSocket_Qt.cpp @@ -174,13 +174,16 @@ void ServerSocket_Qt::close_and_wait() noexcept{ m_close_requested = false; } -size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::milliseconds timeout){ +size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::milliseconds timeout, uint64_t connection){ QTcpSocket* socket; { std::lock_guard lg(m_lock); - socket = m_state.load(std::memory_order_relaxed) == State::CONNECTED - ? m_socket - : nullptr; + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ + return 0; + } + socket = m_socket; } if (socket == nullptr){ return 0; @@ -188,6 +191,7 @@ size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::millis QSharedPointer send_data = QSharedPointer::create(); send_data->packet.assign((const char*)data, bytes); + send_data->connection = connection; // "time_point::max()" (the infinite default) is never reached; using // it also avoids overflowing "now() + timeout". send_data->deadline = @@ -213,7 +217,8 @@ size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::millis std::lock_guard lg(m_lock); registered = m_state.load(std::memory_order_relaxed) == State::CONNECTED - && m_socket != nullptr; + && m_socket != nullptr + && (connection == 0 || m_connection_id == connection); if (registered){ m_inflight_sends.push_back(send_data); } @@ -310,6 +315,7 @@ void ServerSocket_Qt::on_new_connection(){ }else{ m_socket = sock; m_close_requested = false; + m_connection_id++; m_state.store(State::CONNECTED, std::memory_order_release); accepted = true; rejected = false; @@ -434,11 +440,28 @@ void ServerSocket_Qt::on_close_connection(QSharedPointer comple completion->cv.notify_all(); } +uint64_t ServerSocket_Qt::connection_id(){ + std::lock_guard lg(m_lock); + return m_state.load(std::memory_order_relaxed) == State::CONNECTED + ? m_connection_id + : 0; +} + + void ServerSocket_Qt::on_send(QSharedPointer send_data){ QTcpSocket* sock; { std::lock_guard lg(m_lock); - sock = m_socket; + if (send_data->connection != 0 + && m_connection_id != send_data->connection + ){ + // The connection this send was queued for was replaced (or + // torn down) while it was waiting in the queue: the write + // loop must not reach into the new connection's socket. + sock = nullptr; + }else{ + sock = m_socket; + } } { diff --git a/Common/Cpp/Sockets/ServerSocket_Qt.h b/Common/Cpp/Sockets/ServerSocket_Qt.h index 0bcf70c0a6..370152e98c 100644 --- a/Common/Cpp/Sockets/ServerSocket_Qt.h +++ b/Common/Cpp/Sockets/ServerSocket_Qt.h @@ -54,6 +54,12 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{ // "close_and_wait()"/"close_connection()" cancel the send. bool done = false; + // The connection_id() the send was queued for; 0 skips the + // check. "on_send" refuses to write if the connection was + // replaced while the send was queued, so it cannot reach the + // new client's socket. + uint64_t connection = 0; + Mutex lock; ConditionVariable cv; }; @@ -86,9 +92,15 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{ virtual size_t send( const void* data, size_t bytes, - std::chrono::milliseconds timeout = std::chrono::milliseconds::max() + std::chrono::milliseconds timeout = std::chrono::milliseconds::max(), + uint64_t connection = 0 ) override; + // A token identifying the connection the socket is currently + // serving. It changes every time a new client is accepted, and it + // is 0 when no connection is active. + virtual uint64_t connection_id() override; + signals: void internal_send(QSharedPointer send_data); @@ -142,6 +154,12 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{ bool m_close_requested = false; bool m_starting = false; + // A connection token, incremented (under m_lock) on every accept. + // "send()" compares it against the token the caller captured, so a + // write that outlived its connection cannot reach the client that + // replaced it. 0 means no connection is active. + uint64_t m_connection_id = 0; + // The "send()" calls that are queued on the event thread but not // finished yet, so that "close_and_wait()"/"close_connection()" can // cancel diff --git a/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp b/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp index 55889f4b38..066a76ab24 100644 --- a/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp +++ b/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp @@ -94,6 +94,14 @@ void ServerSocket_WinSocket::start(const std::string& address, uint16_t port){ } } +uint64_t ServerSocket_WinSocket::connection_id(){ + std::lock_guard lg(m_lock); + return m_state.load(std::memory_order_relaxed) == State::CONNECTED + ? m_connection_id + : 0; +} + + void ServerSocket_WinSocket::close_connection(){ SOCKET socket; { @@ -170,13 +178,16 @@ void ServerSocket_WinSocket::close_and_wait() noexcept{ } } -size_t ServerSocket_WinSocket::send(const void* data, size_t bytes, std::chrono::milliseconds timeout){ +size_t ServerSocket_WinSocket::send(const void* data, size_t bytes, std::chrono::milliseconds timeout, uint64_t connection){ SOCKET socket; { std::lock_guard lg(m_lock); - socket = m_state.load(std::memory_order_relaxed) == State::CONNECTED - ? m_client - : INVALID_SOCKET; + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ + return 0; + } + socket = m_client; } if (socket == INVALID_SOCKET){ return 0; @@ -195,7 +206,12 @@ size_t ServerSocket_WinSocket::send(const void* data, size_t bytes, std::chrono: while (bytes > 0){ { std::lock_guard lg(m_lock); - if (m_state.load(std::memory_order_relaxed) != State::CONNECTED){ + // The connection may have been replaced (or closed) since the + // last iteration: in that case this write must stop rather + // than reach the new client's socket. + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ return sent; } } @@ -379,6 +395,7 @@ void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t po ioctlsocket(client_socket, FIONBIO, &client_non_blocking); m_client = client_socket; m_close_requested = false; + m_connection_id++; m_state.store(State::CONNECTED, std::memory_order_release); accepted = true; } diff --git a/Common/Cpp/Sockets/ServerSocket_WinSocket.h b/Common/Cpp/Sockets/ServerSocket_WinSocket.h index 6b68519ab8..f478e69b46 100644 --- a/Common/Cpp/Sockets/ServerSocket_WinSocket.h +++ b/Common/Cpp/Sockets/ServerSocket_WinSocket.h @@ -45,9 +45,15 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{ virtual size_t send( const void* data, size_t bytes, - std::chrono::milliseconds timeout = std::chrono::milliseconds::max() + std::chrono::milliseconds timeout = std::chrono::milliseconds::max(), + uint64_t connection = 0 ) override; + // A token identifying the connection the socket is currently + // serving. It changes every time a new client is accepted, and it + // is 0 when no connection is active. + virtual uint64_t connection_id() override; + private: // Winsock has to be started before any of its functions can be used, and @@ -96,6 +102,12 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{ SOCKET m_client = INVALID_SOCKET; bool m_close_requested = false; + // A connection token, incremented (under m_lock) on every accept. + // send() compares it against the token the caller captured, so a + // write that outlived its connection cannot reach the client that + // replaced it. 0 means no connection is active. + uint64_t m_connection_id = 0; + // One task per start() call. start() must never destroy the previous // task's AsyncTask here: its destructor blocks until the task has // finished, and a task from an earlier attempt may still be running diff --git a/Common/Cpp/Sockets/Sockets_Tests.cpp b/Common/Cpp/Sockets/Sockets_Tests.cpp index 7864a7d34a..3fe3bd915d 100644 --- a/Common/Cpp/Sockets/Sockets_Tests.cpp +++ b/Common/Cpp/Sockets/Sockets_Tests.cpp @@ -822,6 +822,117 @@ class Test_Sockets_RejectSecondClient : public TestSockets +class Test_Sockets_StaleSend : public TestSockets{ +public: + using TestSockets::TestSockets; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + ServerListener server_listener; + ServerSocketType server(this->m_thread_pool); + server.add_listener(server_listener); + + uint16_t port; + if (!this->start_server(server, server_listener, port)){ + return "Failed to start the server"; + } + + std::string error; + if (server.connection_id() != 0){ + error = "connection_id() was not 0 before any connection"; + } + + uint64_t first_id = 0; + { + ClientListener client_listener; + ClientSocketType client(this->m_thread_pool); + client.add_listener(client_listener); + + if (!this->connect_client(client, client_listener, port, 1)){ + error = "The first client did not finish connecting"; + }else if (!server_listener.waiter.wait_for( + SOCKET_TEST_WAIT, + [&server_listener]{ return server_listener.connected.load() >= 1; } + )){ + error = "The server did not call on_connect"; + }else{ + first_id = server.connection_id(); + if (first_id == 0){ + error = "The server did not assign a connection id"; + } + client.close(); + } + } + + if (error.empty() && !server_listener.waiter.wait_for( + SOCKET_TEST_WAIT, + [&server_listener]{ return server_listener.disconnected.load() >= 1; } + )){ + error = "The server did not call on_disconnect"; + } + + // The counter is monotonic and not reset, but with no active + // connection the id must report 0. + if (error.empty() && server.connection_id() != 0){ + error = "connection_id() was not 0 while no connection was active"; + } + + uint64_t second_id = 0; + { + ClientListener client_listener; + ClientSocketType client(this->m_thread_pool); + client.add_listener(client_listener); + + if (!this->connect_client(client, client_listener, port, 1)){ + error = "The second client did not finish connecting"; + }else if (!server_listener.waiter.wait_for( + SOCKET_TEST_WAIT, + [&server_listener]{ return server_listener.connected.load() >= 2; } + )){ + error = "The server did not call on_connect for the second client"; + }else{ + second_id = server.connection_id(); + if (second_id == first_id){ + error = "The connection id did not change after reconnect"; + } + // A send for the replaced connection must be refused, + // not delivered into the new client's socket. + if (error.empty() && + server.send("stale", 5, std::chrono::milliseconds::max(), first_id) != 0 + ){ + error = "A send for a replaced connection was not refused"; + } + // A send for the current connection still goes through. + if (error.empty() && + server.send("hello", 5, std::chrono::milliseconds::max(), second_id) != 5 + ){ + error = "The send for the current connection was not delivered"; + } + if (error.empty() && !client_listener.waiter.wait_for( + SOCKET_TEST_WAIT, + [&client_listener]{ return client_listener.received.size() >= 5; } + )){ + error = "The second client did not receive the data"; + }else if (error.empty() && + std::memcmp(client_listener.received.data(), "hello", 5) != 0 + ){ + error = "The second client received the wrong data"; + } + client.close(); + } + } + + if (!error.empty()){ + return error; + } + return true; + } +}; + + // Starting a second server on an already-bound port must fail through // on_listen_finished and leave the server not running. template @@ -2676,6 +2787,9 @@ void add_socket_tests(UnitTestDatabase& database, ThreadPool& thread_pool, const database.add>( thread_pool, prefix + "Reject a second client" ); + database.add>( + thread_pool, prefix + "A send for a replaced connection is refused" + ); database.add>( thread_pool, prefix + "Port in use" ); From 73fecab6efc236e94360d4d07a240b660f94c790 Mon Sep 17 00:00:00 2001 From: Dalton <59972310+Dalton-V@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:38:18 -0600 Subject: [PATCH 3/8] FRLG Box Sorter - BallReader (#1908) --- .../Inference/PokemonFRLG_BallReader.cpp | 138 ++++++++++++++++++ .../Inference/PokemonFRLG_BallReader.h | 40 +++++ .../PokemonFRLG/PokemonFRLG_Tests.cpp | 2 + .../PokemonFRLG_BoxOrganization.cpp | 35 ++++- .../PokemonFRLG_BoxOrganization.h | 4 +- .../BoxOrganization/PokemonFRLG_BoxSorter.cpp | 8 +- .../PokemonFRLG_BoxSorterLivingDex.cpp | 1 - .../PokemonFRLG_BoxSorter_Tests.cpp | 26 ++++ SerialPrograms/cmake/SourceFiles.cmake | 2 + 9 files changed, 246 insertions(+), 10 deletions(-) create mode 100644 SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.h diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.cpp new file mode 100644 index 0000000000..ae9d4c111c --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.cpp @@ -0,0 +1,138 @@ +/* Ball Reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/Notifications/ProgramInfo.h" +#include "CommonFramework/Recording/StreamHistorySession.h" +#include "CommonFramework/Tools/ErrorDumper.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonTools/ImageMatch/CroppedImageDictionaryMatcher.h" +#include "CommonTools/ImageMatch/ImageCropper.h" +#include "CommonTools/Resources/SpriteDatabase.h" +#include "Tests/ImageUnitTest.h" +#include "Tests/TestUtils.h" +#include "VideoGames/PokemonFRLG/PokemonFRLG_Settings.h" +#include "PokemonFRLG_BallReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonFRLG{ + +namespace{ + +constexpr double MAX_ALPHA = 0.40; +constexpr double ALPHA_SPREAD = 0.02; +constexpr double MIN_BACKGROUND_DISTANCE = 100; +const ImageFloatBox BALL_BOX(0.410, 0.507, 0.063, 0.084); + +class PokeballSpriteMatcher : public ImageMatch::CroppedImageDictionaryMatcher{ +public: + PokeballSpriteMatcher() + : CroppedImageDictionaryMatcher({1, 128}) + { + const SpriteDatabase sprites("PokemonFRLG/PokeballSprites.png", "PokemonFRLG/PokeballSprites.json"); + for (const auto& item : sprites){ + add(item.first, item.second.sprite); + } + } + +private: + std::vector get_crop_candidates(const ImageViewRGB32& image) const override{ + const FloatPixel background = image_border_stats(image).average; + const ImagePixelBox box = ImageMatch::enclosing_rectangle_with_pixel_filter( + image, + [&](Color pixel){ + double r = (double)pixel.red() - background.r; + double g = (double)pixel.green() - background.g; + double b = (double)pixel.blue() - background.b; + return r*r + g*g + b*b >= MIN_BACKGROUND_DISTANCE * MIN_BACKGROUND_DISTANCE; + } + ); + return {extract_box_reference(image, box)}; + } +}; + +} + + +BallReader::BallReader(VideoStream& stream) + : m_stream(stream) + , m_box_sprite(stream.overlay(), GameSettings::instance().GAME_BOX.inner_to_outer(BALL_BOX)) +{} + +std::string BallReader::read_ball(const ImageViewRGB32& screen) const{ + if (!screen){ + return ""; + } + + static const PokeballSpriteMatcher matcher; + const ImageViewRGB32 image = extract_box_reference(screen, m_box_sprite); + ImageMatch::ImageMatchResult result = matcher.match(image, ALPHA_SPREAD); + result.log(m_stream.logger(), MAX_ALPHA); + result.clear_beyond_alpha(MAX_ALPHA); + + if (result.results.size() != 1){ + dump_image(m_stream.logger(), ProgramInfo(), "FRLG-BallReader", screen); + } + if (result.results.empty()){ + return ""; + } + return result.results.begin()->second; +} + + +class Test_BallReader : public ImageUnitTest{ +public: + Test_BallReader(const std::string& image, std::string expected) + : ImageUnitTest("PokemonFRLG::BallReader - " + image, {"PokemonFRLG/StatsReader/Page1/" + image}) + , m_expected(std::move(expected)) + {} + +private: + UnitTestResult run_on_images( + Logger& logger, CancellableScope&, + const std::vector& images + ) const override{ + DummyAudioFeed audio_feed; + DummyVideoFeed video_feed; + StreamHistorySession history(logger); + DummyVideoOverlay overlay; + VideoStream stream(logger, audio_feed, video_feed, history, overlay); + BallReader reader(stream); + std::string result = reader.read_ball(images[0]); + if (result == m_expected){ + return true; + } + return "Expected: " + m_expected + ", received: " + result; + } + + std::string m_expected; +}; + +void add_tests_BallReader(UnitTestDatabase& database){ + database.add("abra_1_deu.png", "master-ball"); + database.add("bulbasaur_1_eng.png", "poke-ball"); + database.add("bulbasaur_1_fra.png", "poke-ball"); + database.add("bulbasaur_1_jpn.png", "poke-ball"); + database.add("bulbasaur_2_eng.png", "poke-ball"); + database.add("bulbasaur_3_eng.png", "poke-ball"); + database.add("bulbasaur_4_eng.png", "poke-ball"); + database.add("deoxys_1_jpn.png", "master-ball"); + database.add("horsea_1_deu.png", "master-ball"); + database.add("horsea_2_deu.png", "master-ball"); + database.add("horsea_3_deu.png", "master-ball"); + database.add("magikarp_1_spa.png", "poke-ball"); + database.add("moltres_1_eng.png", "master-ball"); + database.add("nidoran_m_eng.png", "ultra-ball"); + database.add("pidgey_1_ita.png", "poke-ball"); + database.add("venonat_4k_eng.png", "master-ball"); +} + + +} +} +} diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.h b/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.h new file mode 100644 index 0000000000..f7e38d75c4 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.h @@ -0,0 +1,40 @@ +/* Ball Reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonFRLG_BallReader_H +#define PokemonAutomation_PokemonFRLG_BallReader_H + +#include +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" + +namespace PokemonAutomation{ + +class UnitTestDatabase; +class VideoStream; + +namespace NintendoSwitch{ +namespace PokemonFRLG{ + + +class BallReader{ +public: + BallReader(VideoStream& stream); + + // Read the ball slug from the first summary page. Return an empty string if no match is found. + std::string read_ball(const ImageViewRGB32& screen) const; + +private: + VideoStream& m_stream; + OverlayBoxScope m_box_sprite; +}; + +void add_tests_BallReader(UnitTestDatabase& database); + + +} +} +} +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/PokemonFRLG_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/PokemonFRLG_Tests.cpp index 94dc41dd85..31746134b7 100644 --- a/SerialPrograms/Source/VideoGames/PokemonFRLG/PokemonFRLG_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/PokemonFRLG_Tests.cpp @@ -20,6 +20,7 @@ #include "Inference/Menus/PokemonFRLG_PartyEmptySlotDetector.h" #include "Inference/Menus/PokemonFRLG_PartyHeldItemDetector.h" #include "Inference/Menus/PokemonFRLG_PartyMenuDetector.h" +#include "Inference/PokemonFRLG_BallReader.h" #include "Inference/PokemonFRLG_BattleLevelUpReader.h" #include "Inference/PokemonFRLG_BattleSelectionArrowDetector.h" #include "Inference/PokemonFRLG_PartyLevelUpReader.h" @@ -40,6 +41,7 @@ void add_tests(UnitTestDatabase& database){ add_tests_AdvanceWhiteDialogDetector(database); add_tests_AdvanceBattleDialogDetector(database); add_tests_BagDetector(database); + add_tests_BallReader(database); add_tests_BattleLevelUpReader(database); add_tests_BattleMenuDetector(database); add_tests_BattleMoveSelectionArrow(database); diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxOrganization.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxOrganization.cpp index bfdbfab3f4..99847df668 100644 --- a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxOrganization.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxOrganization.cpp @@ -23,6 +23,7 @@ #include "VideoGames/PokemonFRLG/Inference/Box/PokemonFRLG_BoxGenderReader.h" #include "VideoGames/PokemonFRLG/Inference/Box/PokemonFRLG_BoxSelectionModeDetector.h" #include "VideoGames/PokemonFRLG/Inference/Menus/PokemonFRLG_SummaryDetector.h" +#include "VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.h" #include "VideoGames/PokemonFRLG/Inference/PokemonFRLG_SelectionArrowDetector.h" #include "VideoGames/PokemonFRLG/Inference/PokemonFRLG_ShinySymbolDetector.h" #include "VideoGames/PokemonFRLG/Inference/PokemonFRLG_StatsReader.h" @@ -162,8 +163,7 @@ void open_summary_screen(SingleSwitchProgramEnvironment& env, ProControllerConte wait_for_summary_animation(env, context); } -// Scroll through the detected pokemon and read their summary screens. -// Currently only reads shininess, but could be extended to read other summary information. +// Scroll through the detected Pokemon and read their shininess and ball type. void scan_box_summaries( SingleSwitchProgramEnvironment& env, ProControllerContext& context, Language language, const BoxSelectionModeDetector& selection_detector, BoxCursor& cursor, @@ -180,7 +180,7 @@ void scan_box_summaries( return; } - env.log("Reading shininess in box " + std::to_string(cursor.box + 1)); + env.log("Reading summaries in box " + std::to_string(cursor.box + 1)); move_cursor_to(env, context, selection_detector, cursor, BoxCursor(occupied.back()), BoxSelectionMode::Normal); open_summary_screen(env, context); @@ -189,6 +189,8 @@ void scan_box_summaries( ShinySymbolDetector shiny_detector(COLOR_YELLOW); VideoOverlaySet overlays(env.console.overlay()); shiny_detector.make_overlays(overlays); + BallReader ball_reader(env.console); + for (size_t remaining = occupied.size(); remaining > 0; remaining--){ const size_t index = occupied[remaining - 1]; auto& pokemon = *boxes_data[index]; @@ -203,6 +205,21 @@ void scan_box_summaries( OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, ss.str(), env.console); } pokemon.shiny = shiny_detector.read(env.logger(), screen); + pokemon.ball_slug = ball_reader.read_ball(screen); + if (pokemon.ball_slug.empty()){ + std::ostringstream ss; + ss << "Box Sorter: Unable to read the ball type at " << BoxCursor(index) << "."; + env.log(ss.str(), COLOR_RED); + } + + std::ostringstream overview; + overview << "index:" << index + << ", name:" << pokemon.name_slug + << ", lvl:" << level_str(pokemon.level) + << ", shiny:" << (pokemon.shiny ? "True" : "False") + << ", ball:" << pokemon.ball_slug; + env.log(overview.str()); + pbf_press_dpad(context, DPAD_UP, 80ms, 300ms); context.wait_for_all_requests(); wait_for_summary_animation(env, context); @@ -421,10 +438,16 @@ std::string make_missing_living_dex_report( BoxOrganizationScan scan_box_data( SingleSwitchProgramEnvironment& env, ProControllerContext& context, - Language language, const std::vector& preferences, size_t num_boxes, bool read_shininess + Language language, const std::vector& preferences, size_t num_boxes, bool force_summary_scan ){ - bool read_summary = read_shininess || std::any_of(preferences.begin(), preferences.end(), [](const SortingRule& rule){ - return rule.sort_type == SortingRuleType::Shiny; + const bool read_summary = force_summary_scan || std::any_of(preferences.begin(), preferences.end(), [](const SortingRule& rule){ + switch (rule.sort_type){ + case SortingRuleType::Shiny: + case SortingRuleType::Ball_Slug: + return true; + default: + return false; + } }); auto& stats = env.current_stats(); const size_t slot_count = num_boxes * BOX_ROWS * BOX_COLS; diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxOrganization.h b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxOrganization.h index b2401191e6..6c2e90348c 100644 --- a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxOrganization.h +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxOrganization.h @@ -75,11 +75,11 @@ struct BoxOrganizationScan{ BoxSelectionMode mode; }; -// Detecting Shiny requires reading the summary screen, this is much slower than just reading the box data. +// Scan summaries for rules that need summary data or when the caller requests a summary scan. BoxOrganizationScan scan_box_data( SingleSwitchProgramEnvironment& env, ProControllerContext& context, Language language, const std::vector& preferences, size_t num_boxes, - bool read_shininess = false + bool force_summary_scan = false ); void execute_box_sorting_plan( diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorter.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorter.cpp index 6c2f7edbf5..dd725ec99d 100644 --- a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorter.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorter.cpp @@ -65,7 +65,13 @@ BoxSorter::BoxSorter() ) , SORT_TABLE( "Sort Order Rules:
Sort order rules will be applied top to bottom.", - {SortingRuleType::DexNo, SortingRuleType::Level, SortingRuleType::Gender, SortingRuleType::Shiny} + { + SortingRuleType::DexNo, + SortingRuleType::Level, + SortingRuleType::Gender, + SortingRuleType::Shiny, + SortingRuleType::Ball_Slug, + } ) , DRY_RUN( "Dry Run:
Scan boxes and write frlg-box_order.json and " diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorterLivingDex.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorterLivingDex.cpp index aa48844633..6035c5c36e 100644 --- a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorterLivingDex.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorterLivingDex.cpp @@ -12,7 +12,6 @@ #include "CommonTools/StartupChecks/StartProgramChecks.h" #include "Pokemon/Pokemon_Strings.h" #include "PokemonFRLG_BoxOrganization.h" -#include "PokemonFRLG_BoxSorterLivingDex.h" #include "VideoGames/PokemonFRLG/PokemonFRLG_Navigation.h" #include "PokemonFRLG_BoxSorterLivingDex.h" diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorter_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorter_Tests.cpp index f529d86d53..e63660ec33 100644 --- a/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorter_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Programs/BoxOrganization/PokemonFRLG_BoxSorter_Tests.cpp @@ -7,11 +7,14 @@ #include #include #include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonObject.h" #include "Common/Cpp/TestRunners/UnitTest.h" #include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/GlobalAutoPaths.h" #include "Pokemon/Resources/Pokemon_PokemonSlugs.h" #include "PokemonFRLG_BoxOrganization.h" +#include "PokemonFRLG_BoxSorter.h" #include "PokemonFRLG_BoxSorter_Tests.h" namespace PokemonAutomation{ @@ -73,6 +76,29 @@ class Test_BoxSorter_Ordering : public UnitTest{ if (slots != std::vector>{low, high, shiny, std::nullopt}){ return "Reversed shiny sorting must put non-shinies first and empty slots last."; } + auto poke_ball = low; + poke_ball->ball_slug = "poke-ball"; + auto master_ball = poke_ball; + master_ball->ball_slug = "master-ball"; + auto high_poke_ball = high; + high_poke_ball->ball_slug = "poke-ball"; + rules = {{SortingRuleType::Ball_Slug, false}, {SortingRuleType::Level, false}}; + const std::vector> original{ + poke_ball, master_ball, high_poke_ball, std::nullopt + }; + slots = {high_poke_ball, std::nullopt, poke_ball, master_ball}; + std::stable_sort(slots.begin(), slots.end()); + if (slots != std::vector>{master_ball, poke_ball, high_poke_ball, std::nullopt}){ + return "Ball Type sorting must group balls alphabetically and break ties by level."; + } + if (make_box_sorting_plan(original, slots).chains != std::vector>{{0, 1, 0}}){ + return "The swap plan must distinguish Pokemon that differ only by ball type."; + } + rules[0].reverse = true; + std::stable_sort(slots.begin(), slots.end()); + if (slots != std::vector>{poke_ball, high_poke_ball, master_ball, std::nullopt}){ + return "Reversing Ball Type must leave the level tie-breaker and empty slots in order."; + } if (SLUGS_TO_NATIONAL_DEX().at("bulbasaur") != 1 || SLUGS_TO_NATIONAL_DEX().at("deoxys") != 386){ return "Species conversion must use National Dex numbers."; } diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 03a76d8719..eb81069fbd 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1731,6 +1731,8 @@ file(GLOB LIBRARY_SOURCES Source/VideoGames/PokemonFRLG/Inference/Menus/PokemonFRLG_SummaryDetector.h Source/VideoGames/PokemonFRLG/Inference/Menus/PokemonFRLG_TrainerCardDetector.cpp Source/VideoGames/PokemonFRLG/Inference/Menus/PokemonFRLG_TrainerCardDetector.h + Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.cpp + Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BallReader.h Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BattleLevelUpReader.cpp Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BattleLevelUpReader.h Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_BattlePokemonDetector.cpp From 39f94a99f93dcff18b73ad3a1d78b305867eb46c Mon Sep 17 00:00:00 2001 From: theAstroGoth <52220914+theastrogoth@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:07:19 -0600 Subject: [PATCH 4/8] FRLG SpriteReader fixes and tests (#1913) * FRLG SpriteReader fixes and tests * allow for mirroring sprites left to right in SpriteDatabase --- .../CommonTools/Resources/SpriteDatabase.cpp | 16 +- .../CommonTools/Resources/SpriteDatabase.h | 10 +- .../Source/Pokemon/Pokemon_AdvRng.cpp | 14 +- .../Source/Pokemon/Pokemon_AdvRng.h | 9 +- .../PokemonFRLG_PokemonSpriteReader.cpp | 308 +++++++++++++++--- .../PokemonFRLG_PokemonSpriteReader.h | 76 ++++- .../Inference/PokemonFRLG_StatsReader.cpp | 7 + .../PokemonFRLG_WildEncounterReader.cpp | 28 +- .../PokemonFRLG/PokemonFRLG_Tests.cpp | 3 + .../PokemonFRLG_RngCalibration.cpp | 2 +- .../PokemonFRLG_RngDisplays.cpp | 24 +- .../PokemonFRLG_RngNavigation.cpp | 10 +- .../Resources/PokemonFRLG_PokemonSprites.cpp | 63 +++- .../Resources/PokemonFRLG_PokemonSprites.h | 21 ++ 14 files changed, 501 insertions(+), 90 deletions(-) diff --git a/SerialPrograms/Source/CommonTools/Resources/SpriteDatabase.cpp b/SerialPrograms/Source/CommonTools/Resources/SpriteDatabase.cpp index 9a615e8b4c..abf5aae895 100644 --- a/SerialPrograms/Source/CommonTools/Resources/SpriteDatabase.cpp +++ b/SerialPrograms/Source/CommonTools/Resources/SpriteDatabase.cpp @@ -4,6 +4,7 @@ * */ +#include #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/Json/JsonObject.h" #include "CommonFramework/GlobalAutoPaths.h" @@ -15,7 +16,10 @@ namespace PokemonAutomation{ -SpriteDatabase::SpriteDatabase(const char* sprite_path, const char* json_path) +SpriteDatabase::SpriteDatabase( + const char* sprite_path, const char* json_path, + const std::function& mirror_horizontally +) : m_backing_image(RESOURCE_PATH() + sprite_path) { std::string path = RESOURCE_PATH() + json_path; @@ -37,6 +41,16 @@ SpriteDatabase::SpriteDatabase(const char* sprite_path, const char* json_path) JsonObject& obj = item.second.to_object_throw(path); int y = (int)obj.get_integer_throw("top", path); int x = (int)obj.get_integer_throw("left", path); + if (mirror_horizontally && mirror_horizontally(slug)){ + if (x < 0 || y < 0 || (size_t)(x + width) > m_backing_image.width() || (size_t)(y + height) > m_backing_image.height()){ + throw FileException(nullptr, PA_CURRENT_FUNCTION, "Sprite out of bounds: " + slug, path); + } + for (int r = y; r < y + (int)height; r++){ + for (int c0 = x, c1 = x + (int)width - 1; c0 < c1; c0++, c1--){ + std::swap(m_backing_image.pixel(c0, r), m_backing_image.pixel(c1, r)); + } + } + } ImageViewRGB32 sprite = extract_box_reference(m_backing_image, ImagePixelBox(x, y, x + width, y + height)); m_database.emplace( diff --git a/SerialPrograms/Source/CommonTools/Resources/SpriteDatabase.h b/SerialPrograms/Source/CommonTools/Resources/SpriteDatabase.h index 3136ffd6ec..3213479227 100644 --- a/SerialPrograms/Source/CommonTools/Resources/SpriteDatabase.h +++ b/SerialPrograms/Source/CommonTools/Resources/SpriteDatabase.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_CommonTools_Resources_SpriteCompositeImage_H #define PokemonAutomation_CommonTools_Resources_SpriteCompositeImage_H +#include #include #include "CommonFramework/ImageTypes/ImageRGB32.h" @@ -32,7 +33,14 @@ class SpriteDatabase{ // (next pokemon) ... // } // } - SpriteDatabase(const char* sprite_path, const char* json_path); + // + // If "mirror_horizontally" is set, each sprite for which it returns true + // is flipped left-to-right in place. This avoids storing a separate + // flipped copy of the image. + SpriteDatabase( + const char* sprite_path, const char* json_path, + const std::function& mirror_horizontally = nullptr + ); public: struct Sprite{ diff --git a/SerialPrograms/Source/Pokemon/Pokemon_AdvRng.cpp b/SerialPrograms/Source/Pokemon/Pokemon_AdvRng.cpp index aef20552e7..4f888bd9d1 100644 --- a/SerialPrograms/Source/Pokemon/Pokemon_AdvRng.cpp +++ b/SerialPrograms/Source/Pokemon/Pokemon_AdvRng.cpp @@ -270,6 +270,18 @@ std::string base_species_slug(const std::string& species){ return species; } +bool species_matches(const std::string& observed, const std::string& candidate){ + const std::string base = base_species_slug(observed); + if (base != base_species_slug(candidate)){ + return false; + } + // Accept matches with the base slug in case form couldn't be read + if (observed == base || candidate == base){ + return true; + } + return observed == candidate; +} + int slot_to_unownform(AdvEncounterSlot slot){ if (slot.species.find("unown") == std::string::npos){ return -1; @@ -521,7 +533,7 @@ bool check_for_match(AdvPokemonResult res, AdvRngFilters target, int16_t gender_ } bool check_for_match(AdvWildPokemonResult res, AdvRngFilters target, int16_t gender_threshold, uint16_t tid_xor_sid){ - return (base_species_slug(target.species) == base_species_slug(res.species)) + return species_matches(target.species, res.species) && (target.level == res.level) && (target.nature == AdvNature::Any || (res.nature == target.nature)) && (target.ability == AdvAbility::Any || (res.ability == target.ability)) diff --git a/SerialPrograms/Source/Pokemon/Pokemon_AdvRng.h b/SerialPrograms/Source/Pokemon/Pokemon_AdvRng.h index 85a3ad8707..616063c46d 100644 --- a/SerialPrograms/Source/Pokemon/Pokemon_AdvRng.h +++ b/SerialPrograms/Source/Pokemon/Pokemon_AdvRng.h @@ -202,11 +202,14 @@ Pokemon::NatureAdjustments nature_to_adjustment(AdvNature nature); Pokemon::AdvNature string_to_nature(const std::string& nature_string); std::string nature_to_string(const AdvNature& nature); -// Forms that cannot be told apart from an observation (e.g. Unown letters) share a base slug. -// Encounter slots carry the form-specific slug, while an observed species only ever carries -// the base slug, so both sides must be folded before they are compared. +// Forms of the same species (e.g. Unown letters) share a base slug. +// Encounter slots carry the form-specific slug (e.g. "unown-b"). std::string base_species_slug(const std::string& species); +// Return true if an observed species is consistent with a candidate species. +// Species with forms match by base slug, unless both sides name a specific form +bool species_matches(const std::string& observed, const std::string& candidate); + std::string gender_to_string(const AdvGender& gender); AdvGender gender_from_gender_value(uint8_t gender_value, int16_t threshold); diff --git a/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_PokemonSpriteReader.cpp b/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_PokemonSpriteReader.cpp index 6e2c516038..f4e05933ce 100644 --- a/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_PokemonSpriteReader.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/PokemonFRLG_PokemonSpriteReader.cpp @@ -4,10 +4,14 @@ * */ +#include +#include +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" -#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/GlobalSettingsPanel.h" -#include "CommonTools/ImageMatch/ImageCropper.h" +#include "Tests/ImageUnitTest.h" +#include "PokemonFRLG_ShinySymbolDetector.h" #include "VideoGames/PokemonFRLG/Resources/PokemonFRLG_PokemonSprites.h" #include "VideoGames/PokemonFRLG/PokemonFRLG_Settings.h" #include "PokemonFRLG_PokemonSpriteReader.h" @@ -17,78 +21,286 @@ namespace NintendoSwitch{ namespace PokemonFRLG{ -PokemonSpriteMatcherExact::PokemonSpriteMatcherExact(const std::set* subset, bool flipped) - : ExactImageDictionaryMatcher({1, 256}) +namespace{ + +// Positions are in GBA pixels. The GBA screen is 240 x 160. +constexpr int GBA_WIDTH = 240; +constexpr int GBA_HEIGHT = 160; +constexpr int SPRITE_SIZE = 64; + +ImageFloatBox sprite_box(int x, int y){ + return ImageFloatBox( + (double)x / GBA_WIDTH, (double)y / GBA_HEIGHT, + (double)SPRITE_SIZE / GBA_WIDTH, (double)SPRITE_SIZE / GBA_HEIGHT + ); +} + +// When the summary is opened, the sprite bounces up by as much as 9 pixels. +// the following allow 1 extra pixel of slack in each direction. +constexpr int SUMMARY_X_MIN = 27; +constexpr int SUMMARY_X_MAX = 31; +constexpr int SUMMARY_Y_MIN = 23; +constexpr int SUMMARY_Y_MAX = 34; + +// In battle, the sprite's top edge depends on the species and whether it floats. +constexpr int BATTLE_X = 144; +constexpr int BATTLE_Y_MIN = 0; +constexpr int BATTLE_Y_MAX = 36; + +} + + + +PositionedSpriteMatcher::PositionedSpriteMatcher( + const std::set* subset, + bool flipped, // most sprites are flipped on the summary screen + const std::vector& shiny_variants, // {false, true} to match against both normal and shiny sprites + int x_min, int x_max, int y_min, int y_max +) + : m_x_min(x_min), m_x_max(x_max) + , m_y_min(y_min), m_y_max(y_max) { - for (const auto& item : (flipped ? FLIPPED_POKEMON_SPRITES() : ALL_POKEMON_SPRITES())){ - if (subset == nullptr || subset->find(item.first) != subset->end()){ - add(item.first, item.second.sprite.copy()); + for (bool shiny : shiny_variants){ + for (const auto& item : POKEMON_SPRITES(shiny, flipped)){ + if (subset != nullptr && !sprite_in_species_subset(item.first, *subset)){ + continue; + } + m_templates.emplace_back(new Template{ + sprite_slug_to_species(item.first), + shiny, + ImageMatch::WeightedExactImageMatcher(item.second.sprite.copy(), {1, 256}) + }); } } } +ImageFloatBox PositionedSpriteMatcher::search_box() const{ + return ImageFloatBox( + (double)m_x_min / GBA_WIDTH, (double)m_y_min / GBA_HEIGHT, + (double)(m_x_max - m_x_min + SPRITE_SIZE) / GBA_WIDTH, + (double)(m_y_max - m_y_min + SPRITE_SIZE) / GBA_HEIGHT + ); +} -PokemonSpriteMatcherCropped::PokemonSpriteMatcherCropped(const std::set* subset, double min_euclidean_distance, bool flipped) - : CroppedImageDictionaryMatcher({1, 256}) - , m_min_euclidean_distance_squared(min_euclidean_distance * min_euclidean_distance) -{ - for (const auto& item : (flipped ? FLIPPED_POKEMON_SPRITES() : ALL_POKEMON_SPRITES())){ - if (subset == nullptr || subset->find(item.first) != subset->end()){ - add(item.first, item.second.sprite); +PositionedSpriteMatcher::Result PositionedSpriteMatcher::match( + const ImageViewRGB32& game_screen, double alpha_spread +) const{ + Result ret; + if (!game_screen || m_templates.empty()){ + return ret; + } + + // Every possible sprite position scaled to the template size + std::vector candidates; + for (int y = m_y_min; y <= m_y_max; y++){ + for (int x = m_x_min; x <= m_x_max; x++){ + candidates.emplace_back( + extract_box_reference(game_screen, sprite_box(x, y)).scale_to(SPRITE_SIZE, SPRITE_SIZE) + ); } } -} -auto PokemonSpriteMatcherCropped::get_crop_candidates(const ImageViewRGB32& image) const -> std::vector{ - ImageStats border = image_border_stats(image); - ImagePixelBox box = ImageMatch::enclosing_rectangle_with_pixel_filter( - image, - [&](Color pixel){ - double r = (double)pixel.red() - border.average.r; - double g = (double)pixel.green() - border.average.g; - double b = (double)pixel.blue() - border.average.b; - bool stop = r*r + g*g + b*b >= m_min_euclidean_distance_squared; - return stop; + std::map species_best; + double best_alpha = 10000; + for (const std::unique_ptr