From 139803fc64a65a5df7f76b2213b365eb8fefe7e7 Mon Sep 17 00:00:00 2001 From: Dalton-V Date: Sun, 8 Nov 2026 09:12:00 -0600 Subject: [PATCH 1/2] Improve home box arrow detection --- .../PokemonHome_SelectionArrowDetector.cpp | 335 ++++++++++++------ .../PokemonHome_SelectionArrowDetector.h | 79 +++-- .../PokemonHome_TestDatabaseGenerator.cpp | 49 ++- 3 files changed, 296 insertions(+), 167 deletions(-) diff --git a/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.cpp index 278ef8e843..7884ce3993 100644 --- a/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.cpp @@ -4,161 +4,264 @@ * */ -#include "Common/Cpp/Exceptions.h" +#include +#include #include "Kernels/Waterfill/Kernels_Waterfill_Types.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h" #include "CommonTools/Images/WaterfillUtilities.h" #include "Tests/ImageUnitTest.h" -#include "Tests/TestUtils.h" #include "PokemonHome_SelectionArrowDetector.h" namespace PokemonAutomation{ namespace NintendoSwitch{ namespace PokemonHome{ +namespace{ +const ImageFloatBox ARROW_SEARCH_BOX(0.02, 0.085, 0.47, 0.69); class SelectionArrowMatcher : public ImageMatch::WaterfillTemplateMatcher{ public: - // The Pokemon Home arrows are red on a light background. - SelectionArrowMatcher(const char* path) - : WaterfillTemplateMatcher( - path, - Color(0xffa00000), Color(0xffffc0c0), 100 - ) + SelectionArrowMatcher(const char* path, Color min_color, Color max_color) + : WaterfillTemplateMatcher(path, min_color, max_color, 100) { - m_aspect_ratio_lower = 0.9; - m_aspect_ratio_upper = 1.1; - m_area_ratio_lower = 0.85; - m_area_ratio_upper = 1.1; - } - - static const SelectionArrowMatcher& matcher(SelectionArrowType type){ - switch (type){ - case SelectionArrowType::RIGHT: - return RIGHT_ARROW(); - case SelectionArrowType::DOWN: - return DOWN_ARROW(); - default: - throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Invalid enum."); - } - } - static const SelectionArrowMatcher& RIGHT_ARROW(){ - static SelectionArrowMatcher matcher("PokemonHome/SelectionArrows/SelectionArrowRight.png"); - return matcher; - } - static const SelectionArrowMatcher& DOWN_ARROW(){ - static SelectionArrowMatcher matcher("PokemonHome/SelectionArrows/SelectionArrowDown.png"); - return matcher; + m_aspect_ratio_lower = 0.8; + m_aspect_ratio_upper = 1.25; + m_area_ratio_lower = 0.75; + m_area_ratio_upper = 1.25; } }; -SelectionArrowDetector::SelectionArrowDetector( - Color color, - VideoOverlay* overlay, - SelectionArrowType type, - const ImageFloatBox& box -) - : m_color(color) - , m_overlay(overlay) - , m_type(type) - , m_arrow_box(box) -{} -void SelectionArrowDetector::make_overlays(VideoOverlaySet& items) const{ - items.add(m_color, m_arrow_box); -} -bool SelectionArrowDetector::detect(const ImageViewRGB32& screen){ - double screen_rel_size = (screen.height() / 1080.0); - double screen_rel_size_2 = screen_rel_size * screen_rel_size; - - double min_area_1080p = 700; - double rmsd_threshold = 80; - size_t min_area = size_t(screen_rel_size_2 * min_area_1080p); +struct ArrowStyle{ + SelectionArrowMatcher matcher; + BoxSelectionMode mode; + bool down; + uint32_t min_color; + uint32_t max_color; +}; - const std::vector> FILTERS = { - {0xff800000, 0xffff9090}, - {0xffa00000, 0xffffa0a0}, - {0xffc00000, 0xffffc0c0}, - {0xffd00000, 0xffffd0d0}, +const auto& arrow_styles(){ + static const ArrowStyle styles[] = { + {{"PokemonHome/SelectionArrows/SelectionArrowDown.png", Color(160, 0, 0), Color(255, 192, 192)}, + BoxSelectionMode::Normal, true, 0xffa00000, 0xffffc0c0}, + {{"PokemonHome/SelectionArrows/SelectionArrowRight.png", Color(160, 0, 0), Color(255, 192, 192)}, + BoxSelectionMode::Normal, false, 0xffa00000, 0xffffc0c0}, + {{"PokemonHome/SelectionArrows/SelectionArrowDownBlue.png", Color(0, 100, 160), Color(128, 224, 255)}, + BoxSelectionMode::Relocate, true, 0xff0064a0, 0xff80e0ff}, + {{"PokemonHome/SelectionArrows/SelectionArrowRightBlue.png", Color(0, 100, 160), Color(128, 224, 255)}, + BoxSelectionMode::Relocate, false, 0xff0064a0, 0xff80e0ff}, + {{"PokemonHome/SelectionArrows/SelectionArrowDownGreen.png", Color(0, 160, 0), Color(128, 255, 160)}, + BoxSelectionMode::Multi, true, 0xff00a000, 0xff80ffa0}, + {{"PokemonHome/SelectionArrows/SelectionArrowRightGreen.png", Color(0, 160, 0), Color(128, 255, 160)}, + BoxSelectionMode::Multi, false, 0xff00a000, 0xff80ffa0}, }; + return styles; +} - bool found = match_template_by_waterfill( - screen.size(), - extract_box_reference(screen, m_arrow_box), - SelectionArrowMatcher::matcher(m_type), - FILTERS, - {min_area, SIZE_MAX}, - rmsd_threshold, - [&](Kernels::Waterfill::WaterfillObject& object) -> bool { - m_last_detected = translate_to_parent(screen, m_arrow_box, object); - return true; +std::optional selection_at(const ImageFloatBox& arrow, const ArrowStyle& style){ + const double center_x = arrow.x + arrow.width / 2; + const double center_y = arrow.y + arrow.height / 2; + if (!style.down){ + if (std::abs(center_y - 0.123) < 0.025 && std::abs(center_x - 0.12) < 0.025){ + return BoxSelection{style.mode, BoxSelectionLocation::BoxName}; } - ); - - if (m_overlay){ - if (found){ - m_last_detected_box.emplace(*m_overlay, m_last_detected, COLOR_GREEN); - }else{ - m_last_detected_box.reset(); + if (std::abs(center_y - 0.738) < 0.025){ + if (std::abs(center_x - 0.18) < 0.025){ + return BoxSelection{style.mode, BoxSelectionLocation::BoxSpaces}; + } + if (std::abs(center_x - 0.332) < 0.025){ + return BoxSelection{style.mode, BoxSelectionLocation::Newest30}; + } } + return std::nullopt; } - return found; + constexpr double FIRST_GRID_ARROW_X = 0.075; + constexpr double FIRST_GRID_ARROW_Y = 0.16; + constexpr double COLUMN_SPACING = 0.072; + constexpr double ROW_SPACING = 0.1055; + const int column = (int)std::lround((center_x - FIRST_GRID_ARROW_X) / COLUMN_SPACING); + const int row = (int)std::lround((center_y - FIRST_GRID_ARROW_Y) / ROW_SPACING); + if (column < 0 || column >= 6 || row < 0 || row >= 5){ + return std::nullopt; + } + if (std::abs(center_x - (FIRST_GRID_ARROW_X + column * COLUMN_SPACING)) > 0.015 + || std::abs(center_y - (FIRST_GRID_ARROW_Y + row * ROW_SPACING)) > 0.025){ + return std::nullopt; + } + return BoxSelection{style.mode, BoxSelectionLocation::Grid, (size_t)row, (size_t)column}; } -class Test_SelectionArrowDetector : public ImageUnitTest{ -public: +std::string selection_string(const std::optional& selection){ + if (!selection){ + return "none"; + } + std::string result; + switch (selection->mode){ + case BoxSelectionMode::Normal: + result = "Normal "; + break; + case BoxSelectionMode::Relocate: + result = "Relocate "; + break; + case BoxSelectionMode::Multi: + result = "Multi "; + break; + } + switch (selection->location){ + case BoxSelectionLocation::Grid: + return result + "Grid(" + std::to_string(selection->row) + ", " + std::to_string(selection->column) + ")"; + case BoxSelectionLocation::BoxName: + return result + "BoxName"; + case BoxSelectionLocation::BoxSpaces: + return result + "BoxSpaces"; + case BoxSelectionLocation::Newest30: + return result + "Newest30"; + } + return result; +} - Test_SelectionArrowDetector( - const std::string& image, - std::optional expected - ) - : ImageUnitTest("PokemonHome::SelectionArrowDetector - " + image, {image}) - , m_expected(expected) - {} +} - virtual UnitTestResult run_on_images( - Logger& logger, CancellableScope& scope, - const std::vector& images - ) const override{ - DummyVideoOverlay video_overlay; - const ImageRGB32& image = images[0]; - SelectionArrowDetector right_detector(COLOR_BLACK, &video_overlay, SelectionArrowType::RIGHT, ImageFloatBox(0.463, 0.09, 0.04, 0.06)); - SelectionArrowDetector down_detector(COLOR_BLACK, &video_overlay, SelectionArrowType::DOWN, ImageFloatBox(0.463, 0.09, 0.04, 0.06)); +void SelectionArrowDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, ARROW_SEARCH_BOX); +} - const bool right_detected = right_detector.detect(image); - const bool down_detected = down_detector.detect(image); +std::optional SelectionArrowDetector::detect(const ImageViewRGB32& screen) const{ + const ImageViewRGB32 region = extract_box_reference(screen, ARROW_SEARCH_BOX); + const double screen_rel_size = screen.height() / 1080.0; + const size_t min_area = std::max(1, size_t(screen_rel_size * screen_rel_size * 500)); + double best_rmsd = 80; + std::optional best; + for (const ArrowStyle& style : arrow_styles()){ + match_template_by_waterfill( + screen.size(), region, style.matcher, + {{style.min_color, style.max_color}}, {min_area, SIZE_MAX}, best_rmsd, + [&](Kernels::Waterfill::WaterfillObject& object){ + const ImageFloatBox arrow = translate_to_parent(screen, ARROW_SEARCH_BOX, object); + if (arrow.width < 0.01 || arrow.width > 0.035 || arrow.height < 0.015 || arrow.height > 0.065){ + return false; + } + const auto selection = selection_at(arrow, style); + if (!selection){ + return false; + } + const double rmsd = style.matcher.rmsd_original(screen.size(), region, object); + if (rmsd < best_rmsd){ + best_rmsd = rmsd; + best = selection; + } + return false; + } + ); + } + return best; +} - const bool expected_right = m_expected && *m_expected == SelectionArrowType::RIGHT; - const bool expected_down = m_expected && *m_expected == SelectionArrowType::DOWN; +SelectionArrowWatcher::SelectionArrowWatcher(Color color, BoxSelectionMode mode, BoxSelectionLocation location) + : SelectionArrowDetector(color) + , VisualInferenceCallback("SelectionArrowWatcher") + , m_mode(mode) + , m_location(location) +{} - if (right_detected == expected_right && down_detected == expected_down) - return true; +bool SelectionArrowWatcher::process_frame(const ImageViewRGB32& frame, WallClock timestamp){ + m_last_detected = detect(frame); + if (m_last_detected && m_last_detected->mode == m_mode && m_last_detected->location == m_location){ + return true; + } + m_last_detected.reset(); + return false; +} - auto arrow_name = [](std::optional arrow){ - if (!arrow) - return std::string("none"); - return *arrow == SelectionArrowType::RIGHT - ? std::string("RIGHT") - : std::string("DOWN"); - }; +class Test_SelectionArrowDetector : public UnitTest{ +public: + Test_SelectionArrowDetector(const std::string& image, std::optional expected) + : UnitTest("PokemonHome::SelectionArrowDetector - " + image) + , m_image(UNIT_TEST_RESOURCE_PATH() + image) + , m_expected(expected) + {} - std::optional result; - if (right_detected && down_detected){ - return "Expected: " + arrow_name(m_expected) + ", received: RIGHT and DOWN"; - }else if (right_detected){ - result = SelectionArrowType::RIGHT; - }else if (down_detected){ - result = SelectionArrowType::DOWN; + virtual UnitTestResult run( Logger& logger, CancellableScope& scope) const override{ + ImageRGB32 image(m_image); + const auto result = SelectionArrowDetector().detect(image); + if (result.has_value() != m_expected.has_value() + || (result && (result->mode != m_expected->mode || result->location != m_expected->location + || result->row != m_expected->row || result->column != m_expected->column))){ + return "Expected: " + selection_string(m_expected) + ", received: " + selection_string(result); } - - return "Expected: " + arrow_name(m_expected) + ", received: " + arrow_name(result); - }; + return true; + } private: - std::optional m_expected; + std::string m_image; + std::optional m_expected; }; void add_tests_SelectionArrowDetector(UnitTestDatabase& database){ + database.add("PokemonHome/BoxView/BoxView-1.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Grid, 0, 3}); + database.add("PokemonHome/BoxView/image.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Grid, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_Grid_Row0_Column0_01.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Grid, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_01.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_02.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_03.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_04.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_05.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_06.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_07.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_08.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_09.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_01.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_02.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_03.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_04.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_05.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_06.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_07.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_08.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_09.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_10.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_Newest30_01.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Newest30, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_Newest30_02.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Newest30, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_Newest30_03.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Newest30, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_01.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_02.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_03.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_04.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_05.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_06.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_07.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_08.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_09.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_10.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_11.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_12.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_13.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_14.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_15.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxSpaces_16.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_Newest30_01.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::Newest30, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_Newest30_02.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::Newest30, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxName_01.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxName, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxName_02.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxName, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxName_03.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxName, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxName_04.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxName, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_BoxName_05.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::BoxName, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_Grid_Row2_Column0_01.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::Grid, 2, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_Grid_Row2_Column0_02.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::Grid, 2, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_Grid_Row2_Column5_01.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::Grid, 2, 5}); + database.add("PokemonHome/BoxView/SelectionArrow_Relocate_Grid_Row2_Column5_02.png", BoxSelection{BoxSelectionMode::Relocate, BoxSelectionLocation::Grid, 2, 5}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_Grid_Row0_Column0_01.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::Grid, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_Grid_Row0_Column0_02.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::Grid, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_Grid_Row3_Column3_01.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::Grid, 3, 3}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_Grid_Row3_Column3_02.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::Grid, 3, 3}); + database.add("PokemonHome/BoxView/SelectionArrow_Multi_Grid_Row2_Column0_01.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::Grid, 2, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_Grid_Row3_Column0_01.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Grid, 3, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxSpaces_11.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxSpaces, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxName_01.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxName, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_BoxName_02.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::BoxName, 0, 0}); database.add("PokemonHome/SummaryScreen/annihilape_Regular.png", std::nullopt); database.add("PokemonHome/SummaryScreen/bidoof_Regular.png", std::nullopt); database.add("PokemonHome/SummaryScreen/bulbasaur_Regular.png", std::nullopt); diff --git a/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.h b/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.h index 21c17b5693..593ba3b01a 100644 --- a/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.h +++ b/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.h @@ -8,59 +8,66 @@ #define PokemonAutomation_PokemonHome_SelectionArrowDetector_H #include +#include "Common/Cpp/Color.h" #include "Common/Cpp/TestRunners/UnitTestDatabase.h" -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" -#include "CommonTools/VisualDetector.h" +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" namespace PokemonAutomation{ namespace NintendoSwitch{ namespace PokemonHome{ -enum class SelectionArrowType{ - RIGHT, - DOWN, +enum class BoxSelectionMode{ + Normal, // Red + Relocate, // Blue + Multi, // Green }; +enum class BoxSelectionLocation{ + Grid, + BoxName, + BoxSpaces, + Newest30, +}; + +struct BoxSelection{ + BoxSelectionMode mode; + BoxSelectionLocation location; + // Zero-based coordinates in the 5-row, 6-column grid. Only valid for Grid. + size_t row = 0; + size_t column = 0; +}; -class SelectionArrowDetector : public StaticScreenDetector{ +class SelectionArrowDetector{ public: - SelectionArrowDetector( - Color color, - VideoOverlay* overlay, - SelectionArrowType type, - const ImageFloatBox& box - ); + SelectionArrowDetector(Color color = COLOR_RED) : m_color(color) {} - const ImageFloatBox& last_detected() const{ return m_last_detected; } + void make_overlays(VideoOverlaySet& items) const; + // Returns no value when no arrow matches a supported box selection position. + std::optional detect(const ImageViewRGB32& screen) const; - virtual void make_overlays(VideoOverlaySet& items) const override; +private: + Color m_color; +}; - virtual bool detect(const ImageViewRGB32& screen) override; +class SelectionArrowWatcher : public SelectionArrowDetector, public VisualInferenceCallback{ +public: + SelectionArrowWatcher(Color color, BoxSelectionMode mode, BoxSelectionLocation location); -private: - friend class SelectionArrowWatcher; + virtual void make_overlays(VideoOverlaySet& items) const override{ + SelectionArrowDetector::make_overlays(items); + } - const Color m_color; - VideoOverlay* m_overlay; - const SelectionArrowType m_type; - const ImageFloatBox m_arrow_box; + using VisualInferenceCallback::process_frame; + virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override; - ImageFloatBox m_last_detected; - std::optional m_last_detected_box; -}; -class SelectionArrowWatcher : public DetectorToFinder{ -public: - SelectionArrowWatcher( - Color color, - VideoOverlay* overlay, - SelectionArrowType type, - const ImageFloatBox& box, - std::chrono::milliseconds hold_duration = std::chrono::milliseconds(250) - ) - : DetectorToFinder("SelectionArrowWatcher", hold_duration, color, overlay, type, box) - {} + // Contains the matching mode, location and grid coordinates after process_frame() succeeds. + const std::optional& last_detected() const{ return m_last_detected; } + +private: + BoxSelectionMode m_mode; + BoxSelectionLocation m_location; + std::optional m_last_detected; }; void add_tests_SelectionArrowDetector(UnitTestDatabase& database); diff --git a/SerialPrograms/Source/VideoGames/PokemonHome/Programs/TestPrograms/PokemonHome_TestDatabaseGenerator.cpp b/SerialPrograms/Source/VideoGames/PokemonHome/Programs/TestPrograms/PokemonHome_TestDatabaseGenerator.cpp index c9c10139f8..33b2cbf051 100644 --- a/SerialPrograms/Source/VideoGames/PokemonHome/Programs/TestPrograms/PokemonHome_TestDatabaseGenerator.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonHome/Programs/TestPrograms/PokemonHome_TestDatabaseGenerator.cpp @@ -84,11 +84,39 @@ TestDatabaseGenerator::TestDatabaseGenerator() namespace{ -[[nodiscard]] std::string optional_arrow_name(std::optional arrow){ +[[nodiscard]] std::string optional_arrow_name(const std::optional& arrow){ if (!arrow){ - return "none"; + return "std::nullopt"; } - return *arrow == SelectionArrowType::RIGHT ? "right" : "down"; + std::string mode; + switch (arrow->mode){ + case BoxSelectionMode::Normal: + mode = "Normal"; + break; + case BoxSelectionMode::Relocate: + mode = "Relocate"; + break; + case BoxSelectionMode::Multi: + mode = "Multi"; + break; + } + std::string location; + switch (arrow->location){ + case BoxSelectionLocation::Grid: + location = "Grid"; + break; + case BoxSelectionLocation::BoxName: + location = "BoxName"; + break; + case BoxSelectionLocation::BoxSpaces: + location = "BoxSpaces"; + break; + case BoxSelectionLocation::Newest30: + location = "Newest30"; + break; + } + return "BoxSelection{BoxSelectionMode::" + mode + ", BoxSelectionLocation::" + location + + ", " + std::to_string(arrow->row) + ", " + std::to_string(arrow->column) + "}"; } [[nodiscard]] std::string to_uppercase(std::string value){ @@ -162,7 +190,6 @@ void TestDatabaseGenerator::program( for (const Filesystem::Path& path : screenshots){ try{ ImageRGB32 image(path.string()); - ImageFloatBox arrow_box(0.021, 0.073, 0.47, 0.72); ImageFloatBox button_box(0.100, 0.956, 0.107, 0.041); SummaryReader summary_reader; @@ -177,8 +204,7 @@ void TestDatabaseGenerator::program( ShinyDetector shiny_detector(COLOR_RED, &env.console.overlay()); AlphaDetector alpha_detector(COLOR_RED, &env.console.overlay()); GigantamaxDetector gigantamax_detector(COLOR_RED, &env.console.overlay()); - SelectionArrowDetector right_arrow_detector(COLOR_RED, &env.console.overlay(), SelectionArrowType::RIGHT, arrow_box); - SelectionArrowDetector down_arrow_detector(COLOR_RED, &env.console.overlay(), SelectionArrowType::DOWN, arrow_box); + SelectionArrowDetector arrow_detector; ButtonDetector b_detector(COLOR_RED, ButtonType::ButtonB, button_box, &env.console.overlay()); ButtonDetector plus_detector(COLOR_RED, ButtonType::ButtonPlus, button_box, &env.console.overlay()); BallReader ball_reader(env.console); @@ -189,8 +215,7 @@ void TestDatabaseGenerator::program( const bool shiny = shiny_detector.detect(image); const bool alpha = alpha_detector.detect(image); const bool gigantamax = gigantamax_detector.detect(image); - const bool right_arrow = right_arrow_detector.detect(image); - const bool down_arrow = down_arrow_detector.detect(image); + const auto arrow = arrow_detector.detect(image); const bool button_b = b_detector.detect(image); const bool button_plus = plus_detector.detect(image); const std::string ball = ball_reader.read_ball(image); @@ -226,12 +251,6 @@ void TestDatabaseGenerator::program( } } - std::optional arrow; - if (right_arrow) - arrow = SelectionArrowType::RIGHT; - else if (down_arrow) - arrow = SelectionArrowType::DOWN; - const Filesystem::Path database_file = renamed_path.empty() ? path : Filesystem::Path(renamed_path); const std::string print_directory = (std::string)PRINT_DIRECTORY; const std::string database_path = print_directory.empty() @@ -313,7 +332,7 @@ void TestDatabaseGenerator::program( entry.selection_arrow = std::format( "database.add({}, {});", quote_string(database_path), - arrow ? "SelectionArrowType::" + to_uppercase(optional_arrow_name(arrow)) : "std::nullopt" + optional_arrow_name(arrow) ); entry.button = std::format( From 977f56caa352572b90a17a69940f5e7bacc05549 Mon Sep 17 00:00:00 2001 From: Dalton-V Date: Sun, 8 Nov 2026 13:46:52 -0600 Subject: [PATCH 2/2] Update test image name --- .../Inference/PokemonHome_SelectionArrowDetector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.cpp index 7884ce3993..c7aad61df6 100644 --- a/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SelectionArrowDetector.cpp @@ -202,7 +202,7 @@ class Test_SelectionArrowDetector : public UnitTest{ void add_tests_SelectionArrowDetector(UnitTestDatabase& database){ database.add("PokemonHome/BoxView/BoxView-1.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Grid, 0, 3}); - database.add("PokemonHome/BoxView/image.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Grid, 0, 0}); + database.add("PokemonHome/BoxView/SelectionArrow_Normal_Grid_Row0_Column0_02.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Grid, 0, 0}); database.add("PokemonHome/BoxView/SelectionArrow_Normal_Grid_Row0_Column0_01.png", BoxSelection{BoxSelectionMode::Normal, BoxSelectionLocation::Grid, 0, 0}); database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_01.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0}); database.add("PokemonHome/BoxView/SelectionArrow_Multi_BoxSpaces_02.png", BoxSelection{BoxSelectionMode::Multi, BoxSelectionLocation::BoxSpaces, 0, 0});