From 1d8d0260f1d4cf13867654750b28eefc6bd1ba0f Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 00:15:31 -0800 Subject: [PATCH 1/3] NintendoSwitch: read which game tile HOME has selected --- .../workflows/cpp-ci-serial-programs-base.yml | 4 +- .../ComputerPrograms_UnitTestRunner.cpp | 2 + .../NintendoSwitch_HomeTileReader.cpp | 249 ++++++++++++++++++ .../Inference/NintendoSwitch_HomeTileReader.h | 55 ++++ SerialPrograms/cmake/SourceFiles.cmake | 2 + 5 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index efb3c80a04..e321d2dc90 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -245,7 +245,9 @@ jobs: uses: actions/checkout@v7 if: inputs.run-tests with: - repository: 'PokemonAutomation/CommandLineTests' + # The fork's fixtures: upstream's plus the HomeTileReader frames (fork only until they land upstream). + repository: 'dayfine/CommandLineTests' + ref: '0195f3b0c6ffb15bf6df176cfd3a64d781346fcf' path: 'CommandLineTests' # The unit tests assume the downloadable resources they need are already downloaded diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 6cc20db2fc..7106155ace 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -34,6 +34,7 @@ #include "NintendoSwitch/Inference/NintendoSwitch_CheckOnlineDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h" #include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h" #include "NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h" #include "NintendoSwitch/Controllers/SysbotBase/SysbotBase_Connection.h" @@ -86,6 +87,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ Pokemon::add_tests_AutoStory(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); + NintendoSwitch::add_tests_HomeTileReader(ret); NintendoSwitch::add_tests_UpdatePopupDetector(ret); NintendoSwitch::Typewriter::add_tests_TypewriterTextMapping(ret); NintendoSwitch::PokemonBDSP::add_tests_DialogDetector(ret); diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp new file mode 100644 index 0000000000..b8913f5001 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp @@ -0,0 +1,249 @@ +/* Home Tile Reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/Images/ImageFilter.h" +#include "CommonTools/OCR/OCR_Routines.h" +#include "CommonTools/OCR/OCR_StringNormalization.h" +#include "Tests/ImageUnitTest.h" +#include "NintendoSwitch_HomeTileReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +namespace{ + +// The selected tile's frame lines, about 0.008 of the screen tall, just +// outside the tile art: strips across the screen that the art of the +// other tiles stays out of. +const ImageFloatBox FRAME_TOP_STRIP{0.0, 0.252, 1.0, 0.014}; +const ImageFloatBox FRAME_BOTTOM_STRIP{0.0, 0.627, 1.0, 0.014}; +constexpr double FRAME_TOP = 0.256; +constexpr double FRAME_BOTTOM = 0.637; +// A column crosses a frame line when this many of its pixels in the strip +// are the frame's cyan. +constexpr size_t MIN_LINE_PIXELS = 3; +// A tile frame is 0.214 of the screen wide. +constexpr double MIN_FRAME_WIDTH = 0.18; +constexpr double MAX_FRAME_WIDTH = 0.25; + +// The band above the row where HOME writes the selected game's name. The +// name starts 0.005 to 0.01 left of the frame, right of a game card icon +// that ends 0.017 or more left of it. +constexpr double NAME_TOP = 0.19; +constexpr double NAME_HEIGHT = 0.06; +constexpr double NAME_LEAD = 0.015; +constexpr double NAME_RIGHT = 0.98; +// The name's cyan (ARGB), dark text on HOME's light background once +// filtered: red low, green and blue high. +constexpr uint32_t NAME_MINS = 0xff000096; +constexpr uint32_t NAME_MAXS = 0xff96ffff; + +// The frame's cyan, which pulses between about (0, 187, 229) and +// (52, 249, 229), with lighter anti-aliased edges. +bool is_frame_cyan(uint32_t pixel){ + const int red = (pixel >> 16) & 0xff; + const int green = (pixel >> 8) & 0xff; + const int blue = pixel & 0xff; + return green >= 160 && blue >= 160 && green - red >= 60 && blue - red >= 60; +} + +// Per column of `strip`, whether it crosses a frame line. +std::vector frame_line_columns(const ImageViewRGB32& strip){ + std::vector columns(strip.width(), false); + for (size_t x = 0; x < strip.width(); x++){ + size_t cyan = 0; + for (size_t y = 0; y < strip.height(); y++){ + cyan += is_frame_cyan(strip.pixel(x, y)) ? 1 : 0; + } + columns[x] = cyan >= MIN_LINE_PIXELS; + } + return columns; +} + +// The longest run of columns that cross both frame lines, as [begin, end). +std::pair longest_framed_run(const std::vector& top, const std::vector& bottom){ + std::pair best{0, 0}; + size_t begin = 0; + for (size_t x = 0; x <= top.size(); x++){ + const bool framed = x < top.size() && x < bottom.size() && top[x] && bottom[x]; + if (framed){ + continue; + } + if (x - begin > best.second - best.first){ + best = {begin, x}; + } + begin = x + 1; + } + return best; +} + +} // namespace + + +std::optional find_selected_home_tile(const ImageViewRGB32& screen){ + if (screen.width() == 0 || screen.height() == 0){ + return std::nullopt; + } + const std::pair run = longest_framed_run( + frame_line_columns(extract_box_reference(screen, FRAME_TOP_STRIP)), + frame_line_columns(extract_box_reference(screen, FRAME_BOTTOM_STRIP)) + ); + const double width = (double)screen.width(); + const double frame_width = (double)(run.second - run.first) / width; + if (frame_width < MIN_FRAME_WIDTH || frame_width > MAX_FRAME_WIDTH){ + return std::nullopt; + } + return ImageFloatBox((double)run.first / width, FRAME_TOP, frame_width, FRAME_BOTTOM - FRAME_TOP); +} + +std::optional read_selected_home_tile_name(Language language, const ImageViewRGB32& screen){ + const std::optional frame = find_selected_home_tile(screen); + if (!frame){ + return std::nullopt; + } + const double left = std::max(0.0, frame->x - NAME_LEAD); + const ImageViewRGB32 band = extract_box_reference(screen, ImageFloatBox(left, NAME_TOP, NAME_RIGHT - left, NAME_HEIGHT)); + const ImageRGB32 text = to_blackwhite_rgb32_range(band, true, NAME_MINS, NAME_MAXS); + return OCR::ocr_read(language, text); +} + +bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std::string_view name){ + const std::optional read = read_selected_home_tile_name(language, screen); + if (!read){ + return false; + } + const std::u32string wanted = OCR::normalize_utf32(std::string(name)); + return !wanted.empty() && OCR::normalize_utf32(*read).find(wanted) != std::u32string::npos; +} + + + +namespace{ + +// The names HOME shows for the two games on console B's frames. +constexpr std::string_view FIRERED = "Pokémon FireRed Version"; +constexpr std::string_view LEGENDS_ZA = "Pokémon Legends: Z-A"; + +// Where the first tile's frame starts, and the step to the next tile, as +// parts of the screen's width, before the row scrolls. +constexpr double FIRST_TILE_LEFT = 0.076; +constexpr double TILE_STEP = 0.211; +constexpr double TILE_TOLERANCE = 0.02; + +struct HomeTileCase{ + struct Input{ + std::string_view image; + }; + struct Expectation{ + // The selected tile's place in the row, from 0; nothing: no tile + // is selected. + std::optional tile = std::nullopt; + // The name the selected tile shows: FIRERED or LEGENDS_ZA. + std::string_view game = {}; + }; + std::string_view name; + Input input; + Expectation expectation; +}; + +// Console B, 2026-11-14: HOME after the order changed (the game played +// last moves to the first tile), and the user select over HOME. +const std::vector HOME_TILE_CASES = { + {.name = "FireRed on the first tile 1", + .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_1.png"}, + .expectation = {.tile = 0, .game = FIRERED}}, + {.name = "FireRed on the first tile 2", + .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_2.png"}, + .expectation = {.tile = 0, .game = FIRERED}}, + {.name = "FireRed on the first tile 3", + .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_3.png"}, + .expectation = {.tile = 0, .game = FIRERED}}, + {.name = "Legends Z-A on the first tile", + .input = {.image = "NintendoSwitch/HomeTileReader/LegendsZAFirstTile.png"}, + .expectation = {.tile = 0, .game = LEGENDS_ZA}}, + {.name = "Legends Z-A on the second tile", + .input = {.image = "NintendoSwitch/HomeTileReader/LegendsZASecondTile.png"}, + .expectation = {.tile = 1, .game = LEGENDS_ZA}}, + {.name = "user select over HOME", + .input = {.image = "NintendoSwitch/HomeTileReader/UserSelectOverHome.png"}, + .expectation = {}}, +}; + +// What went wrong with the frame on `screen`; empty when it is where `test` expects. +std::string check_frame(const HomeTileCase& test, const ImageViewRGB32& screen){ + const std::optional frame = find_selected_home_tile(screen); + if (!test.expectation.tile){ + return frame ? std::format("Expected no selected tile, found a frame at x={:.3f}.", frame->x) : ""; + } + if (!frame){ + return std::format("Expected a frame on tile {}, found none.", *test.expectation.tile); + } + const double expected_left = FIRST_TILE_LEFT + TILE_STEP * (double)*test.expectation.tile; + if (std::abs(frame->x - expected_left) > TILE_TOLERANCE){ + return std::format("Expected the frame on tile {} (x={:.3f}), found it at x={:.3f}.", *test.expectation.tile, expected_left, frame->x); + } + return ""; +} + +// What went wrong with the names on `screen`: it must be the game `test` +// expects and not the other one. +std::string check_names(const HomeTileCase& test, const ImageViewRGB32& screen){ + const std::string read = read_selected_home_tile_name(Language::English, screen).value_or(""); + for (std::string_view game : {FIRERED, LEGENDS_ZA}){ + const bool expected = game == test.expectation.game; + if (selected_home_tile_is(Language::English, screen, game) != expected){ + return std::format("Expected the tile {} \"{}\", read \"{}\".", expected ? "to be" : "not to be", game, read); + } + } + return ""; +} + +class Test_HomeTileReader : public ImageUnitTest{ +public: + explicit Test_HomeTileReader(const HomeTileCase& test) + : ImageUnitTest(std::format("NintendoSwitch::HomeTileReader - {}", test.name), {std::string(test.input.image)}) + , m_case(test) + { + // Plus the PaddleOCR (ONNX) recognition session and its working set. + m_memory += PADDLE_OCR_SINGLE_LINE_MEMORY_ESTIMATE; + } + + virtual UnitTestResult run_on_images( + Logger&, CancellableScope&, + const std::vector& images + ) const override{ + std::string error = check_frame(m_case, images[0]); + if (error.empty()){ + error = check_names(m_case, images[0]); + } + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } + +private: + HomeTileCase m_case; +}; + +} // namespace + + +void add_tests_HomeTileReader(UnitTestDatabase& database){ + for (const HomeTileCase& test : HOME_TILE_CASES){ + database.add(test); + } +} + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h new file mode 100644 index 0000000000..c0c57f1052 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h @@ -0,0 +1,55 @@ +/* Home Tile Reader + * + * From: https://github.com/PokemonAutomation/ + * + * Which game tile the HOME menu's cursor is on. HOME draws a cyan frame + * around the selected tile of the game row and writes that game's name + * above the row, also in cyan. The frame says that a tile is selected and + * where; the name says which game it is. Neither depends on the order of + * the row, which changes as games are played (the last one played moves + * to the first tile). + * + * Measured on the Switch 1 HOME menu at 1920x1080; positions are parts of + * the screen, so other capture sizes read the same. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_HomeTileReader_H +#define PokemonAutomation_NintendoSwitch_HomeTileReader_H + +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "CommonFramework/Language.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// The selected tile's cyan frame on `screen`; nothing when no tile of the +// game row is selected (the cursor on the icons below it, a dialog or the +// user select over HOME). +std::optional find_selected_home_tile(const ImageViewRGB32& screen); + +// The name HOME writes above the selected tile, as OCR reads it in +// `language` (the console's, not the game's); nothing when no tile is +// selected. +std::optional read_selected_home_tile_name(Language language, const ImageViewRGB32& screen); + +// Whether a tile is selected and its name contains `name` ("Pokémon +// Legends: Z-A", or a distinctive part of it), compared as OCR text: +// case, accents, spaces and punctuation ignored. +bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std::string_view name); + + +// The screenshot tests in CommandLineTests/NintendoSwitch/HomeTileReader/. +void add_tests_HomeTileReader(UnitTestDatabase& database); + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index df9a705d19..8887df0042 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1374,6 +1374,8 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h + Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp + Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.cpp From 33bdfb5d4f7e30c8fb654560e048edbf7d951eb6 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 01:05:11 -0800 Subject: [PATCH 2/3] HomeTileReader: CI keeps upstream's fixtures; comments describe the screens The workflow change that pointed every suite at a fork of CommandLineTests is dropped; the HomeTileReader frames come from the separate CommandLineTests change. The colour filter's comment says what it keeps, and the test comments describe the HOME screens instead of our console and date. --- .github/workflows/cpp-ci-serial-programs-base.yml | 4 +--- .../Inference/NintendoSwitch_HomeTileReader.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index e321d2dc90..efb3c80a04 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -245,9 +245,7 @@ jobs: uses: actions/checkout@v7 if: inputs.run-tests with: - # The fork's fixtures: upstream's plus the HomeTileReader frames (fork only until they land upstream). - repository: 'dayfine/CommandLineTests' - ref: '0195f3b0c6ffb15bf6df176cfd3a64d781346fcf' + repository: 'PokemonAutomation/CommandLineTests' path: 'CommandLineTests' # The unit tests assume the downloadable resources they need are already downloaded diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp index b8913f5001..8ff4cbfae6 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp @@ -45,7 +45,8 @@ constexpr double NAME_HEIGHT = 0.06; constexpr double NAME_LEAD = 0.015; constexpr double NAME_RIGHT = 0.98; // The name's cyan (ARGB), dark text on HOME's light background once -// filtered: red low, green and blue high. +// filtered: red at most 150 and blue at least 150, any green. The name's +// blue-cyan text passes; the white tile and the grey background do not. constexpr uint32_t NAME_MINS = 0xff000096; constexpr uint32_t NAME_MAXS = 0xff96ffff; @@ -131,7 +132,7 @@ bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std: namespace{ -// The names HOME shows for the two games on console B's frames. +// The names HOME shows for the two games in the test frames. constexpr std::string_view FIRERED = "Pokémon FireRed Version"; constexpr std::string_view LEGENDS_ZA = "Pokémon Legends: Z-A"; @@ -157,8 +158,8 @@ struct HomeTileCase{ Expectation expectation; }; -// Console B, 2026-11-14: HOME after the order changed (the game played -// last moves to the first tile), and the user select over HOME. +// HOME after the order changed (the game played last moves to the first +// tile), and the user select over HOME. const std::vector HOME_TILE_CASES = { {.name = "FireRed on the first tile 1", .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_1.png"}, From 6aa80a7892a2216bb17ade7fb16ed4ef0a93d0fa Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 12:26:52 -0800 Subject: [PATCH 3/3] Drop HomeTileReader: PLZA reloads with reset_game_from_home() --- .../ComputerPrograms_UnitTestRunner.cpp | 2 - .../NintendoSwitch_HomeTileReader.cpp | 250 ------------------ .../Inference/NintendoSwitch_HomeTileReader.h | 55 ---- SerialPrograms/cmake/SourceFiles.cmake | 2 - 4 files changed, 309 deletions(-) delete mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp delete mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index fb6409cf43..af5fe356e2 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -35,7 +35,6 @@ #include "NintendoSwitch/Inference/NintendoSwitch_CheckOnlineDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h" -#include "NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h" #include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h" #include "NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h" #include "NintendoSwitch/Controllers/SysbotBase/SysbotBase_Connection.h" @@ -89,7 +88,6 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ Pokemon::add_tests_AutoStory(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); - NintendoSwitch::add_tests_HomeTileReader(ret); NintendoSwitch::add_tests_UpdatePopupDetector(ret); NintendoSwitch::Typewriter::add_tests_TypewriterTextMapping(ret); NintendoSwitch::PokemonBDSP::add_tests_DialogDetector(ret); diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp deleted file mode 100644 index 8ff4cbfae6..0000000000 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp +++ /dev/null @@ -1,250 +0,0 @@ -/* Home Tile Reader - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include -#include -#include -#include -#include -#include "CommonFramework/ImageTypes/ImageRGB32.h" -#include "CommonTools/Images/ImageFilter.h" -#include "CommonTools/OCR/OCR_Routines.h" -#include "CommonTools/OCR/OCR_StringNormalization.h" -#include "Tests/ImageUnitTest.h" -#include "NintendoSwitch_HomeTileReader.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ - - -namespace{ - -// The selected tile's frame lines, about 0.008 of the screen tall, just -// outside the tile art: strips across the screen that the art of the -// other tiles stays out of. -const ImageFloatBox FRAME_TOP_STRIP{0.0, 0.252, 1.0, 0.014}; -const ImageFloatBox FRAME_BOTTOM_STRIP{0.0, 0.627, 1.0, 0.014}; -constexpr double FRAME_TOP = 0.256; -constexpr double FRAME_BOTTOM = 0.637; -// A column crosses a frame line when this many of its pixels in the strip -// are the frame's cyan. -constexpr size_t MIN_LINE_PIXELS = 3; -// A tile frame is 0.214 of the screen wide. -constexpr double MIN_FRAME_WIDTH = 0.18; -constexpr double MAX_FRAME_WIDTH = 0.25; - -// The band above the row where HOME writes the selected game's name. The -// name starts 0.005 to 0.01 left of the frame, right of a game card icon -// that ends 0.017 or more left of it. -constexpr double NAME_TOP = 0.19; -constexpr double NAME_HEIGHT = 0.06; -constexpr double NAME_LEAD = 0.015; -constexpr double NAME_RIGHT = 0.98; -// The name's cyan (ARGB), dark text on HOME's light background once -// filtered: red at most 150 and blue at least 150, any green. The name's -// blue-cyan text passes; the white tile and the grey background do not. -constexpr uint32_t NAME_MINS = 0xff000096; -constexpr uint32_t NAME_MAXS = 0xff96ffff; - -// The frame's cyan, which pulses between about (0, 187, 229) and -// (52, 249, 229), with lighter anti-aliased edges. -bool is_frame_cyan(uint32_t pixel){ - const int red = (pixel >> 16) & 0xff; - const int green = (pixel >> 8) & 0xff; - const int blue = pixel & 0xff; - return green >= 160 && blue >= 160 && green - red >= 60 && blue - red >= 60; -} - -// Per column of `strip`, whether it crosses a frame line. -std::vector frame_line_columns(const ImageViewRGB32& strip){ - std::vector columns(strip.width(), false); - for (size_t x = 0; x < strip.width(); x++){ - size_t cyan = 0; - for (size_t y = 0; y < strip.height(); y++){ - cyan += is_frame_cyan(strip.pixel(x, y)) ? 1 : 0; - } - columns[x] = cyan >= MIN_LINE_PIXELS; - } - return columns; -} - -// The longest run of columns that cross both frame lines, as [begin, end). -std::pair longest_framed_run(const std::vector& top, const std::vector& bottom){ - std::pair best{0, 0}; - size_t begin = 0; - for (size_t x = 0; x <= top.size(); x++){ - const bool framed = x < top.size() && x < bottom.size() && top[x] && bottom[x]; - if (framed){ - continue; - } - if (x - begin > best.second - best.first){ - best = {begin, x}; - } - begin = x + 1; - } - return best; -} - -} // namespace - - -std::optional find_selected_home_tile(const ImageViewRGB32& screen){ - if (screen.width() == 0 || screen.height() == 0){ - return std::nullopt; - } - const std::pair run = longest_framed_run( - frame_line_columns(extract_box_reference(screen, FRAME_TOP_STRIP)), - frame_line_columns(extract_box_reference(screen, FRAME_BOTTOM_STRIP)) - ); - const double width = (double)screen.width(); - const double frame_width = (double)(run.second - run.first) / width; - if (frame_width < MIN_FRAME_WIDTH || frame_width > MAX_FRAME_WIDTH){ - return std::nullopt; - } - return ImageFloatBox((double)run.first / width, FRAME_TOP, frame_width, FRAME_BOTTOM - FRAME_TOP); -} - -std::optional read_selected_home_tile_name(Language language, const ImageViewRGB32& screen){ - const std::optional frame = find_selected_home_tile(screen); - if (!frame){ - return std::nullopt; - } - const double left = std::max(0.0, frame->x - NAME_LEAD); - const ImageViewRGB32 band = extract_box_reference(screen, ImageFloatBox(left, NAME_TOP, NAME_RIGHT - left, NAME_HEIGHT)); - const ImageRGB32 text = to_blackwhite_rgb32_range(band, true, NAME_MINS, NAME_MAXS); - return OCR::ocr_read(language, text); -} - -bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std::string_view name){ - const std::optional read = read_selected_home_tile_name(language, screen); - if (!read){ - return false; - } - const std::u32string wanted = OCR::normalize_utf32(std::string(name)); - return !wanted.empty() && OCR::normalize_utf32(*read).find(wanted) != std::u32string::npos; -} - - - -namespace{ - -// The names HOME shows for the two games in the test frames. -constexpr std::string_view FIRERED = "Pokémon FireRed Version"; -constexpr std::string_view LEGENDS_ZA = "Pokémon Legends: Z-A"; - -// Where the first tile's frame starts, and the step to the next tile, as -// parts of the screen's width, before the row scrolls. -constexpr double FIRST_TILE_LEFT = 0.076; -constexpr double TILE_STEP = 0.211; -constexpr double TILE_TOLERANCE = 0.02; - -struct HomeTileCase{ - struct Input{ - std::string_view image; - }; - struct Expectation{ - // The selected tile's place in the row, from 0; nothing: no tile - // is selected. - std::optional tile = std::nullopt; - // The name the selected tile shows: FIRERED or LEGENDS_ZA. - std::string_view game = {}; - }; - std::string_view name; - Input input; - Expectation expectation; -}; - -// HOME after the order changed (the game played last moves to the first -// tile), and the user select over HOME. -const std::vector HOME_TILE_CASES = { - {.name = "FireRed on the first tile 1", - .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_1.png"}, - .expectation = {.tile = 0, .game = FIRERED}}, - {.name = "FireRed on the first tile 2", - .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_2.png"}, - .expectation = {.tile = 0, .game = FIRERED}}, - {.name = "FireRed on the first tile 3", - .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_3.png"}, - .expectation = {.tile = 0, .game = FIRERED}}, - {.name = "Legends Z-A on the first tile", - .input = {.image = "NintendoSwitch/HomeTileReader/LegendsZAFirstTile.png"}, - .expectation = {.tile = 0, .game = LEGENDS_ZA}}, - {.name = "Legends Z-A on the second tile", - .input = {.image = "NintendoSwitch/HomeTileReader/LegendsZASecondTile.png"}, - .expectation = {.tile = 1, .game = LEGENDS_ZA}}, - {.name = "user select over HOME", - .input = {.image = "NintendoSwitch/HomeTileReader/UserSelectOverHome.png"}, - .expectation = {}}, -}; - -// What went wrong with the frame on `screen`; empty when it is where `test` expects. -std::string check_frame(const HomeTileCase& test, const ImageViewRGB32& screen){ - const std::optional frame = find_selected_home_tile(screen); - if (!test.expectation.tile){ - return frame ? std::format("Expected no selected tile, found a frame at x={:.3f}.", frame->x) : ""; - } - if (!frame){ - return std::format("Expected a frame on tile {}, found none.", *test.expectation.tile); - } - const double expected_left = FIRST_TILE_LEFT + TILE_STEP * (double)*test.expectation.tile; - if (std::abs(frame->x - expected_left) > TILE_TOLERANCE){ - return std::format("Expected the frame on tile {} (x={:.3f}), found it at x={:.3f}.", *test.expectation.tile, expected_left, frame->x); - } - return ""; -} - -// What went wrong with the names on `screen`: it must be the game `test` -// expects and not the other one. -std::string check_names(const HomeTileCase& test, const ImageViewRGB32& screen){ - const std::string read = read_selected_home_tile_name(Language::English, screen).value_or(""); - for (std::string_view game : {FIRERED, LEGENDS_ZA}){ - const bool expected = game == test.expectation.game; - if (selected_home_tile_is(Language::English, screen, game) != expected){ - return std::format("Expected the tile {} \"{}\", read \"{}\".", expected ? "to be" : "not to be", game, read); - } - } - return ""; -} - -class Test_HomeTileReader : public ImageUnitTest{ -public: - explicit Test_HomeTileReader(const HomeTileCase& test) - : ImageUnitTest(std::format("NintendoSwitch::HomeTileReader - {}", test.name), {std::string(test.input.image)}) - , m_case(test) - { - // Plus the PaddleOCR (ONNX) recognition session and its working set. - m_memory += PADDLE_OCR_SINGLE_LINE_MEMORY_ESTIMATE; - } - - virtual UnitTestResult run_on_images( - Logger&, CancellableScope&, - const std::vector& images - ) const override{ - std::string error = check_frame(m_case, images[0]); - if (error.empty()){ - error = check_names(m_case, images[0]); - } - return error.empty() ? UnitTestResult(true) : UnitTestResult(error); - } - -private: - HomeTileCase m_case; -}; - -} // namespace - - -void add_tests_HomeTileReader(UnitTestDatabase& database){ - for (const HomeTileCase& test : HOME_TILE_CASES){ - database.add(test); - } -} - - - -} // namespace NintendoSwitch -} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h deleted file mode 100644 index c0c57f1052..0000000000 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h +++ /dev/null @@ -1,55 +0,0 @@ -/* Home Tile Reader - * - * From: https://github.com/PokemonAutomation/ - * - * Which game tile the HOME menu's cursor is on. HOME draws a cyan frame - * around the selected tile of the game row and writes that game's name - * above the row, also in cyan. The frame says that a tile is selected and - * where; the name says which game it is. Neither depends on the order of - * the row, which changes as games are played (the last one played moves - * to the first tile). - * - * Measured on the Switch 1 HOME menu at 1920x1080; positions are parts of - * the screen, so other capture sizes read the same. - * - */ - -#ifndef PokemonAutomation_NintendoSwitch_HomeTileReader_H -#define PokemonAutomation_NintendoSwitch_HomeTileReader_H - -#include -#include -#include -#include "Common/Cpp/TestRunners/UnitTestDatabase.h" -#include "CommonFramework/Language.h" -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTypes/ImageViewRGB32.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ - - -// The selected tile's cyan frame on `screen`; nothing when no tile of the -// game row is selected (the cursor on the icons below it, a dialog or the -// user select over HOME). -std::optional find_selected_home_tile(const ImageViewRGB32& screen); - -// The name HOME writes above the selected tile, as OCR reads it in -// `language` (the console's, not the game's); nothing when no tile is -// selected. -std::optional read_selected_home_tile_name(Language language, const ImageViewRGB32& screen); - -// Whether a tile is selected and its name contains `name` ("Pokémon -// Legends: Z-A", or a distinctive part of it), compared as OCR text: -// case, accents, spaces and punctuation ignored. -bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std::string_view name); - - -// The screenshot tests in CommandLineTests/NintendoSwitch/HomeTileReader/. -void add_tests_HomeTileReader(UnitTestDatabase& database); - - - -} // namespace NintendoSwitch -} // namespace PokemonAutomation -#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index cdd0cae870..f1e4071700 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1376,8 +1376,6 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h - Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp - Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.cpp