Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/cpp-ci-serial-programs-base.yml
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,9 @@ jobs:
uses: actions/checkout@v7
if: inputs.run-tests
with:
repository: 'PokemonAutomation/CommandLineTests'
# The fork's fixtures: upstream's plus the HomeTileReader frames (fork only until they land upstream).
repository: 'dayfine/CommandLineTests'
ref: '0195f3b0c6ffb15bf6df176cfd3a64d781346fcf'
path: 'CommandLineTests'

# The unit tests assume the downloadable resources they need are already downloaded
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

#include "NintendoSwitch/Inference/NintendoSwitch_CheckOnlineDetector.h"
#include "NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h"
#include "NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h"
#include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h"
#include "NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h"
#include "NintendoSwitch/Controllers/SysbotBase/SysbotBase_Connection.h"
Expand All @@ -58,7 +59,7 @@



UnitTestDatabase make_UNIT_TESTS_ALL(){

Check warning on line 62 in SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ComputerPrograms::make_UNIT_TESTS_ALL(): function_lines=43 (advisory above 25, limit 50)
UnitTestDatabase ret;

add_tests_QtEventThreadPool(ret, GlobalThreadPools::computation_normal());
Expand Down Expand Up @@ -86,6 +87,7 @@
Pokemon::add_tests_AutoStory(ret);
NintendoSwitch::add_tests_CheckOnlineDetector(ret);
NintendoSwitch::add_tests_FailedToConnectDetector(ret);
NintendoSwitch::add_tests_HomeTileReader(ret);
NintendoSwitch::add_tests_UpdatePopupDetector(ret);
NintendoSwitch::Typewriter::add_tests_TypewriterTextMapping(ret);
NintendoSwitch::PokemonBDSP::add_tests_DialogDetector(ret);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
/* Home Tile Reader
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <format>
#include <utility>
#include <vector>
#include "CommonFramework/ImageTypes/ImageRGB32.h"
#include "CommonTools/Images/ImageFilter.h"
#include "CommonTools/OCR/OCR_Routines.h"
#include "CommonTools/OCR/OCR_StringNormalization.h"
#include "Tests/ImageUnitTest.h"
#include "NintendoSwitch_HomeTileReader.h"

namespace PokemonAutomation{
namespace NintendoSwitch{


namespace{

// The selected tile's frame lines, about 0.008 of the screen tall, just
// outside the tile art: strips across the screen that the art of the
// other tiles stays out of.
const ImageFloatBox FRAME_TOP_STRIP{0.0, 0.252, 1.0, 0.014};
const ImageFloatBox FRAME_BOTTOM_STRIP{0.0, 0.627, 1.0, 0.014};
constexpr double FRAME_TOP = 0.256;
constexpr double FRAME_BOTTOM = 0.637;
// A column crosses a frame line when this many of its pixels in the strip
// are the frame's cyan.
constexpr size_t MIN_LINE_PIXELS = 3;
// A tile frame is 0.214 of the screen wide.
constexpr double MIN_FRAME_WIDTH = 0.18;
constexpr double MAX_FRAME_WIDTH = 0.25;

// The band above the row where HOME writes the selected game's name. The
// name starts 0.005 to 0.01 left of the frame, right of a game card icon
// that ends 0.017 or more left of it.
constexpr double NAME_TOP = 0.19;
constexpr double NAME_HEIGHT = 0.06;
constexpr double NAME_LEAD = 0.015;
constexpr double NAME_RIGHT = 0.98;
// The name's cyan (ARGB), dark text on HOME's light background once
// filtered: red low, green and blue high.
constexpr uint32_t NAME_MINS = 0xff000096;
constexpr uint32_t NAME_MAXS = 0xff96ffff;

// The frame's cyan, which pulses between about (0, 187, 229) and
// (52, 249, 229), with lighter anti-aliased edges.
bool is_frame_cyan(uint32_t pixel){
const int red = (pixel >> 16) & 0xff;
const int green = (pixel >> 8) & 0xff;
const int blue = pixel & 0xff;
return green >= 160 && blue >= 160 && green - red >= 60 && blue - red >= 60;
}

// Per column of `strip`, whether it crosses a frame line.
std::vector<bool> frame_line_columns(const ImageViewRGB32& strip){
std::vector<bool> columns(strip.width(), false);
for (size_t x = 0; x < strip.width(); x++){
size_t cyan = 0;
for (size_t y = 0; y < strip.height(); y++){
cyan += is_frame_cyan(strip.pixel(x, y)) ? 1 : 0;
}
columns[x] = cyan >= MIN_LINE_PIXELS;
}
return columns;
}

// The longest run of columns that cross both frame lines, as [begin, end).
std::pair<size_t, size_t> longest_framed_run(const std::vector<bool>& top, const std::vector<bool>& bottom){
std::pair<size_t, size_t> best{0, 0};
size_t begin = 0;
for (size_t x = 0; x <= top.size(); x++){
const bool framed = x < top.size() && x < bottom.size() && top[x] && bottom[x];
if (framed){
continue;
}
if (x - begin > best.second - best.first){
best = {begin, x};
}
begin = x + 1;
}
return best;
}

} // namespace


std::optional<ImageFloatBox> find_selected_home_tile(const ImageViewRGB32& screen){
if (screen.width() == 0 || screen.height() == 0){
return std::nullopt;
}
const std::pair<size_t, size_t> run = longest_framed_run(
frame_line_columns(extract_box_reference(screen, FRAME_TOP_STRIP)),
frame_line_columns(extract_box_reference(screen, FRAME_BOTTOM_STRIP))
);
const double width = (double)screen.width();
const double frame_width = (double)(run.second - run.first) / width;
if (frame_width < MIN_FRAME_WIDTH || frame_width > MAX_FRAME_WIDTH){
return std::nullopt;
}
return ImageFloatBox((double)run.first / width, FRAME_TOP, frame_width, FRAME_BOTTOM - FRAME_TOP);
}

std::optional<std::string> read_selected_home_tile_name(Language language, const ImageViewRGB32& screen){
const std::optional<ImageFloatBox> frame = find_selected_home_tile(screen);
if (!frame){
return std::nullopt;
}
const double left = std::max(0.0, frame->x - NAME_LEAD);
const ImageViewRGB32 band = extract_box_reference(screen, ImageFloatBox(left, NAME_TOP, NAME_RIGHT - left, NAME_HEIGHT));
const ImageRGB32 text = to_blackwhite_rgb32_range(band, true, NAME_MINS, NAME_MAXS);
return OCR::ocr_read(language, text);
}

bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std::string_view name){
const std::optional<std::string> read = read_selected_home_tile_name(language, screen);
if (!read){
return false;
}
const std::u32string wanted = OCR::normalize_utf32(std::string(name));
return !wanted.empty() && OCR::normalize_utf32(*read).find(wanted) != std::u32string::npos;
}



namespace{

// The names HOME shows for the two games on console B's frames.
constexpr std::string_view FIRERED = "Pokémon FireRed Version";
constexpr std::string_view LEGENDS_ZA = "Pokémon Legends: Z-A";

// Where the first tile's frame starts, and the step to the next tile, as
// parts of the screen's width, before the row scrolls.
constexpr double FIRST_TILE_LEFT = 0.076;
constexpr double TILE_STEP = 0.211;
constexpr double TILE_TOLERANCE = 0.02;

struct HomeTileCase{
struct Input{
std::string_view image;
};
struct Expectation{
// The selected tile's place in the row, from 0; nothing: no tile
// is selected.
std::optional<size_t> tile = std::nullopt;
// The name the selected tile shows: FIRERED or LEGENDS_ZA.
std::string_view game = {};
};
std::string_view name;
Input input;
Expectation expectation;
};

// Console B, 2026-11-14: HOME after the order changed (the game played
// last moves to the first tile), and the user select over HOME.
const std::vector<HomeTileCase> HOME_TILE_CASES = {
{.name = "FireRed on the first tile 1",
.input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_1.png"},
.expectation = {.tile = 0, .game = FIRERED}},
{.name = "FireRed on the first tile 2",
.input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_2.png"},
.expectation = {.tile = 0, .game = FIRERED}},
{.name = "FireRed on the first tile 3",
.input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_3.png"},
.expectation = {.tile = 0, .game = FIRERED}},
{.name = "Legends Z-A on the first tile",
.input = {.image = "NintendoSwitch/HomeTileReader/LegendsZAFirstTile.png"},
.expectation = {.tile = 0, .game = LEGENDS_ZA}},
{.name = "Legends Z-A on the second tile",
.input = {.image = "NintendoSwitch/HomeTileReader/LegendsZASecondTile.png"},
.expectation = {.tile = 1, .game = LEGENDS_ZA}},
{.name = "user select over HOME",
.input = {.image = "NintendoSwitch/HomeTileReader/UserSelectOverHome.png"},
.expectation = {}},
};

// What went wrong with the frame on `screen`; empty when it is where `test` expects.
std::string check_frame(const HomeTileCase& test, const ImageViewRGB32& screen){
const std::optional<ImageFloatBox> frame = find_selected_home_tile(screen);
if (!test.expectation.tile){
return frame ? std::format("Expected no selected tile, found a frame at x={:.3f}.", frame->x) : "";
}
if (!frame){
return std::format("Expected a frame on tile {}, found none.", *test.expectation.tile);
}
const double expected_left = FIRST_TILE_LEFT + TILE_STEP * (double)*test.expectation.tile;
if (std::abs(frame->x - expected_left) > TILE_TOLERANCE){
return std::format("Expected the frame on tile {} (x={:.3f}), found it at x={:.3f}.", *test.expectation.tile, expected_left, frame->x);
}
return "";
}

// What went wrong with the names on `screen`: it must be the game `test`
// expects and not the other one.
std::string check_names(const HomeTileCase& test, const ImageViewRGB32& screen){
const std::string read = read_selected_home_tile_name(Language::English, screen).value_or("<no tile>");
for (std::string_view game : {FIRERED, LEGENDS_ZA}){
const bool expected = game == test.expectation.game;
if (selected_home_tile_is(Language::English, screen, game) != expected){
return std::format("Expected the tile {} \"{}\", read \"{}\".", expected ? "to be" : "not to be", game, read);
}
}
return "";
}

class Test_HomeTileReader : public ImageUnitTest{
public:
explicit Test_HomeTileReader(const HomeTileCase& test)
: ImageUnitTest(std::format("NintendoSwitch::HomeTileReader - {}", test.name), {std::string(test.input.image)})
, m_case(test)
{
// Plus the PaddleOCR (ONNX) recognition session and its working set.
m_memory += PADDLE_OCR_SINGLE_LINE_MEMORY_ESTIMATE;
}

virtual UnitTestResult run_on_images(
Logger&, CancellableScope&,
const std::vector<ImageRGB32>& images
) const override{
std::string error = check_frame(m_case, images[0]);
if (error.empty()){
error = check_names(m_case, images[0]);
}
return error.empty() ? UnitTestResult(true) : UnitTestResult(error);
}

private:
HomeTileCase m_case;
};

} // namespace


void add_tests_HomeTileReader(UnitTestDatabase& database){
for (const HomeTileCase& test : HOME_TILE_CASES){
database.add<Test_HomeTileReader>(test);
}
}



} // namespace NintendoSwitch
} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/* Home Tile Reader
*
* From: https://github.com/PokemonAutomation/
*
* Which game tile the HOME menu's cursor is on. HOME draws a cyan frame
* around the selected tile of the game row and writes that game's name
* above the row, also in cyan. The frame says that a tile is selected and
* where; the name says which game it is. Neither depends on the order of
* the row, which changes as games are played (the last one played moves
* to the first tile).
*
* Measured on the Switch 1 HOME menu at 1920x1080; positions are parts of
* the screen, so other capture sizes read the same.
*
*/

#ifndef PokemonAutomation_NintendoSwitch_HomeTileReader_H
#define PokemonAutomation_NintendoSwitch_HomeTileReader_H

#include <optional>
#include <string>
#include <string_view>
#include "Common/Cpp/TestRunners/UnitTestDatabase.h"
#include "CommonFramework/Language.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"

namespace PokemonAutomation{
namespace NintendoSwitch{


// The selected tile's cyan frame on `screen`; nothing when no tile of the
// game row is selected (the cursor on the icons below it, a dialog or the
// user select over HOME).
std::optional<ImageFloatBox> find_selected_home_tile(const ImageViewRGB32& screen);

// The name HOME writes above the selected tile, as OCR reads it in
// `language` (the console's, not the game's); nothing when no tile is
// selected.
std::optional<std::string> read_selected_home_tile_name(Language language, const ImageViewRGB32& screen);

// Whether a tile is selected and its name contains `name` ("Pokémon
// Legends: Z-A", or a distinctive part of it), compared as OCR text:
// case, accents, spaces and punctuation ignored.
bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std::string_view name);


// The screenshot tests in CommandLineTests/NintendoSwitch/HomeTileReader/.
void add_tests_HomeTileReader(UnitTestDatabase& database);



} // namespace NintendoSwitch
} // namespace PokemonAutomation
#endif
Loading
Loading