Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,298 @@
/* Party Member Menu Reader
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <format>
#include <utility>
#include "Common/Cpp/Json/JsonArray.h"
#include "Common/Cpp/Json/JsonObject.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/ImageTypes/ImageRGB32.h"
#include "CommonTools/OCR/OCR_SmallDictionaryMatcher.h"
#include "Tests/ImageUnitTest.h"
#include "VideoGames/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.h"
#include "VideoGames/PokemonFRLG/PokemonFRLG_Settings.h"
#include "PokemonFRLG_PartyMemberMenuReader.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonFRLG{


namespace{

// Measured on a capture of the menu (console B, 2026-11-13, a Pidgey's
// SUMMARY / FLY / SWITCH / ITEM / CANCEL), in the game's 240x160 pixels:
// CANCEL's text from y 140 to 150, each row above it 16 higher; the names
// from x 160, the cursor to their left.
constexpr double ROW_STEP = 16;
constexpr double BOTTOM_ROW_TOP = 140;
constexpr double ROW_HEIGHT = 10;
constexpr double TEXT_LEFT = 160;
constexpr double TEXT_WIDTH = 70;
// SUMMARY, four field moves, SWITCH, ITEM and CANCEL.
constexpr size_t MAX_ROWS = 8;
// SWITCH and CANCEL at least, under the field moves (an egg has no ITEM).
constexpr size_t MIN_PLAIN_ROWS_BELOW = 2;

// The field moves (pokefirered party_menu.c, sFieldMoves), as the menu
// writes them (src/data/text/move_names.h, in capitals).
constexpr std::pair<std::string_view, std::string_view> FIELD_MOVE_NAMES[] = {
{"flash", "FLASH"},
{"cut", "CUT"},
{"fly", "FLY"},
{"strength", "STRENGTH"},
{"surf", "SURF"},
{"rock-smash", "ROCK SMASH"},
{"waterfall", "WATERFALL"},
{"teleport", "TELEPORT"},
{"dig", "DIG"},
{"milk-drink", "MILK DRINK"},
{"soft-boiled", "SOFTBOILED"},
{"sweet-scent", "SWEET SCENT"},
};

// The blue of a field move's name: (32, 80, 224) and its darker edges;
// its shadow, (160, 192, 240), is left out.
const std::vector<OCR::TextColorRange>& BLUE_TEXT_FILTERS(){
static const std::vector<OCR::TextColorRange> filters{
{combine_rgb(0, 0, 160), combine_rgb(80, 120, 255)},
{combine_rgb(0, 0, 128), combine_rgb(112, 152, 255)},
{combine_rgb(0, 0, 112), combine_rgb(136, 168, 255)},
};
return filters;
}

class FieldMoveNameMatcher : public OCR::SmallDictionaryMatcher{
public:
static constexpr double MAX_LOG10P = -1.40;
static constexpr double MAX_LOG10P_SPREAD = 0.50;

static const FieldMoveNameMatcher& instance(){
static const FieldMoveNameMatcher matcher;
return matcher;
}

std::string read(Logger& logger, Language language, const ImageViewRGB32& image) const{
if (!languages()[language]){
return "";
}
const ImageRGB32 ready = preprocess_for_ocr(image);
const OCR::StringMatchResult result = match_substring_from_image_multifiltered(
&logger, language, ready, BLUE_TEXT_FILTERS(),
MAX_LOG10P, MAX_LOG10P_SPREAD, 0.01, 0.50
);
return result.results.empty() ? "" : result.results.begin()->second.token;
}

private:
FieldMoveNameMatcher()
: SmallDictionaryMatcher(dictionary())
{}

static JsonObject dictionary(){
JsonObject english;
for (const auto& [slug, name] : FIELD_MOVE_NAMES){
JsonArray names;
names.push_back(std::string(name));
english[std::string(slug)] = std::move(names);
}
JsonObject json;
json["eng"] = std::move(english);
return json;
}
};

enum class RowKind{
NONE, // not a row of the menu's white window
PLAIN, // dark gray text: SUMMARY, SWITCH, ITEM, CANCEL
FIELD, // blue text: a field move
};

// Row `row`, counted up from CANCEL's row 0, as a box of the game screen.
ImageFloatBox row_box(size_t row){
return ImageFloatBox(
TEXT_LEFT / 240, (BOTTOM_ROW_TOP - ROW_STEP * double(row)) / 160, TEXT_WIDTH / 240, ROW_HEIGHT / 160
);
}

bool is_white(uint32_t pixel){
return ((pixel >> 16) & 0xff) > 192 && ((pixel >> 8) & 0xff) > 192 && (pixel & 0xff) > 176;
}
bool is_gray_text(uint32_t pixel){
const int r = (pixel >> 16) & 0xff;
const int g = (pixel >> 8) & 0xff;
const int b = pixel & 0xff;
return r < 136 && g < 136 && b < 136 && std::abs(r - b) < 32 && std::abs(g - b) < 32;
}
bool is_blue_text(uint32_t pixel){
const int r = (pixel >> 16) & 0xff;
const int g = (pixel >> 8) & 0xff;
const int b = pixel & 0xff;
return b > 176 && r < 96 && g < 136;
}
// The light blue shadow of a field move's name, (160, 192, 240).
bool is_blue_shadow(uint32_t pixel){
const int r = (pixel >> 16) & 0xff;
const int g = (pixel >> 8) & 0xff;
const int b = pixel & 0xff;
return r >= 128 && r <= 192 && g >= 160 && g <= 224 && b > 208;
}

RowKind classify_row(const ImageViewRGB32& row){

Check warning on line 144 in SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/Menus/PokemonFRLG_PartyMemberMenuReader.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::NintendoSwitch::PokemonFRLG::classify_row( const ImageViewRGB32 & row): function_lines=34 (advisory above 25, limit 50)
const size_t area = row.width() * row.height();
if (area == 0){
return RowKind::NONE;
}
size_t white = 0;
size_t gray = 0;
size_t blue = 0;
size_t blue_shadow = 0;
for (size_t y = 0; y < row.height(); y++){
for (size_t x = 0; x < row.width(); x++){
const uint32_t pixel = row.pixel(x, y);
white += is_white(pixel) ? 1 : 0;
gray += is_gray_text(pixel) ? 1 : 0;
blue += is_blue_text(pixel) ? 1 : 0;
blue_shadow += is_blue_shadow(pixel) ? 1 : 0;
}
}
// Measured: a name covers 5% to 25% of its row, the window's white
// the most of the rest; the party's slots behind the window are teal
// and blue. A long blue name and its shadow cover about half its row
// (SWEET SCENT: 20% blue, 27% shadow, 47% white), so the shadow
// counts as the window's.
if ((white + blue_shadow) * 2 < area){
return RowKind::NONE;
}
if (blue * 50 >= area && gray * 4 < blue){
return RowKind::FIELD;
}
if (gray * 50 >= area && blue * 4 < gray){
return RowKind::PLAIN;
}
return RowKind::NONE;
}

} // namespace


std::optional<size_t> PartyMemberMenu::row_of(std::string_view move) const{
for (size_t c = 0; c < field_moves.size(); c++){
if (field_moves[c] == move){
return c + 1;
}
}
return std::nullopt;
}

bool PartyMemberMenu::all_read() const{
for (const std::string& move : field_moves){
if (move.empty()){
return false;
}
}
return true;
}

std::optional<PartyMemberMenu> read_party_member_menu(Logger& logger, Language language, const ImageViewRGB32& screen){

Check warning on line 200 in SerialPrograms/Source/VideoGames/PokemonFRLG/Inference/Menus/PokemonFRLG_PartyMemberMenuReader.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::NintendoSwitch::PokemonFRLG::read_party_member_menu( Logger & logger , Language language , const ImageViewRGB32 & screen): function_lines=38 (advisory above 25, limit 50)
const ImageViewRGB32 game = extract_box_reference(screen, GameSettings::instance().GAME_BOX);
// The rows from CANCEL up, to the first that isn't one.
std::vector<RowKind> kinds;
for (size_t row = 0; row < MAX_ROWS; row++){
const RowKind kind = classify_row(extract_box_reference(game, row_box(row)));
if (kind == RowKind::NONE){
break;
}
kinds.push_back(kind);
}
// Plain rows (CANCEL, ITEM, SWITCH), the field moves, then SUMMARY.
size_t plain_below = 0;
while (plain_below < kinds.size() && kinds[plain_below] == RowKind::PLAIN){
plain_below++;
}
size_t fields = 0;
while (plain_below + fields < kinds.size() && kinds[plain_below + fields] == RowKind::FIELD){
fields++;
}
const size_t summary_row = plain_below + fields;
const bool shape = fields == 0
? plain_below >= MIN_PLAIN_ROWS_BELOW + 1 && plain_below == kinds.size()
: plain_below >= MIN_PLAIN_ROWS_BELOW && summary_row + 1 == kinds.size()
&& kinds[summary_row] == RowKind::PLAIN;
if (!shape){
return std::nullopt;
}
PartyMemberMenu menu;
// From the top: the field move under SUMMARY first.
for (size_t c = 0; c < fields; c++){
const size_t row = summary_row - 1 - c;
menu.field_moves.push_back(
FieldMoveNameMatcher::instance().read(logger, language, extract_box_reference(game, row_box(row)))
);
}
return menu;
}


namespace{

// A capture, named <anything>_<language code>.png, and what it shows:
// no member's menu, or one listing `field_moves`.
class Test_PartyMemberMenuReader : public ImageUnitTest{
public:
Test_PartyMemberMenuReader(const std::string& image, std::optional<std::vector<std::string>> field_moves)
: ImageUnitTest(std::format("PokemonFRLG::PartyMemberMenuReader - {}", image), {image})
, m_field_moves(std::move(field_moves))
{}

virtual UnitTestResult run_on_images(
Logger& logger, CancellableScope&, const std::vector<ImageRGB32>& images
) const override{
const std::optional<PartyMemberMenu> menu = read_party_member_menu(logger, Language::English, images[0]);
if (!m_field_moves){
return menu ? UnitTestResult("read a member's menu where none shows.") : UnitTestResult(true);
}
if (!menu){
return "the member's menu isn't read.";
}
if (menu->field_moves != *m_field_moves){
std::string read;
for (const std::string& move : menu->field_moves){
read += std::format("{}'{}'", read.empty() ? "" : ", ", move);
}
return std::format("field moves read: [{}].", read);
}
return true;
}

private:
std::optional<std::vector<std::string>> m_field_moves;
};

} // namespace

void add_tests_PartyMemberMenuReader(UnitTestDatabase& database){
// FireRed on console B, 2026-11-13: the Pidgey's menu, the cursor on
// SUMMARY, FLY in blue under it.
database.add<Test_PartyMemberMenuReader>(
"PokemonFRLG/PartyMemberMenuReader/PidgeyFly_eng.png", std::vector<std::string>{"fly"}
);
// FireRed on console B, 2026-11-13: the Oddish's menu, two field
// moves in blue; SWEET SCENT's row is half name and shadow.
database.add<Test_PartyMemberMenuReader>(
"PokemonFRLG/PartyMemberMenuReader/OddishSweetScentCut_eng.png", std::vector<std::string>{"sweet-scent", "cut"}
);
// The party menu before A: no member's menu.
database.add<Test_PartyMemberMenuReader>(
"PokemonFRLG/PartyMemberMenuReader/PartyList_eng.png", std::nullopt
);
}



} // namespace PokemonFRLG
} // namespace NintendoSwitch
} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/* Party Member Menu Reader
*
* From: https://github.com/PokemonAutomation/
*
* Reads the menu A opens on a member in the party menu: SUMMARY, the
* member's field moves in blue (FLY, CUT, ...), then SWITCH, ITEM and
* CANCEL (pokefirered party_menu.c, SetPartyMonFieldSelectionActions()).
* The window grows upwards from the bottom right corner, one row of
* 16 pixels per action, so its rows are found from CANCEL up.
*
* It knows the field moves' names in English only so far: in another
* language a blue row is never read.
*
*/

#ifndef PokemonAutomation_PokemonFRLG_PartyMemberMenuReader_H
#define PokemonAutomation_PokemonFRLG_PartyMemberMenuReader_H

#include <cstddef>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
#include "Common/Cpp/Logging/AbstractLogger.h"
#include "Common/Cpp/TestRunners/UnitTestDatabase.h"
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
#include "CommonFramework/Language.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonFRLG{


// A member's menu as read.
struct PartyMemberMenu{
// The field moves' slugs ("fly", "rock-smash"), from the top; empty
// for a blue row whose name wasn't read.
std::vector<std::string> field_moves = {};

// The row of `move`, counted from SUMMARY's row 0; nothing if the
// menu doesn't list it.
std::optional<size_t> row_of(std::string_view move) const;
// Whether every blue row's name was read.
bool all_read() const;
};

// The member's menu on `screen`; nothing if the screen doesn't show one
// (no CANCEL at the bottom right, or rows that don't make SUMMARY, field
// moves, then two or more plain rows).
std::optional<PartyMemberMenu> read_party_member_menu(Logger& logger, Language language, const ImageViewRGB32& screen);


// Registers this reader's tests.
void add_tests_PartyMemberMenuReader(UnitTestDatabase& database);


} // namespace PokemonFRLG
} // namespace NintendoSwitch
} // namespace PokemonAutomation
#endif
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include "Inference/Menus/PokemonFRLG_PartyHeldItemDetector.h"
#include "Inference/Menus/PokemonFRLG_PartyMenuDetector.h"
#include "Inference/Menus/PokemonFRLG_SaveCardReader.h"
#include "Inference/Menus/PokemonFRLG_PartyMemberMenuReader.h"
#include "Inference/Menus/PokemonFRLG_SummaryMovesReader.h"
#include "Inference/Menus/PokemonFRLG_TeachScreenReader.h"
#include "Inference/PokemonFRLG_BallReader.h"
Expand Down Expand Up @@ -53,6 +54,7 @@
#include "Programs/AutoStory/PokemonFRLG_AutoStoryPcStorage.h"
#include "Programs/AutoStory/PokemonFRLG_AutoStoryAbilities.h"
#include "Programs/AutoStory/PokemonFRLG_AutoStoryRngLoop.h"
#include "Programs/AutoStory/PokemonFRLG_AutoStoryPartyRoster.h"
#include "Programs/AutoStory/PokemonFRLG_AutoStoryRows_Tests.h"
#include "Programs/AutoStory/PokemonFRLG_AutoStoryHealing.h"
#include "Programs/AutoStory/PokemonFRLG_AutoStoryRows12_Tests.h"
Expand Down Expand Up @@ -88,6 +90,7 @@ static void add_tests_AutoStoryReaders(UnitTestDatabase& database){
add_tests_ItemListReader(database);
add_tests_OptionMenuReader(database);
add_tests_SaveCardReader(database);
add_tests_PartyMemberMenuReader(database);
add_tests_SummaryMovesReader(database);
add_tests_TeachScreenReader(database);
add_tests_AutoStoryScreenEvents(database);
Expand All @@ -111,6 +114,7 @@ static void add_tests_AutoStoryRows(UnitTestDatabase& database){
add_tests_AutoStoryVermilionGym(database);
add_tests_AutoStoryCeladon(database);
add_tests_AutoStorySnorlax(database);
add_tests_AutoStoryPartyRoster(database);
add_tests_AutoStorySlope(database);
add_tests_AutoStorySafari(database);
add_tests_AutoStoryFuchsiaGym(database);
Expand Down
Loading
Loading